From fba84d449ab0c70cc2d38b068b9e0aa74738cef1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 8 Sep 2024 16:28:22 -0700 Subject: [PATCH 001/311] add tensor.Indexed, rename table IndexView -> Indexed --- base/randx/dists_test.go | 10 +- core/list.go | 2 +- plot/plotcore/ploteditor.go | 22 +- plot/plotcore/tablexy.go | 10 +- plot/plotcore/typegen.go | 4 +- tensor/README.md | 2 +- tensor/base.go | 87 ++---- tensor/bits.go | 85 ++---- tensor/cmd/tablecat/tablecat.go | 2 +- tensor/databrowser/datatab.go | 4 +- tensor/examples/dataproc/README.md | 2 +- tensor/examples/dataproc/dataproc.go | 2 +- tensor/indexed.go | 275 ++++++++++++++++++ tensor/io.go | 5 +- tensor/number.go | 24 +- tensor/stats/clust/clust_test.go | 2 +- tensor/stats/glm/README.md | 2 +- tensor/stats/glm/glm.go | 6 +- tensor/stats/pca/covar.go | 10 +- tensor/stats/pca/pca.go | 12 +- tensor/stats/pca/pca_test.go | 2 +- tensor/stats/pca/svd.go | 12 +- tensor/stats/pca/svd_test.go | 2 +- tensor/stats/simat/simat.go | 8 +- tensor/stats/simat/simat_test.go | 2 +- tensor/stats/split/agg_test.go | 4 +- tensor/stats/split/group.go | 12 +- tensor/stats/split/random.go | 2 +- tensor/stats/split/random_test.go | 2 +- tensor/stats/stats/README.md | 8 +- tensor/stats/stats/desc.go | 10 +- tensor/stats/stats/doc.go | 2 +- tensor/stats/stats/if.go | 24 +- tensor/stats/stats/indexview.go | 184 ++++++------ tensor/stats/stats/indexview_test.go | 4 +- tensor/stats/stats/quantiles.go | 8 +- tensor/string.go | 24 +- tensor/table/README.md | 12 +- tensor/table/{indexview.go => indexed.go} | 271 ++++++----------- tensor/table/io.go | 10 +- tensor/table/splits.go | 12 +- tensor/table/typegen.go | 2 +- tensor/tensor.go | 35 +-- tensor/tensor_test.go | 18 +- tensor/tensorcore/table.go | 12 +- .../cogentcore_org-core-tensor-table.go | 7 +- 46 files changed, 686 insertions(+), 571 deletions(-) create mode 100644 tensor/indexed.go rename tensor/table/{indexview.go => indexed.go} (60%) diff --git a/base/randx/dists_test.go b/base/randx/dists_test.go index 059dd031d8..f2c8764873 100644 --- a/base/randx/dists_test.go +++ b/base/randx/dists_test.go @@ -27,7 +27,7 @@ func TestGaussianGen(t *testing.T) { vl := GaussianGen(mean, sig) dt.SetFloat("Val", i, vl) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) desc := stats.DescAll(ix) meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] @@ -63,7 +63,7 @@ func TestBinomialGen(t *testing.T) { vl := BinomialGen(n, p) dt.SetFloat("Val", i, vl) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) desc := stats.DescAll(ix) meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] @@ -108,7 +108,7 @@ func TestPoissonGen(t *testing.T) { vl := PoissonGen(lambda) dt.SetFloat("Val", i, vl) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) desc := stats.DescAll(ix) meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] @@ -154,7 +154,7 @@ func TestGammaGen(t *testing.T) { vl := GammaGen(alpha, beta) dt.SetFloat("Val", i, vl) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) desc := stats.DescAll(ix) meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] @@ -190,7 +190,7 @@ func TestBetaGen(t *testing.T) { vl := BetaGen(alpha, beta) dt.SetFloat("Val", i, vl) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) desc := stats.DescAll(ix) meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] diff --git a/core/list.go b/core/list.go index 2125877571..dadebd91fa 100644 --- a/core/list.go +++ b/core/list.go @@ -90,7 +90,7 @@ type Lister interface { // SliceIndex returns the logical slice index: si = i + StartIndex, // the actual value index vi into the slice value (typically = si), // which can be different if there is an index indirection as in - // tensorcore table.IndexView), and a bool that is true if the + // tensorcore table.Indexed), and a bool that is true if the // index is beyond the available data and is thus invisible, // given the row index provided. SliceIndex(i int) (si, vi int, invis bool) diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index f0916e8952..70f09055c5 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -32,14 +32,14 @@ import ( ) // PlotEditor is a widget that provides an interactive 2D plot -// of selected columns of tabular data, represented by a [table.IndexView] into +// of selected columns of tabular data, represented by a [table.Indexed] into // a [table.Table]. Other types of tabular data can be converted into this format. // The user can change various options for the plot and also modify the underlying data. type PlotEditor struct { //types:add core.Frame // table is the table of data being plotted. - table *table.IndexView + table *table.Indexed // Options are the overall plot options. Options PlotOptions @@ -67,7 +67,7 @@ func (pl *PlotEditor) CopyFieldsFrom(frm tree.Node) { fr := frm.(*PlotEditor) pl.Frame.CopyFieldsFrom(&fr.Frame) pl.Options = fr.Options - pl.setIndexView(fr.table) + pl.setIndexed(fr.table) mx := min(len(pl.Columns), len(fr.Columns)) for i := 0; i < mx; i++ { *pl.Columns[i] = *fr.Columns[i] @@ -131,11 +131,11 @@ func (pl *PlotEditor) Init() { }) } -// setIndexView sets the table to view and does Update +// setIndexed sets the table to view and does Update // to update the Column list, which will also trigger a Layout // and updating of the plot on next render pass. // This is safe to call from a different goroutine. -func (pl *PlotEditor) setIndexView(tab *table.IndexView) *PlotEditor { +func (pl *PlotEditor) setIndexed(tab *table.Indexed) *PlotEditor { pl.table = tab pl.Update() return pl @@ -146,7 +146,7 @@ func (pl *PlotEditor) setIndexView(tab *table.IndexView) *PlotEditor { // and updating of the plot on next render pass. // This is safe to call from a different goroutine. func (pl *PlotEditor) SetTable(tab *table.Table) *PlotEditor { - pl.table = table.NewIndexView(tab) + pl.table = table.NewIndexed(tab) pl.Update() return pl } @@ -271,9 +271,9 @@ func (pl *PlotEditor) xLabel() string { return "X" } -// GoUpdatePlot updates the display based on current IndexView into table. +// GoUpdatePlot updates the display based on current Indexed into table. // This version can be called from goroutines. It does Sequential() on -// the [table.IndexView], under the assumption that it is used for tracking a +// the [table.Indexed], under the assumption that it is used for tracking a // the latest updates of a running process. func (pl *PlotEditor) GoUpdatePlot() { if pl == nil || pl.This == nil { @@ -292,8 +292,8 @@ func (pl *PlotEditor) GoUpdatePlot() { pl.Scene.AsyncUnlock() } -// UpdatePlot updates the display based on current IndexView into table. -// It does not automatically update the [table.IndexView] unless it is +// UpdatePlot updates the display based on current Indexed into table. +// It does not automatically update the [table.Indexed] unless it is // nil or out date. func (pl *PlotEditor) UpdatePlot() { if pl == nil || pl.This == nil { @@ -377,7 +377,7 @@ func (pl *PlotEditor) configPlot(plt *plot.Plot) { } // plotXAxis processes the XAxis and returns its index -func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.IndexView) (xi int, xview *table.IndexView, err error) { +func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.Indexed) (xi int, xview *table.Indexed, err error) { xi, err = ixvw.Table.ColumnIndex(pl.Options.XAxis) if err != nil { // log.Println("plot.PlotXAxis: " + err.Error()) diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index 93d22fdc3e..6bcf32406d 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -21,7 +21,7 @@ import ( type tableXY struct { // the index view of data table to plot from - table *table.IndexView + table *table.Indexed // the indexes of the tensor columns to use for the X and Y data, respectively xColumn, yColumn int @@ -47,18 +47,18 @@ var _ plot.Valuer = &tableXY{} var _ plot.Labeler = &tableXY{} var _ plots.YErrorer = &tableXY{} -// newTableXY returns a new XY plot view onto the given IndexView of table.Table (makes a copy), +// newTableXY returns a new XY plot view onto the given Indexed of table.Table (makes a copy), // from given column indexes, and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. -func newTableXY(dt *table.IndexView, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { +func newTableXY(dt *table.Indexed, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { txy := &tableXY{table: dt.Clone(), xColumn: xcol, yColumn: ycol, xIndex: xtsrIndex, yIndex: ytsrIndex, yRange: yrng} return txy, txy.validate() } -// newTableXYName returns a new XY plot view onto the given IndexView of table.Table (makes a copy), +// newTableXYName returns a new XY plot view onto the given Indexed of table.Table (makes a copy), // from given column name and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. -func newTableXYName(dt *table.IndexView, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { +func newTableXYName(dt *table.Indexed, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { yi, err := dt.Table.ColumnIndex(ycol) if errors.Log(err) != nil { return nil, err diff --git a/plot/plotcore/typegen.go b/plot/plotcore/typegen.go index 9a925c9750..44d974329f 100644 --- a/plot/plotcore/typegen.go +++ b/plot/plotcore/typegen.go @@ -26,11 +26,11 @@ func NewPlot(parent ...tree.Node) *Plot { return tree.New[Plot](parent...) } // documents or other cases where the overall plot size will be small. func (t *Plot) SetScale(v float32) *Plot { t.Scale = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.IndexView] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "Options", Doc: "Options are the overall plot options."}, {Name: "Columns", Doc: "Columns are the options for each column of the table."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Indexed] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "Options", Doc: "Options are the overall plot options."}, {Name: "Columns", Doc: "Columns are the options for each column of the table."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) // NewPlotEditor returns a new [PlotEditor] with the given optional parent: // PlotEditor is a widget that provides an interactive 2D plot -// of selected columns of tabular data, represented by a [table.IndexView] into +// of selected columns of tabular data, represented by a [table.Indexed] into // a [table.Table]. Other types of tabular data can be converted into this format. // The user can change various options for the plot and also modify the underlying data. func NewPlotEditor(parent ...tree.Node) *PlotEditor { return tree.New[PlotEditor](parent...) } diff --git a/tensor/README.md b/tensor/README.md index 4dadca823b..31448b1344 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -2,7 +2,7 @@ Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used `numpy` and `pandas` libraries in python, and the commercial MATLAB framework. -* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common row dimension as the outer-most dimension of each tensor. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `IndexView` that provides an indexed view into the rows of the table for highly efficient filtering and sorting of data. +* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common row dimension as the outer-most dimension of each tensor. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `Indexed` that provides an indexed view into the rows of the table for highly efficient filtering and sorting of data. Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including the plotcore. diff --git a/tensor/base.go b/tensor/base.go index 8bad3301f1..28434dcf31 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -10,34 +10,39 @@ import ( "reflect" "unsafe" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" "cogentcore.org/core/base/slicesx" ) // Base is an n-dim array of float64s. type Base[T any] struct { - Shp Shape + shape Shape Values []T - Meta map[string]string + Meta metadata.Data } // Shape returns a pointer to the shape that fully parametrizes the tensor shape -func (tsr *Base[T]) Shape() *Shape { return &tsr.Shp } +func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } + +// Metadata returns the metadata for this tensor, which can be used +// to encode plotting options, etc. +func (tsr *Base[T]) Metadata() *metadata.Data { return &tsr.Meta } // Len returns the number of elements in the tensor (product of shape dimensions). -func (tsr *Base[T]) Len() int { return tsr.Shp.Len() } +func (tsr *Base[T]) Len() int { return tsr.shape.Len() } // NumDims returns the total number of dimensions. -func (tsr *Base[T]) NumDims() int { return tsr.Shp.NumDims() } +func (tsr *Base[T]) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension -func (tsr *Base[T]) DimSize(dim int) int { return tsr.Shp.DimSize(dim) } +func (tsr *Base[T]) DimSize(dim int) int { return tsr.shape.DimSize(dim) } // RowCellSize returns the size of the outer-most Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. func (tsr *Base[T]) RowCellSize() (rows, cells int) { - return tsr.Shp.RowCellSize() + return tsr.shape.RowCellSize() } // DataType returns the type of the data elements in the tensor. @@ -56,14 +61,14 @@ func (tsr *Base[T]) Bytes() []byte { return slicesx.ToBytes(tsr.Values) } -func (tsr *Base[T]) Value(i []int) T { j := tsr.Shp.Offset(i); return tsr.Values[j] } +func (tsr *Base[T]) Value(i []int) T { j := tsr.shape.Offset(i); return tsr.Values[j] } func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[i] } -func (tsr *Base[T]) Set(i []int, val T) { j := tsr.Shp.Offset(i); tsr.Values[j] = val } +func (tsr *Base[T]) Set(i []int, val T) { j := tsr.shape.Offset(i); tsr.Values[j] = val } func (tsr *Base[T]) Set1D(i int, val T) { tsr.Values[i] = val } // SetShape sets the shape params, resizing backing storage appropriately func (tsr *Base[T]) SetShape(sizes []int, names ...string) { - tsr.Shp.SetShape(sizes, names...) + tsr.shape.SetShape(sizes, names...) nln := tsr.Len() tsr.Values = slicesx.SetLength(tsr.Values, nln) } @@ -71,9 +76,9 @@ func (tsr *Base[T]) SetShape(sizes []int, names ...string) { // SetNumRows sets the number of rows (outer-most dimension) in a RowMajor organized tensor. func (tsr *Base[T]) SetNumRows(rows int) { rows = max(1, rows) // must be > 0 - _, cells := tsr.Shp.RowCellSize() + _, cells := tsr.shape.RowCellSize() nln := rows * cells - tsr.Shp.Sizes[0] = rows + tsr.shape.Sizes[0] = rows tsr.Values = slicesx.SetLength(tsr.Values, nln) } @@ -90,29 +95,29 @@ func (tsr *Base[T]) subSpaceImpl(offs []int) *Base[T] { return nil } stsr := &Base[T]{} - stsr.SetShape(tsr.Shp.Sizes[od:], tsr.Shp.Names[od:]...) + stsr.SetShape(tsr.shape.Sizes[od:], tsr.shape.Names[od:]...) sti := make([]int, nd) copy(sti, offs) - stoff := tsr.Shp.Offset(sti) + stoff := tsr.shape.Offset(sti) sln := stsr.Len() stsr.Values = tsr.Values[stoff : stoff+sln] return stsr } func (tsr *Base[T]) StringValue(i []int) string { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) return reflectx.ToString(tsr.Values[j]) } func (tsr *Base[T]) String1D(off int) string { return reflectx.ToString(tsr.Values[off]) } func (tsr *Base[T]) StringRowCell(row, cell int) string { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() return reflectx.ToString(tsr.Values[row*sz+cell]) } // Label satisfies the core.Labeler interface for a summary description of the tensor func (tsr *Base[T]) Label() string { - return fmt.Sprintf("Tensor: %s", tsr.Shp.String()) + return fmt.Sprintf("Tensor: %s", tsr.shape.String()) } // Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the @@ -123,7 +128,7 @@ func (tsr *Base[T]) Dims() (r, c int) { log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") return 0, 0 } - return tsr.Shp.DimSize(nd - 2), tsr.Shp.DimSize(nd - 1) + return tsr.shape.DimSize(nd - 2), tsr.shape.DimSize(nd - 1) } // Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric @@ -134,11 +139,11 @@ func (tsr *Base[T]) Symmetric() (r int) { log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") return 0 } - if tsr.Shp.DimSize(nd-2) != tsr.Shp.DimSize(nd-1) { + if tsr.shape.DimSize(nd-2) != tsr.shape.DimSize(nd-1) { log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") return 0 } - return tsr.Shp.DimSize(nd - 1) + return tsr.shape.DimSize(nd - 1) } // SymmetricDim returns the number of rows/columns in the matrix. @@ -148,47 +153,9 @@ func (tsr *Base[T]) SymmetricDim() int { log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") return 0 } - if tsr.Shp.DimSize(nd-2) != tsr.Shp.DimSize(nd-1) { + if tsr.shape.DimSize(nd-2) != tsr.shape.DimSize(nd-1) { log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") return 0 } - return tsr.Shp.DimSize(nd - 1) -} - -// SetMetaData sets a key=value meta data (stored as a map[string]string). -// For TensorGrid display: top-zero=+/-, odd-row=+/-, image=+/-, -// min, max set fixed min / max values, background=color -func (tsr *Base[T]) SetMetaData(key, val string) { - if tsr.Meta == nil { - tsr.Meta = make(map[string]string) - } - tsr.Meta[key] = val -} - -// MetaData retrieves value of given key, bool = false if not set -func (tsr *Base[T]) MetaData(key string) (string, bool) { - if tsr.Meta == nil { - return "", false - } - val, ok := tsr.Meta[key] - return val, ok -} - -// MetaDataMap returns the underlying map used for meta data -func (tsr *Base[T]) MetaDataMap() map[string]string { - return tsr.Meta -} - -// CopyMetaData copies meta data from given source tensor -func (tsr *Base[T]) CopyMetaData(frm Tensor) { - fmap := frm.MetaDataMap() - if len(fmap) == 0 { - return - } - if tsr.Meta == nil { - tsr.Meta = make(map[string]string) - } - for k, v := range fmap { - tsr.Meta[k] = v - } + return tsr.shape.DimSize(nd - 1) } diff --git a/tensor/bits.go b/tensor/bits.go index 76a15bfd39..ebbce80fc9 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -10,6 +10,7 @@ import ( "reflect" "strings" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" "cogentcore.org/core/base/slicesx" "cogentcore.org/core/tensor/bitslice" @@ -19,9 +20,9 @@ import ( // Bits is a tensor of bits backed by a bitslice.Slice for efficient storage // of binary data type Bits struct { - Shp Shape + shape Shape Values bitslice.Slice - Meta map[string]string + Meta metadata.Data } // NewBits returns a new n-dimensional tensor of bit values @@ -37,7 +38,7 @@ func NewBits(sizes []int, names ...string) *Bits { // using given shape. func NewBitsShape(shape *Shape) *Bits { tsr := &Bits{} - tsr.Shp.CopyShape(shape) + tsr.shape.CopyShape(shape) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr } @@ -77,36 +78,40 @@ func (tsr *Bits) Bytes() []byte { } // Shape returns a pointer to the shape that fully parametrizes the tensor shape -func (tsr *Bits) Shape() *Shape { return &tsr.Shp } +func (tsr *Bits) Shape() *Shape { return &tsr.shape } + +// Metadata returns the metadata for this tensor, which can be used +// to encode plotting options, etc. +func (tsr *Bits) Metadata() *metadata.Data { return &tsr.Meta } // Len returns the number of elements in the tensor (product of shape dimensions). -func (tsr *Bits) Len() int { return tsr.Shp.Len() } +func (tsr *Bits) Len() int { return tsr.shape.Len() } // NumDims returns the total number of dimensions. -func (tsr *Bits) NumDims() int { return tsr.Shp.NumDims() } +func (tsr *Bits) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension -func (tsr *Bits) DimSize(dim int) int { return tsr.Shp.DimSize(dim) } +func (tsr *Bits) DimSize(dim int) int { return tsr.shape.DimSize(dim) } // RowCellSize returns the size of the outer-most Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. func (tsr *Bits) RowCellSize() (rows, cells int) { - return tsr.Shp.RowCellSize() + return tsr.shape.RowCellSize() } // Value returns value at given tensor index -func (tsr *Bits) Value(i []int) bool { j := int(tsr.Shp.Offset(i)); return tsr.Values.Index(j) } +func (tsr *Bits) Value(i []int) bool { j := int(tsr.shape.Offset(i)); return tsr.Values.Index(j) } // Value1D returns value at given tensor 1D (flat) index func (tsr *Bits) Value1D(i int) bool { return tsr.Values.Index(i) } -func (tsr *Bits) Set(i []int, val bool) { j := int(tsr.Shp.Offset(i)); tsr.Values.Set(j, val) } +func (tsr *Bits) Set(i []int, val bool) { j := int(tsr.shape.Offset(i)); tsr.Values.Set(j, val) } func (tsr *Bits) Set1D(i int, val bool) { tsr.Values.Set(i, val) } // SetShape sets the shape params, resizing backing storage appropriately func (tsr *Bits) SetShape(sizes []int, names ...string) { - tsr.Shp.SetShape(sizes, names...) + tsr.shape.SetShape(sizes, names...) nln := tsr.Len() tsr.Values.SetLen(nln) } @@ -114,9 +119,9 @@ func (tsr *Bits) SetShape(sizes []int, names ...string) { // SetNumRows sets the number of rows (outer-most dimension) in a RowMajor organized tensor. func (tsr *Bits) SetNumRows(rows int) { rows = max(1, rows) // must be > 0 - _, cells := tsr.Shp.RowCellSize() + _, cells := tsr.shape.RowCellSize() nln := rows * cells - tsr.Shp.Sizes[0] = rows + tsr.shape.Sizes[0] = rows tsr.Values.SetLen(nln) } @@ -126,23 +131,23 @@ func (tsr *Bits) SubSpace(offs []int) Tensor { } func (tsr *Bits) Float(i []int) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) return BoolToFloat64(tsr.Values.Index(j)) } func (tsr *Bits) SetFloat(i []int, val float64) { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values.Set(j, Float64ToBool(val)) } func (tsr *Bits) StringValue(i []int) string { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) return reflectx.ToString(tsr.Values.Index(j)) } func (tsr *Bits) SetString(i []int, val string) { if bv, err := reflectx.ToBool(val); err == nil { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values.Set(j, bv) } } @@ -203,45 +208,7 @@ func (tsr *Bits) SetStringRowCell(row, cell int, val string) { // Label satisfies the core.Labeler interface for a summary description of the tensor func (tsr *Bits) Label() string { - return fmt.Sprintf("tensor.Bits: %s", tsr.Shp.String()) -} - -// SetMetaData sets a key=value meta data (stored as a map[string]string). -// For TensorGrid display: top-zero=+/-, odd-row=+/-, image=+/-, -// min, max set fixed min / max values, background=color -func (tsr *Bits) SetMetaData(key, val string) { - if tsr.Meta == nil { - tsr.Meta = make(map[string]string) - } - tsr.Meta[key] = val -} - -// MetaData retrieves value of given key, bool = false if not set -func (tsr *Bits) MetaData(key string) (string, bool) { - if tsr.Meta == nil { - return "", false - } - val, ok := tsr.Meta[key] - return val, ok -} - -// MetaDataMap returns the underlying map used for meta data -func (tsr *Bits) MetaDataMap() map[string]string { - return tsr.Meta -} - -// CopyMetaData copies meta data from given source tensor -func (tsr *Bits) CopyMetaData(frm Tensor) { - fmap := frm.MetaDataMap() - if len(fmap) == 0 { - return - } - if tsr.Meta == nil { - tsr.Meta = make(map[string]string) - } - for k, v := range fmap { - tsr.Meta[k] = v - } + return fmt.Sprintf("tensor.Bits: %s", tsr.shape.String()) } // Range is not applicable to Bits tensor @@ -263,7 +230,7 @@ func (tsr *Bits) SetZeros() { // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). func (tsr *Bits) Clone() Tensor { - csr := NewBitsShape(&tsr.Shp) + csr := NewBitsShape(&tsr.shape) csr.Values = tsr.Values.Clone() return csr } @@ -337,9 +304,9 @@ func (tsr *Bits) String() string { b.WriteString(str) b.WriteString("\n") oddRow := true - rows, cols, _, _ := Projection2DShape(&tsr.Shp, oddRow) + rows, cols, _, _ := Projection2DShape(&tsr.shape, oddRow) for r := 0; r < rows; r++ { - rc, _ := Projection2DCoords(&tsr.Shp, oddRow, r, 0) + rc, _ := Projection2DCoords(&tsr.shape, oddRow, r, 0) b.WriteString(fmt.Sprintf("%v: ", rc)) for c := 0; c < cols; c++ { vl := Projection2DValue(tsr, oddRow, r, c) diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index 5c3273257f..77092ef86f 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -145,7 +145,7 @@ func AvgByColumn(files []string, column string) { fmt.Printf("File %v empty\n", fn) continue } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) var spl *table.Splits if column == "" { spl = split.All(ix) diff --git a/tensor/databrowser/datatab.go b/tensor/databrowser/datatab.go index c24f1243b6..5ddd6e431f 100644 --- a/tensor/databrowser/datatab.go +++ b/tensor/databrowser/datatab.go @@ -28,9 +28,9 @@ func NewTab[T any](br *Browser, label string, mkfun func(tab *core.Frame) T) T { } // NewTabTensorTable creates a tab with a tensorcore.Table widget -// to view given table.Table, using its own table.IndexView as tv.Table. +// to view given table.Table, using its own table.Indexed as tv.Table. // Use tv.Table.Table to get the underlying *table.Table -// Use tv.Table.Sequential to update the IndexView to view +// Use tv.Table.Sequential to update the Indexed to view // all of the rows when done updating the Table, and then call br.Update() func (br *Browser) NewTabTensorTable(label string, dt *table.Table) *tensorcore.Table { tv := NewTab[*tensorcore.Table](br, label, func(tab *core.Frame) *tensorcore.Table { diff --git a/tensor/examples/dataproc/README.md b/tensor/examples/dataproc/README.md index a12d4d1266..cb757dd23b 100644 --- a/tensor/examples/dataproc/README.md +++ b/tensor/examples/dataproc/README.md @@ -1,6 +1,6 @@ Build and run this `main` package to see a full demo of how to use this system for data analysis, paralleling the example in [Python Data Science](https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html) using pandas, to see directly how that translates into this framework. -Most of the code is in the `AnalyzePlanets` function, which opens a .csv file, and then uses a number of `IndexView` views of the data to perform various analyses as shown in the GUI tables. Click on the tabs at the very top of the window to see the various analyzed versions of the data shown in the first tab. +Most of the code is in the `AnalyzePlanets` function, which opens a .csv file, and then uses a number of `Indexed` views of the data to perform various analyses as shown in the GUI tables. Click on the tabs at the very top of the window to see the various analyzed versions of the data shown in the first tab. You can also click on headers of the columns to sort by those columns (toggles between ascending and descending), diff --git a/tensor/examples/dataproc/dataproc.go b/tensor/examples/dataproc/dataproc.go index d5455b695d..7e146bd0b6 100644 --- a/tensor/examples/dataproc/dataproc.go +++ b/tensor/examples/dataproc/dataproc.go @@ -52,7 +52,7 @@ func AnalyzePlanets() { Planets = table.NewTable("planets") Planets.OpenFS(csv, "planets.csv", table.Comma) - PlanetsAll := table.NewIndexView(Planets) // full original data + PlanetsAll := table.NewIndexed(Planets) // full original data PlanetsDesc = stats.DescAll(PlanetsAll) // individually excludes Null values in each col, but not row-wise PlanetsNNDesc = stats.DescAll(PlanetsAll) // standard descriptive stats for row-wise non-nulls diff --git a/tensor/indexed.go b/tensor/indexed.go new file mode 100644 index 0000000000..9125dbb23a --- /dev/null +++ b/tensor/indexed.go @@ -0,0 +1,275 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "cmp" + "errors" + "math/rand" + "slices" + "sort" + "strings" +) + +// Indexed is an indexed wrapper around a tensor.Tensor that provides a +// specific view onto the Tensor defined by the set of indexes, which +// apply to the outer-most ("row") dimension. +// This provides an efficient way of sorting and filtering a tensor by only +// updating the indexes while doing nothing to the Tensor itself. +// To produce a tensor that has data actually organized according to the +// indexed order, call the NewTensor method. +// Indexed views on a tensor can also be organized together as Splits +// of the tensor rows, e.g., by grouping values along a given column. +type Indexed struct { //types:add + + // Tensor that we are an indexed view onto + Tensor Tensor + + // current indexes into Tensor + Indexes []int +} + +// NewIndexed returns a new Indexed based on given tensor. +// If a list of indexes is passed, then our indexes are initialized +// as a copy of those. This is used e.g. from a Indext Table column. +// Otherwise it is initialized with sequential indexes. +func NewIndexed(tsr Tensor, idxs ...[]int) *Indexed { + ix := &Indexed{} + if len(idxs) == 1 { + ix.Tensor = tsr + ix.Indexes = slices.Clone(idxs[0]) + } else { + ix.SetTensor(tsr) + } + return ix +} + +// SetTensor sets as indexes into given tensor with sequential initial indexes +func (ix *Indexed) SetTensor(tsr Tensor) { + ix.Tensor = tsr + ix.Sequential() +} + +// DeleteInvalid deletes all invalid indexes from the list. +// Call this if rows (could) have been deleted from tensor. +func (ix *Indexed) DeleteInvalid() { + if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + ix.Indexes = nil + return + } + ni := ix.Len() + for i := ni - 1; i >= 0; i-- { + if ix.Indexes[i] >= ix.Tensor.DimSize(0) { + ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + } + } +} + +// Sequential sets indexes to sequential row-wise indexes into tensor. +func (ix *Indexed) Sequential() { //types:add + if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + ix.Indexes = nil + return + } + ix.Indexes = make([]int, ix.Tensor.DimSize(0)) + for i := range ix.Indexes { + ix.Indexes[i] = i + } +} + +// Permuted sets indexes to a permuted order -- if indexes already exist +// then existing list of indexes is permuted, otherwise a new set of +// permuted indexes are generated +func (ix *Indexed) Permuted() { + if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + ix.Indexes = nil + return + } + if len(ix.Indexes) == 0 { + ix.Indexes = rand.Perm(ix.Tensor.DimSize(0)) + } else { + rand.Shuffle(len(ix.Indexes), func(i, j int) { + ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] + }) + } +} + +// AddIndex adds a new index to the list +func (ix *Indexed) AddIndex(idx int) { + ix.Indexes = append(ix.Indexes, idx) +} + +const ( + // Ascending specifies an ascending sort direction for tensor Sort routines + Ascending = true + + // Descending specifies a descending sort direction for tensor Sort routines + Descending = false +) + +// SortFunc sorts the indexes into 1D Tensor using given compare function. +// Returns an error if called on a higher-dimensional tensor. +// The compare function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) error { + if ix.Tensor.NumDims() > 1 { + return errors.New("tensor Sorting is only for 1D tensors") + } + slices.SortFunc(ix.Indexes, func(a, b int) int { + return cmp(ix.Tensor, ix.Indexes[a], ix.Indexes[b]) + }) + return nil +} + +// SortIndexes sorts the indexes into our Tensor directly in +// numerical order, producing the native ordering, while preserving +// any filtering that might have occurred. +func (ix *Indexed) SortIndexes() { + sort.Ints(ix.Indexes) +} + +// Sort compare function for string values. +func CompareStrings(a, b string, ascending bool) int { + cmp := strings.Compare(a, b) + if !ascending { + cmp = -cmp + } + return cmp +} + +func CompareNumbers(a, b float64, ascending bool) int { + cmp := cmp.Compare(a, b) + if !ascending { + cmp = -cmp + } + return cmp +} + +// Sort does default alpha or numeric sort of 1D tensor based on data type. +// Returns an error if called on a higher-dimensional tensor. +func (ix *Indexed) Sort(ascending bool) error { + if ix.Tensor.NumDims() > 1 { + return errors.New("tensor Sorting is only for 1D tensors") + } + if ix.Tensor.IsString() { + ix.SortFunc(func(tsr Tensor, i, j int) int { + return CompareStrings(tsr.String1D(i), tsr.String1D(j), ascending) + }) + } else { + ix.SortFunc(func(tsr Tensor, i, j int) int { + return CompareNumbers(tsr.Float1D(i), tsr.Float1D(j), ascending) + }) + } + return nil +} + +// SortStableFunc stably sorts the indexes of 1D Tensor using given compare function. +// The compare function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +// It is *essential* that it always returns 0 when the two are equal +// for the stable function to actually work. +func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { + slices.SortStableFunc(ix.Indexes, func(a, b int) int { + return cmp(ix.Tensor, ix.Indexes[a], ix.Indexes[b]) + }) +} + +// SortStable does default alpha or numeric stable sort +// of 1D tensor based on data type. +// Returns an error if called on a higher-dimensional tensor. +func (ix *Indexed) SortStable(ascending bool) error { + if ix.Tensor.NumDims() > 1 { + return errors.New("tensor Sorting is only for 1D tensors") + } + if ix.Tensor.IsString() { + ix.SortStableFunc(func(tsr Tensor, i, j int) int { + return CompareStrings(tsr.String1D(i), tsr.String1D(j), ascending) + }) + } else { + ix.SortStableFunc(func(tsr Tensor, i, j int) int { + return CompareNumbers(tsr.Float1D(i), tsr.Float1D(j), ascending) + }) + } + return nil +} + +// FilterFunc is a function used for filtering that returns +// true if Tensor row should be included in the current filtered +// view of the tensor, and false if it should be removed. +type FilterFunc func(tsr Tensor, row int) bool + +// Filter filters the indexes into our Tensor using given Filter function. +// The Filter function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { + sz := len(ix.Indexes) + for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering + if !filterer(ix.Tensor, ix.Indexes[i]) { // delete + ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + } + } +} + +// NewTensor returns a new tensor with column data organized according to +// the indexes. +func (ix *Indexed) NewTensor() Tensor { + rows := len(ix.Indexes) + nt := ix.Tensor.Clone() + nt.SetNumRows(rows) + return nt +} + +// Clone returns a copy of the current index view with its own index memory. +func (ix *Indexed) Clone() *Indexed { + nix := &Indexed{} + nix.CopyFrom(ix) + return nix +} + +// CopyFrom copies from given other Indexed (we have our own unique copy of indexes). +func (ix *Indexed) CopyFrom(oix *Indexed) { + ix.Tensor = oix.Tensor + ix.Indexes = slices.Clone(oix.Indexes) +} + +// AddRows adds n rows to end of underlying Tensor, and to the indexes in this view +func (ix *Indexed) AddRows(n int) { //types:add + stidx := ix.Tensor.DimSize(0) + ix.Tensor.SetNumRows(stidx + n) + for i := stidx; i < stidx+n; i++ { + ix.Indexes = append(ix.Indexes, i) + } +} + +// InsertRows adds n rows to end of underlying Tensor, and to the indexes starting at +// given index in this view +func (ix *Indexed) InsertRows(at, n int) { + stidx := ix.Tensor.DimSize(0) + ix.Tensor.SetNumRows(stidx + n) + nw := make([]int, n, n+len(ix.Indexes)-at) + for i := 0; i < n; i++ { + nw[i] = stidx + i + } + ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) +} + +// DeleteRows deletes n rows of indexes starting at given index in the list of indexes +func (ix *Indexed) DeleteRows(at, n int) { + ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) +} + +// Len returns the length of the index list +func (ix *Indexed) Len() int { + return len(ix.Indexes) +} + +// Swap switches the indexes for i and j +func (ix *Indexed) Swap(i, j int) { + ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] +} diff --git a/tensor/io.go b/tensor/io.go index 00b5548fd2..78bfa69f62 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -11,6 +11,7 @@ import ( "os" "strconv" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/core" ) @@ -53,8 +54,8 @@ func OpenCSV(tsr Tensor, filename core.Filename, delim rune) error { // Reading just grabs all values and doesn't care about shape. func WriteCSV(tsr Tensor, w io.Writer, delim rune) error { prec := -1 - if ps, ok := tsr.MetaData("precision"); ok { - prec, _ = strconv.Atoi(ps) + if ps, err := metadata.Get[int](*tsr.Metadata(), "precision"); err == nil { + prec = ps } cw := csv.NewWriter(w) if delim != 0 { diff --git a/tensor/number.go b/tensor/number.go index 91a6adb142..da84cc1a51 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -79,7 +79,7 @@ func NewNumber[T num.Number](sizes []int, names ...string) *Number[T] { // using given shape. func NewNumberShape[T num.Number](shape *Shape) *Number[T] { tsr := &Number[T]{} - tsr.Shp.CopyShape(shape) + tsr.shape.CopyShape(shape) tsr.Values = make([]T, tsr.Len()) return tsr } @@ -89,20 +89,20 @@ func (tsr *Number[T]) IsString() bool { } func (tsr *Number[T]) AddScalar(i []int, val float64) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values[j] += T(val) return float64(tsr.Values[j]) } func (tsr *Number[T]) MulScalar(i []int, val float64) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values[j] *= T(val) return float64(tsr.Values[j]) } func (tsr *Number[T]) SetString(i []int, val string) { if fv, err := strconv.ParseFloat(val, 64); err == nil { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values[j] = T(fv) } } @@ -114,7 +114,7 @@ func (tsr Number[T]) SetString1D(off int, val string) { } func (tsr *Number[T]) SetStringRowCell(row, cell int, val string) { if fv, err := strconv.ParseFloat(val, 64); err == nil { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = T(fv) } } @@ -130,9 +130,9 @@ func (tsr *Number[T]) String() string { b.WriteString(str) b.WriteString("\n") oddRow := true - rows, cols, _, _ := Projection2DShape(&tsr.Shp, oddRow) + rows, cols, _, _ := Projection2DShape(&tsr.shape, oddRow) for r := 0; r < rows; r++ { - rc, _ := Projection2DCoords(&tsr.Shp, oddRow, r, 0) + rc, _ := Projection2DCoords(&tsr.shape, oddRow, r, 0) b.WriteString(fmt.Sprintf("%v: ", rc)) for c := 0; c < cols; c++ { vl := Projection2DValue(tsr, oddRow, r, c) @@ -144,12 +144,12 @@ func (tsr *Number[T]) String() string { } func (tsr *Number[T]) Float(i []int) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) return float64(tsr.Values[j]) } func (tsr *Number[T]) SetFloat(i []int, val float64) { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values[j] = T(val) } @@ -162,13 +162,13 @@ func (tsr *Number[T]) SetFloat1D(i int, val float64) { } func (tsr *Number[T]) FloatRowCell(row, cell int) float64 { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() i := row*sz + cell return float64(tsr.Values[i]) } func (tsr *Number[T]) SetFloatRowCell(row, cell int, val float64) { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = T(val) } @@ -257,7 +257,7 @@ func (tsr *Number[T]) SetZeros() { // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). func (tsr *Number[T]) Clone() Tensor { - csr := NewNumberShape[T](&tsr.Shp) + csr := NewNumberShape[T](&tsr.shape) copy(csr.Values, tsr.Values) return csr } diff --git a/tensor/stats/clust/clust_test.go b/tensor/stats/clust/clust_test.go index 858dc0dd6b..139e603c7e 100644 --- a/tensor/stats/clust/clust_test.go +++ b/tensor/stats/clust/clust_test.go @@ -33,7 +33,7 @@ func TestClust(t *testing.T) { if err != nil { t.Error(err) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) smat := &simat.SimMat{} smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) diff --git a/tensor/stats/glm/README.md b/tensor/stats/glm/README.md index ffe5d5703d..1e4b5503c1 100644 --- a/tensor/stats/glm/README.md +++ b/tensor/stats/glm/README.md @@ -2,7 +2,7 @@ GLM contains results and parameters for running a [general linear model](https://en.wikipedia.org/wiki/General_linear_model), which is a general form of multivariate linear regression, supporting multiple independent and dependent variables. -Make a `NewGLM` and then do `Run()` on a tensor [IndexView](../table/IndexView) with the relevant data in columns of the table. +Make a `NewGLM` and then do `Run()` on a tensor [Indexed](../table/Indexed) with the relevant data in columns of the table. # Fitting Methods diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index cbac41a520..a625ce8599 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -18,7 +18,7 @@ import ( // linear model, which is a general form of multivariate linear // regression, supporting multiple independent and dependent // variables. Make a NewGLM and then do Run() on a tensor -// table.IndexView with the relevant data in columns of the table. +// table.Indexed with the relevant data in columns of the table. // Batch-mode gradient descent is used and the relevant parameters // can be altered from defaults before calling Run as needed. type GLM struct { @@ -85,7 +85,7 @@ type GLM struct { // Cached values from the table // Table of data - Table *table.IndexView + Table *table.Indexed // tensor columns from table with the respective variables IndepVars, DepVars, PredVars, ErrVars tensor.Tensor @@ -122,7 +122,7 @@ func (glm *GLM) init(nIv, nDv int) { // each of the Vars args specifies a column in the table, which can have either a // single scalar value for each row, or a tensor cell with multiple values. // predVars and errVars (predicted values and error values) are optional. -func (glm *GLM) SetTable(ix *table.IndexView, indepVars, depVars, predVars, errVars string) error { +func (glm *GLM) SetTable(ix *table.Indexed, indepVars, depVars, predVars, errVars string) error { dt := ix.Table iv, err := dt.ColumnByName(indepVars) if err != nil { diff --git a/tensor/stats/pca/covar.go b/tensor/stats/pca/covar.go index 5aa1e9f0d0..7afc2b09ec 100644 --- a/tensor/stats/pca/covar.go +++ b/tensor/stats/pca/covar.go @@ -13,7 +13,7 @@ import ( ) // CovarTableColumn generates a covariance matrix from given column name -// in given IndexView of an table.Table, and given metric function +// in given Indexed of an table.Table, and given metric function // (typically Covariance or Correlation -- use Covar if vars have similar // overall scaling, which is typical in neural network models, and use // Correl if they are on very different scales -- Correl effectively rescales). @@ -22,7 +22,7 @@ import ( // cell co-varies in its value with each other cell across the rows of the table. // This is the input to the PCA eigenvalue decomposition of the resulting // covariance matrix. -func CovarTableColumn(cmat tensor.Tensor, ix *table.IndexView, column string, mfun metric.Func64) error { +func CovarTableColumn(cmat tensor.Tensor, ix *table.Indexed, column string, mfun metric.Func64) error { col, err := ix.Table.ColumnByName(column) if err != nil { return err @@ -139,7 +139,7 @@ func CovarTensor(cmat tensor.Tensor, tsr tensor.Tensor, mfun metric.Func64) erro // TableColumnRowsVec extracts row-wise vector from given cell index into vec. // vec must be of size ix.Len() -- number of rows -func TableColumnRowsVec(vec []float64, ix *table.IndexView, col tensor.Tensor, cidx int) { +func TableColumnRowsVec(vec []float64, ix *table.Indexed, col tensor.Tensor, cidx int) { rows := ix.Len() ln := col.Len() sz := ln / col.DimSize(0) // size of cell @@ -162,7 +162,7 @@ func TensorRowsVec(vec []float64, tsr tensor.Tensor, cidx int) { } // CovarTableColumnStd generates a covariance matrix from given column name -// in given IndexView of an table.Table, and given metric function +// in given Indexed of an table.Table, and given metric function // (typically Covariance or Correlation -- use Covar if vars have similar // overall scaling, which is typical in neural network models, and use // Correl if they are on very different scales -- Correl effectively rescales). @@ -172,7 +172,7 @@ func TensorRowsVec(vec []float64, tsr tensor.Tensor, cidx int) { // This is the input to the PCA eigenvalue decomposition of the resulting // covariance matrix. // This Std version is usable e.g., in Python where the func cannot be passed. -func CovarTableColumnStd(cmat tensor.Tensor, ix *table.IndexView, column string, met metric.StdMetrics) error { +func CovarTableColumnStd(cmat tensor.Tensor, ix *table.Indexed, column string, met metric.StdMetrics) error { return CovarTableColumn(cmat, ix, column, metric.StdFunc64(met)) } diff --git a/tensor/stats/pca/pca.go b/tensor/stats/pca/pca.go index b69b57e1a3..47fcf44c87 100644 --- a/tensor/stats/pca/pca.go +++ b/tensor/stats/pca/pca.go @@ -48,7 +48,7 @@ func (pa *PCA) Init() { // This is the input to the PCA eigenvalue decomposition of the resulting // covariance matrix, which extracts the eigenvectors as directions with maximal // variance in this matrix. -func (pa *PCA) TableColumn(ix *table.IndexView, column string, mfun metric.Func64) error { +func (pa *PCA) TableColumn(ix *table.Indexed, column string, mfun metric.Func64) error { if pa.Covar == nil { pa.Init() } @@ -97,7 +97,7 @@ func (pa *PCA) Tensor(tsr tensor.Tensor, mfun metric.Func64) error { // covariance matrix, which extracts the eigenvectors as directions with maximal // variance in this matrix. // This Std version is usable e.g., in Python where the func cannot be passed. -func (pa *PCA) TableColumnStd(ix *table.IndexView, column string, met metric.StdMetrics) error { +func (pa *PCA) TableColumnStd(ix *table.Indexed, column string, met metric.StdMetrics) error { return pa.TableColumn(ix, column, metric.StdFunc64(met)) } @@ -145,10 +145,10 @@ func (pa *PCA) PCA() error { return nil } -// ProjectColumn projects values from the given column of given table (via IndexView) +// ProjectColumn projects values from the given column of given table (via Indexed) // onto the idx'th eigenvector (0 = largest eigenvalue, 1 = next, etc). // Must have already called PCA() method. -func (pa *PCA) ProjectColumn(vals *[]float64, ix *table.IndexView, column string, idx int) error { +func (pa *PCA) ProjectColumn(vals *[]float64, ix *table.Indexed, column string, idx int) error { col, err := ix.Table.ColumnByName(column) if err != nil { return err @@ -188,11 +188,11 @@ func (pa *PCA) ProjectColumn(vals *[]float64, ix *table.IndexView, column string return nil } -// ProjectColumnToTable projects values from the given column of given table (via IndexView) +// ProjectColumnToTable projects values from the given column of given table (via Indexed) // onto the given set of eigenvectors (idxs, 0 = largest eigenvalue, 1 = next, etc), // and stores results along with labels from column labNm into results table. // Must have already called PCA() method. -func (pa *PCA) ProjectColumnToTable(projections *table.Table, ix *table.IndexView, column, labNm string, idxs []int) error { +func (pa *PCA) ProjectColumnToTable(projections *table.Table, ix *table.Indexed, column, labNm string, idxs []int) error { _, err := ix.Table.ColumnByName(column) if err != nil { return err diff --git a/tensor/stats/pca/pca_test.go b/tensor/stats/pca/pca_test.go index fb62388c8b..136bd95273 100644 --- a/tensor/stats/pca/pca_test.go +++ b/tensor/stats/pca/pca_test.go @@ -24,7 +24,7 @@ func TestPCAIris(t *testing.T) { if err != nil { t.Error(err) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) pc := &PCA{} // pc.TableColumn(ix, "data", metric.Covariance64) // fmt.Printf("covar: %v\n", pc.Covar) diff --git a/tensor/stats/pca/svd.go b/tensor/stats/pca/svd.go index d94ab60041..33bdaa192a 100644 --- a/tensor/stats/pca/svd.go +++ b/tensor/stats/pca/svd.go @@ -58,7 +58,7 @@ func (svd *SVD) Init() { // This is the input to the SVD eigenvalue decomposition of the resulting // covariance matrix, which extracts the eigenvectors as directions with maximal // variance in this matrix. -func (svd *SVD) TableColumn(ix *table.IndexView, column string, mfun metric.Func64) error { +func (svd *SVD) TableColumn(ix *table.Indexed, column string, mfun metric.Func64) error { if svd.Covar == nil { svd.Init() } @@ -107,7 +107,7 @@ func (svd *SVD) Tensor(tsr tensor.Tensor, mfun metric.Func64) error { // covariance matrix, which extracts the eigenvectors as directions with maximal // variance in this matrix. // This Std version is usable e.g., in Python where the func cannot be passed. -func (svd *SVD) TableColumnStd(ix *table.IndexView, column string, met metric.StdMetrics) error { +func (svd *SVD) TableColumnStd(ix *table.Indexed, column string, met metric.StdMetrics) error { return svd.TableColumn(ix, column, metric.StdFunc64(met)) } @@ -158,10 +158,10 @@ func (svd *SVD) SVD() error { return nil } -// ProjectColumn projects values from the given column of given table (via IndexView) +// ProjectColumn projects values from the given column of given table (via Indexed) // onto the idx'th eigenvector (0 = largest eigenvalue, 1 = next, etc). // Must have already called SVD() method. -func (svd *SVD) ProjectColumn(vals *[]float64, ix *table.IndexView, column string, idx int) error { +func (svd *SVD) ProjectColumn(vals *[]float64, ix *table.Indexed, column string, idx int) error { col, err := ix.Table.ColumnByName(column) if err != nil { return err @@ -201,11 +201,11 @@ func (svd *SVD) ProjectColumn(vals *[]float64, ix *table.IndexView, column strin return nil } -// ProjectColumnToTable projects values from the given column of given table (via IndexView) +// ProjectColumnToTable projects values from the given column of given table (via Indexed) // onto the given set of eigenvectors (idxs, 0 = largest eigenvalue, 1 = next, etc), // and stores results along with labels from column labNm into results table. // Must have already called SVD() method. -func (svd *SVD) ProjectColumnToTable(projections *table.Table, ix *table.IndexView, column, labNm string, idxs []int) error { +func (svd *SVD) ProjectColumnToTable(projections *table.Table, ix *table.Indexed, column, labNm string, idxs []int) error { _, err := ix.Table.ColumnByName(column) if errors.Log(err) != nil { return err diff --git a/tensor/stats/pca/svd_test.go b/tensor/stats/pca/svd_test.go index f1caad8b7c..3c65a63dc7 100644 --- a/tensor/stats/pca/svd_test.go +++ b/tensor/stats/pca/svd_test.go @@ -25,7 +25,7 @@ func TestSVDIris(t *testing.T) { if err != nil { t.Error(err) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) pc := &SVD{} pc.Init() pc.Kind = mat.SVDFull diff --git a/tensor/stats/simat/simat.go b/tensor/stats/simat/simat.go index 49f4f06b0d..b668e74d85 100644 --- a/tensor/stats/simat/simat.go +++ b/tensor/stats/simat/simat.go @@ -40,19 +40,19 @@ func (smat *SimMat) Init() { } // TableColumnStd generates a similarity / distance matrix from given column name -// in given IndexView of an table.Table, and given standard metric function. +// in given Indexed of an table.Table, and given standard metric function. // if labNm is not empty, uses given column name for labels, which if blankRepeat // is true are filtered so that any sequentially repeated labels are blank. // This Std version is usable e.g., in Python where the func cannot be passed. -func (smat *SimMat) TableColumnStd(ix *table.IndexView, column, labNm string, blankRepeat bool, met metric.StdMetrics) error { +func (smat *SimMat) TableColumnStd(ix *table.Indexed, column, labNm string, blankRepeat bool, met metric.StdMetrics) error { return smat.TableColumn(ix, column, labNm, blankRepeat, metric.StdFunc64(met)) } // TableColumn generates a similarity / distance matrix from given column name -// in given IndexView of an table.Table, and given metric function. +// in given Indexed of an table.Table, and given metric function. // if labNm is not empty, uses given column name for labels, which if blankRepeat // is true are filtered so that any sequentially repeated labels are blank. -func (smat *SimMat) TableColumn(ix *table.IndexView, column, labNm string, blankRepeat bool, mfun metric.Func64) error { +func (smat *SimMat) TableColumn(ix *table.Indexed, column, labNm string, blankRepeat bool, mfun metric.Func64) error { col, err := ix.Table.ColumnByName(column) if err != nil { return err diff --git a/tensor/stats/simat/simat_test.go b/tensor/stats/simat/simat_test.go index 103f7ae9a6..5cc5e24696 100644 --- a/tensor/stats/simat/simat_test.go +++ b/tensor/stats/simat/simat_test.go @@ -34,7 +34,7 @@ func TestSimMat(t *testing.T) { if err != nil { t.Error(err) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) smat := &SimMat{} smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) diff --git a/tensor/stats/split/agg_test.go b/tensor/stats/split/agg_test.go index e9fae284c7..9cb9d34b5d 100644 --- a/tensor/stats/split/agg_test.go +++ b/tensor/stats/split/agg_test.go @@ -25,7 +25,7 @@ func TestAgg(t *testing.T) { dt.SetString("Group", i, gp) dt.SetFloat("Value", i, float64(i)) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) spl := GroupBy(ix, "Group") assert.Equal(t, 2, len(spl.Splits)) @@ -50,7 +50,7 @@ func TestAggEmpty(t *testing.T) { dt.SetString("Group", i, gp) dt.SetFloat("Value", i, float64(i)) } - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) ix.Filter(func(et *table.Table, row int) bool { return false // exclude all }) diff --git a/tensor/stats/split/group.go b/tensor/stats/split/group.go index 3a8259fdef..c807b473b3 100644 --- a/tensor/stats/split/group.go +++ b/tensor/stats/split/group.go @@ -16,7 +16,7 @@ import ( // All returns a single "split" with all of the rows in given view // useful for leveraging the aggregation management functions in splits -func All(ix *table.IndexView) *table.Splits { +func All(ix *table.Indexed) *table.Splits { spl := &table.Splits{} spl.Levels = []string{"All"} spl.New(ix.Table, []string{"All"}, ix.Indexes...) @@ -26,7 +26,7 @@ func All(ix *table.IndexView) *table.Splits { // GroupByIndex returns a new Splits set based on the groups of values // across the given set of column indexes. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupByIndex(ix *table.IndexView, colIndexes []int) *table.Splits { +func GroupByIndex(ix *table.Indexed, colIndexes []int) *table.Splits { nc := len(colIndexes) if nc == 0 || ix.Table == nil { return nil @@ -44,7 +44,7 @@ func GroupByIndex(ix *table.IndexView, colIndexes []int) *table.Splits { srt.SortStableColumns(colIndexes, true) // important for consistency lstValues := make([]string, nc) curValues := make([]string, nc) - var curIx *table.IndexView + var curIx *table.Indexed for _, rw := range srt.Indexes { diff := false for i, ci := range colIndexes { @@ -71,7 +71,7 @@ func GroupByIndex(ix *table.IndexView, colIndexes []int) *table.Splits { // GroupBy returns a new Splits set based on the groups of values // across the given set of column names. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupBy(ix *table.IndexView, columns ...string) *table.Splits { +func GroupBy(ix *table.Indexed, columns ...string) *table.Splits { return GroupByIndex(ix, errors.Log1(ix.Table.ColumnIndexesByNames(columns...))) } @@ -80,7 +80,7 @@ func GroupBy(ix *table.IndexView, columns ...string) *table.Splits { // The function should always return the same number of values -- if // it doesn't behavior is undefined. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupByFunc(ix *table.IndexView, fun func(row int) []string) *table.Splits { +func GroupByFunc(ix *table.Indexed, fun func(row int) []string) *table.Splits { if ix.Table == nil { return nil } @@ -113,7 +113,7 @@ func GroupByFunc(ix *table.IndexView, fun func(row int) []string) *table.Splits // now do our usual grouping operation spl := &table.Splits{} lstValues := make([]string, nv) - var curIx *table.IndexView + var curIx *table.Indexed for _, rw := range srt.Indexes { curValues := funvals[rw] diff := (curIx == nil) diff --git a/tensor/stats/split/random.go b/tensor/stats/split/random.go index 4099d22e18..047f49ef96 100644 --- a/tensor/stats/split/random.go +++ b/tensor/stats/split/random.go @@ -16,7 +16,7 @@ import ( // which will be normalized to sum to 1 (error returned if sum = 0) // names are optional names for each split (e.g., Train, Test) which will be // used to label the Values of the resulting Splits. -func Permuted(ix *table.IndexView, probs []float64, names []string) (*table.Splits, error) { +func Permuted(ix *table.Indexed, probs []float64, names []string) (*table.Splits, error) { if ix == nil || ix.Len() == 0 { return nil, fmt.Errorf("split.Random table is nil / empty") } diff --git a/tensor/stats/split/random_test.go b/tensor/stats/split/random_test.go index 507ebd77d5..9055caf8c9 100644 --- a/tensor/stats/split/random_test.go +++ b/tensor/stats/split/random_test.go @@ -17,7 +17,7 @@ func TestPermuted(t *testing.T) { dt.AddStringColumn("Name") dt.AddFloat32TensorColumn("Input", []int{5, 5}, "Y", "X") dt.AddFloat32TensorColumn("Output", []int{5, 5}, "Y", "X") - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) spl, err := Permuted(ix, []float64{.5, .5}, nil) if err != nil { t.Error(err) diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index 5225e42383..fa7d0e2ebf 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -6,7 +6,7 @@ The `stats` package provides standard statistic computations operating over floa * `tensor.Float32`, `tensor.Float64` using the underlying `Values` slice, and other generic `Tensor` using the `Floats` interface (less efficient). -* `table.IndexView` indexed views of `table.Table` data, with `*Column` functions (e.g., `MeanColumn`) using names to specify columns, and `*Index` versions operating on column indexes. Also available for this type are `CountIf*`, `PctIf*`, `PropIf*` functions that return count, percentage, or propoprtion of values according to given function. +* `table.Indexed` indexed views of `table.Table` data, with `*Column` functions (e.g., `MeanColumn`) using names to specify columns, and `*Index` versions operating on column indexes. Also available for this type are `CountIf*`, `PctIf*`, `PropIf*` functions that return count, percentage, or propoprtion of values according to given function. ## Stats @@ -29,8 +29,8 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `VarPop`: population variance (squared diffs from mean, divided by n) * `StdPop`: population standard deviation (sqrt of VarPop) * `SemPop`: population standard error of the mean (StdPop divided by sqrt(n)) -* `Median`: middle value in sorted ordering (only for IndexView) -* `Q1`: Q1 first quartile = 25%ile value = .25 quantile value (only for IndexView) -* `Q3`: Q3 third quartile = 75%ile value = .75 quantile value (only for IndexView) +* `Median`: middle value in sorted ordering (only for Indexed) +* `Q1`: Q1 first quartile = 25%ile value = .25 quantile value (only for Indexed) +* `Q3`: Q3 third quartile = 75%ile value = .75 quantile value (only for Indexed) diff --git a/tensor/stats/stats/desc.go b/tensor/stats/stats/desc.go index e6795cef8c..d26869af9a 100644 --- a/tensor/stats/stats/desc.go +++ b/tensor/stats/stats/desc.go @@ -17,7 +17,7 @@ var DescStatsND = []Stats{Count, Mean, Std, Sem, Min, Max} // DescAll returns a table of standard descriptive stats for // all numeric columns in given table, operating over all non-Null, non-NaN elements // in each column. -func DescAll(ix *table.IndexView) *table.Table { +func DescAll(ix *table.Indexed) *table.Table { st := ix.Table nAgg := len(DescStats) dt := table.NewTable().SetNumRows(nAgg) @@ -63,9 +63,9 @@ func DescAll(ix *table.IndexView) *table.Table { } // DescIndex returns a table of standard descriptive aggregates -// of non-Null, non-NaN elements in given IndexView indexed view of an +// of non-Null, non-NaN elements in given Indexed indexed view of an // table.Table, for given column index. -func DescIndex(ix *table.IndexView, colIndex int) *table.Table { +func DescIndex(ix *table.Indexed, colIndex int) *table.Table { st := ix.Table col := st.Columns[colIndex] stats := DescStats @@ -100,12 +100,12 @@ func DescIndex(ix *table.IndexView, colIndex int) *table.Table { } // DescColumn returns a table of standard descriptive stats -// of non-NaN elements in given IndexView indexed view of an +// of non-NaN elements in given Indexed indexed view of an // table.Table, for given column name. // If name not found, returns error message. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func DescColumn(ix *table.IndexView, column string) (*table.Table, error) { +func DescColumn(ix *table.Indexed, column string) (*table.Table, error) { colIndex, err := ix.Table.ColumnIndex(column) if err != nil { return nil, err diff --git a/tensor/stats/stats/doc.go b/tensor/stats/stats/doc.go index e65f1ce9da..ea364ea36c 100644 --- a/tensor/stats/stats/doc.go +++ b/tensor/stats/stats/doc.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -Package agg provides aggregation functions operating on IndexView indexed views +Package agg provides aggregation functions operating on Indexed indexed views of table.Table data, along with standard AggFunc functions that can be used at any level of aggregation from tensor on up. diff --git a/tensor/stats/stats/if.go b/tensor/stats/stats/if.go index fc29404cf8..1673784b16 100644 --- a/tensor/stats/stats/if.go +++ b/tensor/stats/stats/if.go @@ -16,11 +16,11 @@ type IfFunc func(idx int, val float64) bool // CountIf // CountIfIndex returns the count of true return values for given IfFunc on -// non-NaN elements in given IndexView indexed view of an +// non-NaN elements in given Indexed indexed view of an // table.Table, for given column index. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func CountIfIndex(ix *table.IndexView, colIndex int, iffun IfFunc) []float64 { +func CountIfIndex(ix *table.Indexed, colIndex int, iffun IfFunc) []float64 { return StatIndexFunc(ix, colIndex, 0, func(idx int, val float64, agg float64) float64 { if iffun(idx, val) { return agg + 1 @@ -30,12 +30,12 @@ func CountIfIndex(ix *table.IndexView, colIndex int, iffun IfFunc) []float64 { } // CountIfColumn returns the count of true return values for given IfFunc on -// non-NaN elements in given IndexView indexed view of an +// non-NaN elements in given Indexed indexed view of an // table.Table, for given column name. // If name not found, nil is returned. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func CountIfColumn(ix *table.IndexView, column string, iffun IfFunc) []float64 { +func CountIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -47,11 +47,11 @@ func CountIfColumn(ix *table.IndexView, column string, iffun IfFunc) []float64 { // PropIf // PropIfIndex returns the proportion (0-1) of true return values for given IfFunc on -// non-Null, non-NaN elements in given IndexView indexed view of an +// non-Null, non-NaN elements in given Indexed indexed view of an // table.Table, for given column index. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func PropIfIndex(ix *table.IndexView, colIndex int, iffun IfFunc) []float64 { +func PropIfIndex(ix *table.Indexed, colIndex int, iffun IfFunc) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -66,12 +66,12 @@ func PropIfIndex(ix *table.IndexView, colIndex int, iffun IfFunc) []float64 { } // PropIfColumn returns the proportion (0-1) of true return values for given IfFunc on -// non-NaN elements in given IndexView indexed view of an +// non-NaN elements in given Indexed indexed view of an // table.Table, for given column name. // If name not found, nil is returned -- use Try version for error message. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func PropIfColumn(ix *table.IndexView, column string, iffun IfFunc) []float64 { +func PropIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -83,11 +83,11 @@ func PropIfColumn(ix *table.IndexView, column string, iffun IfFunc) []float64 { // PctIf // PctIfIndex returns the percentage (0-100) of true return values for given IfFunc on -// non-Null, non-NaN elements in given IndexView indexed view of an +// non-Null, non-NaN elements in given Indexed indexed view of an // table.Table, for given column index. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func PctIfIndex(ix *table.IndexView, colIndex int, iffun IfFunc) []float64 { +func PctIfIndex(ix *table.Indexed, colIndex int, iffun IfFunc) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -102,12 +102,12 @@ func PctIfIndex(ix *table.IndexView, colIndex int, iffun IfFunc) []float64 { } // PctIfColumn returns the percentage (0-100) of true return values for given IfFunc on -// non-Null, non-NaN elements in given IndexView indexed view of an +// non-Null, non-NaN elements in given Indexed indexed view of an // table.Table, for given column name. // If name not found, nil is returned -- use Try version for error message. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func PctIfColumn(ix *table.IndexView, column string, iffun IfFunc) []float64 { +func PctIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil diff --git a/tensor/stats/stats/indexview.go b/tensor/stats/stats/indexview.go index e849f4cfba..18cd995f52 100644 --- a/tensor/stats/stats/indexview.go +++ b/tensor/stats/stats/indexview.go @@ -11,20 +11,20 @@ import ( "cogentcore.org/core/tensor/table" ) -// Every IndexView Stats method in this file follows one of these signatures: +// Every Indexed Stats method in this file follows one of these signatures: -// IndexViewFuncIndex is a stats function operating on IndexView, taking a column index arg -type IndexViewFuncIndex func(ix *table.IndexView, colIndex int) []float64 +// IndexedFuncIndex is a stats function operating on Indexed, taking a column index arg +type IndexedFuncIndex func(ix *table.Indexed, colIndex int) []float64 -// IndexViewFuncColumn is a stats function operating on IndexView, taking a column name arg -type IndexViewFuncColumn func(ix *table.IndexView, column string) []float64 +// IndexedFuncColumn is a stats function operating on Indexed, taking a column name arg +type IndexedFuncColumn func(ix *table.Indexed, column string) []float64 -// StatIndex returns IndexView statistic according to given Stats type applied -// to all non-NaN elements in given IndexView indexed view of +// StatIndex returns Indexed statistic according to given Stats type applied +// to all non-NaN elements in given Indexed indexed view of // an table.Table, for given column index. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func StatIndex(ix *table.IndexView, colIndex int, stat Stats) []float64 { +func StatIndex(ix *table.Indexed, colIndex int, stat Stats) []float64 { switch stat { case Count: return CountIndex(ix, colIndex) @@ -70,13 +70,13 @@ func StatIndex(ix *table.IndexView, colIndex int, stat Stats) []float64 { return nil } -// StatColumn returns IndexView statistic according to given Stats type applied -// to all non-NaN elements in given IndexView indexed view of +// StatColumn returns Indexed statistic according to given Stats type applied +// to all non-NaN elements in given Indexed indexed view of // an table.Table, for given column name. // If name not found, returns error message. // Return value(s) is size of column cell: 1 for scalar 1D columns // and N for higher-dimensional columns. -func StatColumn(ix *table.IndexView, column string, stat Stats) ([]float64, error) { +func StatColumn(ix *table.Indexed, column string, stat Stats) ([]float64, error) { colIndex, err := ix.Table.ColumnIndex(column) if err != nil { return nil, err @@ -89,7 +89,7 @@ func StatColumn(ix *table.IndexView, column string, stat Stats) ([]float64, erro // using float64 conversions of the values. ini is the initial value for the agg variable. // Operates independently over each cell on n-dimensional columns and returns the result as a slice // of values per cell. -func StatIndexFunc(ix *table.IndexView, colIndex int, ini float64, fun StatFunc) []float64 { +func StatIndexFunc(ix *table.Indexed, colIndex int, ini float64, fun StatFunc) []float64 { cl := ix.Table.Columns[colIndex] _, csz := cl.RowCellSize() @@ -122,19 +122,19 @@ func StatIndexFunc(ix *table.IndexView, colIndex int, ini float64, fun StatFunc) // Count // CountIndex returns the count of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func CountIndex(ix *table.IndexView, colIndex int) []float64 { +func CountIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, 0, CountFunc) } // CountColumn returns the count of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func CountColumn(ix *table.IndexView, column string) []float64 { +func CountColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -146,19 +146,19 @@ func CountColumn(ix *table.IndexView, column string) []float64 { // Sum // SumIndex returns the sum of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SumIndex(ix *table.IndexView, colIndex int) []float64 { +func SumIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, 0, SumFunc) } // SumColumn returns the sum of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SumColumn(ix *table.IndexView, column string) []float64 { +func SumColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -170,19 +170,19 @@ func SumColumn(ix *table.IndexView, column string) []float64 { // Prod // ProdIndex returns the product of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func ProdIndex(ix *table.IndexView, colIndex int) []float64 { +func ProdIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, 1, ProdFunc) } // ProdColumn returns the product of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func ProdColumn(ix *table.IndexView, column string) []float64 { +func ProdColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -194,19 +194,19 @@ func ProdColumn(ix *table.IndexView, column string) []float64 { // Min // MinIndex returns the minimum of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MinIndex(ix *table.IndexView, colIndex int) []float64 { +func MinIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, math.MaxFloat64, MinFunc) } // MinColumn returns the minimum of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MinColumn(ix *table.IndexView, column string) []float64 { +func MinColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -218,19 +218,19 @@ func MinColumn(ix *table.IndexView, column string) []float64 { // Max // MaxIndex returns the maximum of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MaxIndex(ix *table.IndexView, colIndex int) []float64 { +func MaxIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, -math.MaxFloat64, MaxFunc) } // MaxColumn returns the maximum of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MaxColumn(ix *table.IndexView, column string) []float64 { +func MaxColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -242,19 +242,19 @@ func MaxColumn(ix *table.IndexView, column string) []float64 { // MinAbs // MinAbsIndex returns the minimum of abs of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MinAbsIndex(ix *table.IndexView, colIndex int) []float64 { +func MinAbsIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, math.MaxFloat64, MinAbsFunc) } // MinAbsColumn returns the minimum of abs of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MinAbsColumn(ix *table.IndexView, column string) []float64 { +func MinAbsColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -266,19 +266,19 @@ func MinAbsColumn(ix *table.IndexView, column string) []float64 { // MaxAbs // MaxAbsIndex returns the maximum of abs of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MaxAbsIndex(ix *table.IndexView, colIndex int) []float64 { +func MaxAbsIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, -math.MaxFloat64, MaxAbsFunc) } // MaxAbsColumn returns the maximum of abs of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MaxAbsColumn(ix *table.IndexView, column string) []float64 { +func MaxAbsColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -290,10 +290,10 @@ func MaxAbsColumn(ix *table.IndexView, column string) []float64 { // Mean // MeanIndex returns the mean of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MeanIndex(ix *table.IndexView, colIndex int) []float64 { +func MeanIndex(ix *table.Indexed, colIndex int) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -308,11 +308,11 @@ func MeanIndex(ix *table.IndexView, colIndex int) []float64 { } // MeanColumn returns the mean of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MeanColumn(ix *table.IndexView, column string) []float64 { +func MeanColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -324,11 +324,11 @@ func MeanColumn(ix *table.IndexView, column string) []float64 { // Var // VarIndex returns the sample variance of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Sample variance is normalized by 1/(n-1) -- see VarPop version for 1/n normalization. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func VarIndex(ix *table.IndexView, colIndex int) []float64 { +func VarIndex(ix *table.Indexed, colIndex int) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -355,12 +355,12 @@ func VarIndex(ix *table.IndexView, colIndex int) []float64 { } // VarColumn returns the sample variance of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // Sample variance is normalized by 1/(n-1) -- see VarPop version for 1/n normalization. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func VarColumn(ix *table.IndexView, column string) []float64 { +func VarColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -372,11 +372,11 @@ func VarColumn(ix *table.IndexView, column string) []float64 { // Std // StdIndex returns the sample std deviation of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Sample std deviation is normalized by 1/(n-1) -- see StdPop version for 1/n normalization. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func StdIndex(ix *table.IndexView, colIndex int) []float64 { +func StdIndex(ix *table.Indexed, colIndex int) []float64 { std := VarIndex(ix, colIndex) for i := range std { std[i] = math.Sqrt(std[i]) @@ -385,12 +385,12 @@ func StdIndex(ix *table.IndexView, colIndex int) []float64 { } // StdColumn returns the sample std deviation of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // Sample std deviation is normalized by 1/(n-1) -- see StdPop version for 1/n normalization. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func StdColumn(ix *table.IndexView, column string) []float64 { +func StdColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -402,11 +402,11 @@ func StdColumn(ix *table.IndexView, column string) []float64 { // Sem // SemIndex returns the sample standard error of the mean of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Sample sem is normalized by 1/(n-1) -- see SemPop version for 1/n normalization. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SemIndex(ix *table.IndexView, colIndex int) []float64 { +func SemIndex(ix *table.Indexed, colIndex int) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -421,12 +421,12 @@ func SemIndex(ix *table.IndexView, colIndex int) []float64 { } // SemColumn returns the sample standard error of the mean of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // Sample sem is normalized by 1/(n-1) -- see SemPop version for 1/n normalization. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SemColumn(ix *table.IndexView, column string) []float64 { +func SemColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -438,19 +438,19 @@ func SemColumn(ix *table.IndexView, column string) []float64 { // L1Norm // L1NormIndex returns the L1 norm (sum abs values) of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func L1NormIndex(ix *table.IndexView, colIndex int) []float64 { +func L1NormIndex(ix *table.Indexed, colIndex int) []float64 { return StatIndexFunc(ix, colIndex, 0, L1NormFunc) } // L1NormColumn returns the L1 norm (sum abs values) of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func L1NormColumn(ix *table.IndexView, column string) []float64 { +func L1NormColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -462,10 +462,10 @@ func L1NormColumn(ix *table.IndexView, column string) []float64 { // SumSq // SumSqIndex returns the sum-of-squares of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SumSqIndex(ix *table.IndexView, colIndex int) []float64 { +func SumSqIndex(ix *table.Indexed, colIndex int) []float64 { cl := ix.Table.Columns[colIndex] _, csz := cl.RowCellSize() @@ -534,11 +534,11 @@ func SumSqIndex(ix *table.IndexView, colIndex int) []float64 { } // SumSqColumn returns the sum-of-squares of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SumSqColumn(ix *table.IndexView, column string) []float64 { +func SumSqColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -550,10 +550,10 @@ func SumSqColumn(ix *table.IndexView, column string) []float64 { // L2Norm // L2NormIndex returns the L2 norm (square root of sum-of-squares) of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func L2NormIndex(ix *table.IndexView, colIndex int) []float64 { +func L2NormIndex(ix *table.Indexed, colIndex int) []float64 { ss := SumSqIndex(ix, colIndex) for i := range ss { ss[i] = math.Sqrt(ss[i]) @@ -562,11 +562,11 @@ func L2NormIndex(ix *table.IndexView, colIndex int) []float64 { } // L2NormColumn returns the L2 norm (square root of sum-of-squares) of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func L2NormColumn(ix *table.IndexView, column string) []float64 { +func L2NormColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -578,11 +578,11 @@ func L2NormColumn(ix *table.IndexView, column string) []float64 { // VarPop // VarPopIndex returns the population variance of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // population variance is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func VarPopIndex(ix *table.IndexView, colIndex int) []float64 { +func VarPopIndex(ix *table.Indexed, colIndex int) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -609,12 +609,12 @@ func VarPopIndex(ix *table.IndexView, colIndex int) []float64 { } // VarPopColumn returns the population variance of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // population variance is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func VarPopColumn(ix *table.IndexView, column string) []float64 { +func VarPopColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -626,11 +626,11 @@ func VarPopColumn(ix *table.IndexView, column string) []float64 { // StdPop // StdPopIndex returns the population std deviation of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // population std dev is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func StdPopIndex(ix *table.IndexView, colIndex int) []float64 { +func StdPopIndex(ix *table.Indexed, colIndex int) []float64 { std := VarPopIndex(ix, colIndex) for i := range std { std[i] = math.Sqrt(std[i]) @@ -639,12 +639,12 @@ func StdPopIndex(ix *table.IndexView, colIndex int) []float64 { } // StdPopColumn returns the population std deviation of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // population std dev is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func StdPopColumn(ix *table.IndexView, column string) []float64 { +func StdPopColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -656,11 +656,11 @@ func StdPopColumn(ix *table.IndexView, column string) []float64 { // SemPop // SemPopIndex returns the population standard error of the mean of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // population sem is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SemPopIndex(ix *table.IndexView, colIndex int) []float64 { +func SemPopIndex(ix *table.Indexed, colIndex int) []float64 { cnt := CountIndex(ix, colIndex) if cnt == nil { return nil @@ -675,12 +675,12 @@ func SemPopIndex(ix *table.IndexView, colIndex int) []float64 { } // SemPopColumn returns the standard error of the mean of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // population sem is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func SemPopColumn(ix *table.IndexView, column string) []float64 { +func SemPopColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -692,19 +692,19 @@ func SemPopColumn(ix *table.IndexView, column string) []float64 { // Median // MedianIndex returns the median of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MedianIndex(ix *table.IndexView, colIndex int) []float64 { +func MedianIndex(ix *table.Indexed, colIndex int) []float64 { return QuantilesIndex(ix, colIndex, []float64{.5}) } // MedianColumn returns the median of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func MedianColumn(ix *table.IndexView, column string) []float64 { +func MedianColumn(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -716,19 +716,19 @@ func MedianColumn(ix *table.IndexView, column string) []float64 { // Q1 // Q1Index returns the first quartile of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func Q1Index(ix *table.IndexView, colIndex int) []float64 { +func Q1Index(ix *table.Indexed, colIndex int) []float64 { return QuantilesIndex(ix, colIndex, []float64{.25}) } // Q1Column returns the first quartile of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func Q1Column(ix *table.IndexView, column string) []float64 { +func Q1Column(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil @@ -740,19 +740,19 @@ func Q1Column(ix *table.IndexView, column string) []float64 { // Q3 // Q3Index returns the third quartile of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func Q3Index(ix *table.IndexView, colIndex int) []float64 { +func Q3Index(ix *table.Indexed, colIndex int) []float64 { return QuantilesIndex(ix, colIndex, []float64{.75}) } // Q3Column returns the third quartile of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned. // Return value is size of each column cell -- 1 for scalar 1D columns // and N for higher-dimensional columns. -func Q3Column(ix *table.IndexView, column string) []float64 { +func Q3Column(ix *table.Indexed, column string) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil diff --git a/tensor/stats/stats/indexview_test.go b/tensor/stats/stats/indexview_test.go index 31c541bdd5..66c89f4afc 100644 --- a/tensor/stats/stats/indexview_test.go +++ b/tensor/stats/stats/indexview_test.go @@ -15,7 +15,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestIndexView(t *testing.T) { +func TestIndexed(t *testing.T) { dt := table.NewTable().SetNumRows(5) dt.AddFloat64Column("data") dt.SetFloat("data", 0, 1) @@ -24,7 +24,7 @@ func TestIndexView(t *testing.T) { dt.SetFloat("data", 3, 4) dt.SetFloat("data", 4, 5) - ix := table.NewIndexView(dt) + ix := table.NewIndexed(dt) results := []float64{5, 15, 120, 1, 5, 1, 5, 3, 2.5, math.Sqrt(2.5), math.Sqrt(2.5) / math.Sqrt(5), 15, 55, math.Sqrt(55), 2, math.Sqrt(2), math.Sqrt(2) / math.Sqrt(5), 3, 2, 4} diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index eb191691a4..dc02c330a9 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -12,12 +12,12 @@ import ( ) // QuantilesIndex returns the given quantile(s) of non-NaN elements in given -// IndexView indexed view of an table.Table, for given column index. +// Indexed indexed view of an table.Table, for given column index. // Column must be a 1d Column -- returns nil for n-dimensional columns. // qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. Uses linear interpolation. // Because this requires a sort, it is more efficient to get as many quantiles // as needed in one pass. -func QuantilesIndex(ix *table.IndexView, colIndex int, qs []float64) []float64 { +func QuantilesIndex(ix *table.Indexed, colIndex int, qs []float64) []float64 { nq := len(qs) if nq == 0 { return nil @@ -58,13 +58,13 @@ func QuantilesIndex(ix *table.IndexView, colIndex int, qs []float64) []float64 { } // Quantiles returns the given quantile(s) of non-Null, non-NaN elements in given -// IndexView indexed view of an table.Table, for given column name. +// Indexed indexed view of an table.Table, for given column name. // If name not found, nil is returned -- use Try version for error message. // Column must be a 1d Column -- returns nil for n-dimensional columns. // qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. Uses linear interpolation. // Because this requires a sort, it is more efficient to get as many quantiles // as needed in one pass. -func Quantiles(ix *table.IndexView, column string, qs []float64) []float64 { +func Quantiles(ix *table.Indexed, column string, qs []float64) []float64 { colIndex := errors.Log1(ix.Table.ColumnIndex(column)) if colIndex == -1 { return nil diff --git a/tensor/string.go b/tensor/string.go index 4a04bfbe68..9a353ccd2b 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -33,7 +33,7 @@ func NewString(sizes []int, names ...string) *String { // using given shape. func NewStringShape(shape *Shape) *String { tsr := &String{} - tsr.Shp.CopyShape(shape) + tsr.shape.CopyShape(shape) tsr.Values = make([]string, tsr.Len()) return tsr } @@ -57,21 +57,21 @@ func (tsr *String) IsString() bool { } func (tsr *String) AddScalar(i []int, val float64) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) fv := StringToFloat64(tsr.Values[j]) + val tsr.Values[j] = Float64ToString(fv) return fv } func (tsr *String) MulScalar(i []int, val float64) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) fv := StringToFloat64(tsr.Values[j]) * val tsr.Values[j] = Float64ToString(fv) return fv } func (tsr *String) SetString(i []int, val string) { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values[j] = val } @@ -80,7 +80,7 @@ func (tsr String) SetString1D(off int, val string) { } func (tsr *String) SetStringRowCell(row, cell int, val string) { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = val } @@ -95,9 +95,9 @@ func (tsr *String) String() string { b.WriteString(str) b.WriteString("\n") oddRow := true - rows, cols, _, _ := Projection2DShape(&tsr.Shp, oddRow) + rows, cols, _, _ := Projection2DShape(&tsr.shape, oddRow) for r := 0; r < rows; r++ { - rc, _ := Projection2DCoords(&tsr.Shp, oddRow, r, 0) + rc, _ := Projection2DCoords(&tsr.shape, oddRow, r, 0) b.WriteString(fmt.Sprintf("%v: ", rc)) for c := 0; c < cols; c++ { idx := Projection2DIndex(tsr.Shape(), oddRow, r, c) @@ -110,12 +110,12 @@ func (tsr *String) String() string { } func (tsr *String) Float(i []int) float64 { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) return StringToFloat64(tsr.Values[j]) } func (tsr *String) SetFloat(i []int, val float64) { - j := tsr.Shp.Offset(i) + j := tsr.shape.Offset(i) tsr.Values[j] = Float64ToString(val) } @@ -128,12 +128,12 @@ func (tsr *String) SetFloat1D(off int, val float64) { } func (tsr *String) FloatRowCell(row, cell int) float64 { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() return StringToFloat64(tsr.Values[row*sz+cell]) } func (tsr *String) SetFloatRowCell(row, cell int, val float64) { - _, sz := tsr.Shp.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = Float64ToString(val) } @@ -212,7 +212,7 @@ func (tsr *String) SetZeros() { // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). func (tsr *String) Clone() Tensor { - csr := NewStringShape(&tsr.Shp) + csr := NewStringShape(&tsr.shape) copy(csr.Values, tsr.Values) return csr } diff --git a/tensor/table/README.md b/tensor/table/README.md index e7ed404e77..17f5d78e02 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -14,13 +14,13 @@ The following packages are included: * [etensor](etensor) is a Tensor (n-dimensional array) object. `etensor.Tensor` is an interface that applies to many different type-specific instances, such as `etensor.Float32`. A tensor is just a `etensor.Shape` plus a slice holding the specific data type. Our tensor is based directly on the [Apache Arrow](https://github.com/apache/arrow/tree/master/go) project's tensor, and it fully interoperates with it. Arrow tensors are designed to be read-only, and we needed some extra support to make our `etable.Table` work well, so we had to roll our own. Our tensors also interoperate fully with Gonum's 2D-specific Matrix type for the 2D case. -* [etable](etable) has the `etable.Table` DataTable / DataFrame object, which is useful for many different data analysis and database functions, and also for holding patterns to present to a neural network, and logs of output from the models, etc. A `etable.Table` is just a slice of `etensor.Tensor` columns, that are all aligned along the outer-most *row* dimension. Index-based indirection, which is essential for efficient Sort, Filter etc, is provided by the `etable.IndexView` type, which is an indexed view into a Table. All data processing operations are defined on the IndexView. +* [etable](etable) has the `etable.Table` DataTable / DataFrame object, which is useful for many different data analysis and database functions, and also for holding patterns to present to a neural network, and logs of output from the models, etc. A `etable.Table` is just a slice of `etensor.Tensor` columns, that are all aligned along the outer-most *row* dimension. Index-based indirection, which is essential for efficient Sort, Filter etc, is provided by the `etable.Indexed` type, which is an indexed view into a Table. All data processing operations are defined on the Indexed. * [eplot](eplot) provides an interactive 2D plotting GUI in [GoGi](https://cogentcore.org/core/gi) for Table data, using the [gonum plot](https://github.com/gonum/plot) plotting package. You can select which columns to plot and specify various basic plot parameters. * [tensorcore](tensorcore) provides an interactive tabular, spreadsheet-style GUI using [GoGi](https://cogentcore.org/core/gi) for viewing and editing `etable.Table` and `etable.Tensor` objects. The `tensorcore.TensorGrid` also provides a colored grid display higher-dimensional tensor data. -* [agg](agg) provides standard aggregation functions (`Sum`, `Mean`, `Var`, `Std` etc) operating over `etable.IndexView` views of Table data. It also defines standard `AggFunc` functions such as `SumFunc` which can be used for `Agg` functions on either a Tensor or IndexView. +* [agg](agg) provides standard aggregation functions (`Sum`, `Mean`, `Var`, `Std` etc) operating over `etable.Indexed` views of Table data. It also defines standard `AggFunc` functions such as `SumFunc` which can be used for `Agg` functions on either a Tensor or Indexed. * [tsragg](tsragg) provides the same agg functions as in `agg`, but operating on all the values in a given `Tensor`. Because of the indexed, row-based nature of tensors in a Table, these are not the same as the `agg` functions. @@ -96,17 +96,17 @@ Other options are `etable.Equals` instead of `Contains` to search for an exact f ## Index Views (Sort, Filter, etc) -The [IndexView](https://godoc.org/github.com/goki/etable/v2/etable#IndexView) provides a list of row-wise indexes into a table, and Sorting, Filtering and Splitting all operate on this index view without changing the underlying table data, for maximum efficiency and flexibility. +The [Indexed](https://godoc.org/github.com/goki/etable/v2/etable#Indexed) provides a list of row-wise indexes into a table, and Sorting, Filtering and Splitting all operate on this index view without changing the underlying table data, for maximum efficiency and flexibility. ```Go -ix := etable.NewIndexView(et) // new view with all rows +ix := etable.NewIndexed(et) // new view with all rows ``` ### Sort ```Go ix.SortColumnName("Name", etable.Ascending) // etable.Ascending or etable.Descending -SortedTable := ix.NewTable() // turn an IndexView back into a new Table organized in order of indexes +SortedTable := ix.NewTable() // turn an Indexed back into a new Table organized in order of indexes ``` or: @@ -142,7 +142,7 @@ gps := byNm.AggsToTable(etable.AddAggName) // etable.AddAggName or etable.ColNam Describe (basic stats) all columns in a table: ```Go -ix := etable.NewIndexView(et) // new view with all rows +ix := etable.NewIndexed(et) // new view with all rows desc := agg.DescAll(ix) // summary stats of all columns // get value at given column name (from original table), row "Mean" mean := desc.Float("ColNm", desc.RowsByString("Agg", "Mean", etable.Equals, etable.UseCase)[0]) diff --git a/tensor/table/indexview.go b/tensor/table/indexed.go similarity index 60% rename from tensor/table/indexview.go rename to tensor/table/indexed.go index 5a8da3e570..4690150d08 100644 --- a/tensor/table/indexview.go +++ b/tensor/table/indexed.go @@ -5,61 +5,50 @@ package table import ( + "errors" "fmt" "log" "math/rand" "slices" "sort" "strings" -) - -// LessFunc is a function used for sort comparisons that returns -// true if Table row i is less than Table row j -- these are the -// raw row numbers, which have already been projected through -// indexes when used for sorting via Indexes. -type LessFunc func(et *Table, i, j int) bool -// Filterer is a function used for filtering that returns -// true if Table row should be included in the current filtered -// view of the table, and false if it should be removed. -type Filterer func(et *Table, row int) bool + "cogentcore.org/core/tensor" +) -// IndexView is an indexed wrapper around an table.Table that provides a +// Indexed is an indexed wrapper around a table.Table that provides a // specific view onto the Table defined by the set of indexes. // This provides an efficient way of sorting and filtering a table by only // updating the indexes while doing nothing to the Table itself. // To produce a table that has data actually organized according to the // indexed order, call the NewTable method. -// IndexView views on a table can also be organized together as Splits +// Indexed views on a table can also be organized together as Splits // of the table rows, e.g., by grouping values along a given column. -type IndexView struct { //types:add +type Indexed struct { //types:add // Table that we are an indexed view onto Table *Table // current indexes into Table Indexes []int - - // current Less function used in sorting - lessFunc LessFunc `copier:"-" display:"-" xml:"-" json:"-"` } -// NewIndexView returns a new IndexView based on given table, initialized with sequential idxes -func NewIndexView(et *Table) *IndexView { - ix := &IndexView{} - ix.SetTable(et) +// NewIndexed returns a new Indexed based on given table, initialized with sequential idxes +func NewIndexed(dt *Table) *Indexed { + ix := &Indexed{} + ix.SetTable(dt) return ix } // SetTable sets as indexes into given table with sequential initial indexes -func (ix *IndexView) SetTable(et *Table) { - ix.Table = et +func (ix *Indexed) SetTable(dt *Table) { + ix.Table = dt ix.Sequential() } // DeleteInvalid deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from table. -func (ix *IndexView) DeleteInvalid() { +func (ix *Indexed) DeleteInvalid() { if ix.Table == nil || ix.Table.Rows <= 0 { ix.Indexes = nil return @@ -73,7 +62,7 @@ func (ix *IndexView) DeleteInvalid() { } // Sequential sets indexes to sequential row-wise indexes into table -func (ix *IndexView) Sequential() { //types:add +func (ix *Indexed) Sequential() { //types:add if ix.Table == nil || ix.Table.Rows <= 0 { ix.Indexes = nil return @@ -87,7 +76,7 @@ func (ix *IndexView) Sequential() { //types:add // Permuted sets indexes to a permuted order -- if indexes already exist // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated -func (ix *IndexView) Permuted() { +func (ix *Indexed) Permuted() { if ix.Table == nil || ix.Table.Rows <= 0 { ix.Indexes = nil return @@ -102,38 +91,33 @@ func (ix *IndexView) Permuted() { } // AddIndex adds a new index to the list -func (ix *IndexView) AddIndex(idx int) { +func (ix *Indexed) AddIndex(idx int) { ix.Indexes = append(ix.Indexes, idx) } -// Sort sorts the indexes into our Table using given Less function. -// The Less function operates directly on row numbers into the Table +// SortFunc sorts the indexes into our Table using given compare function. +// The compare function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. -func (ix *IndexView) Sort(lessFunc func(et *Table, i, j int) bool) { - ix.lessFunc = lessFunc - sort.Sort(ix) +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +func (ix *Indexed) SortFunc(cmp func(dt *Table, i, j int) int) { + slices.SortFunc(ix.Indexes, func(a, b int) int { + return cmp(ix.Table, ix.Indexes[a], ix.Indexes[b]) + }) } // SortIndexes sorts the indexes into our Table directly in // numerical order, producing the native ordering, while preserving // any filtering that might have occurred. -func (ix *IndexView) SortIndexes() { +func (ix *Indexed) SortIndexes() { sort.Ints(ix.Indexes) } -const ( - // Ascending specifies an ascending sort direction for table Sort routines - Ascending = true - - // Descending specifies a descending sort direction for table Sort routines - Descending = false -) - // SortColumnName sorts the indexes into our Table according to values in // given column name, using either ascending or descending order. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *IndexView) SortColumnName(column string, ascending bool) error { //types:add +func (ix *Indexed) SortColumnName(column string, ascending bool) error { //types:add ci, err := ix.Table.ColumnIndex(column) if err != nil { log.Println(err) @@ -145,36 +129,33 @@ func (ix *IndexView) SortColumnName(column string, ascending bool) error { //typ // SortColumn sorts the indexes into our Table according to values in // given column index, using either ascending or descending order. -// Only valid for 1-dimensional columns. -func (ix *IndexView) SortColumn(colIndex int, ascending bool) { +// Only valid for 1-dimensional columns (returns error if not). +func (ix *Indexed) SortColumn(colIndex int, ascending bool) error { cl := ix.Table.Columns[colIndex] + if cl.NumDims() > 1 { + return errors.New("tensor Sorting is only for 1D tensors") + } if cl.IsString() { - ix.Sort(func(et *Table, i, j int) bool { - if ascending { - return cl.String1D(i) < cl.String1D(j) - } else { - return cl.String1D(i) > cl.String1D(j) - } + ix.SortFunc(func(dt *Table, i, j int) int { + return tensor.CompareStrings(cl.String1D(i), cl.String1D(j), ascending) }) } else { - ix.Sort(func(et *Table, i, j int) bool { - if ascending { - return cl.Float1D(i) < cl.Float1D(j) - } else { - return cl.Float1D(i) > cl.Float1D(j) - } + ix.SortFunc(func(dt *Table, i, j int) int { + return tensor.CompareNumbers(cl.Float1D(i), cl.Float1D(j), ascending) }) } + return nil } // SortColumnNames sorts the indexes into our Table according to values in -// given column names, using either ascending or descending order. +// given column names, using either ascending or descending order, +// and optionally using a stable sort. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *IndexView) SortColumnNames(columns []string, ascending bool) error { +func (ix *Indexed) SortColumnNames(columns []string, ascending, stable bool) error { nc := len(columns) if nc == 0 { - return fmt.Errorf("table.IndexView.SortColumnNames: no column names provided") + return fmt.Errorf("table.Indexed.SortColumnNames: no column names provided") } cis := make([]int, nc) for i, cn := range columns { @@ -185,69 +166,76 @@ func (ix *IndexView) SortColumnNames(columns []string, ascending bool) error { } cis[i] = ci } - ix.SortColumns(cis, ascending) + ix.SortColumns(cis, ascending, stable) return nil } // SortColumns sorts the indexes into our Table according to values in // given list of column indexes, using either ascending or descending order for // all of the columns. Only valid for 1-dimensional columns. -func (ix *IndexView) SortColumns(colIndexes []int, ascending bool) { - ix.Sort(func(et *Table, i, j int) bool { +func (ix *Indexed) SortColumns(colIndexes []int, ascending, stable bool) { + sf := ix.SortFunc + if stable { + sf = ix.SortStableFunc + } + sf(func(dt *Table, i, j int) int { for _, ci := range colIndexes { cl := ix.Table.Columns[ci] if cl.IsString() { if ascending { if cl.String1D(i) < cl.String1D(j) { - return true + return 1 } else if cl.String1D(i) > cl.String1D(j) { - return false + return -1 } // if equal, fallthrough to next col } else { if cl.String1D(i) > cl.String1D(j) { - return true + return 1 } else if cl.String1D(i) < cl.String1D(j) { - return false + return -1 } // if equal, fallthrough to next col } } else { if ascending { if cl.Float1D(i) < cl.Float1D(j) { - return true + return 1 } else if cl.Float1D(i) > cl.Float1D(j) { - return false + return -1 } // if equal, fallthrough to next col } else { if cl.Float1D(i) > cl.Float1D(j) { - return true + return 1 } else if cl.Float1D(i) < cl.Float1D(j) { - return false + return -1 } // if equal, fallthrough to next col } } } - return false + return 0 }) } ///////////////////////////////////////////////////////////////////////// // Stable sorts -- sometimes essential.. -// SortStable stably sorts the indexes into our Table using given Less function. -// The Less function operates directly on row numbers into the Table +// SortStableFunc stably sorts the indexes into our Table using given compare function. +// The compare function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. -// It is *essential* that it always returns false when the two are equal +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +// It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (ix *IndexView) SortStable(lessFunc func(et *Table, i, j int) bool) { - ix.lessFunc = lessFunc - sort.Stable(ix) +func (ix *Indexed) SortStableFunc(cmp func(dt *Table, i, j int) int) { + slices.SortStableFunc(ix.Indexes, func(a, b int) int { + return cmp(ix.Table, ix.Indexes[a], ix.Indexes[b]) + }) } // SortStableColumnName sorts the indexes into our Table according to values in // given column name, using either ascending or descending order. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *IndexView) SortStableColumnName(column string, ascending bool) error { +func (ix *Indexed) SortStableColumnName(column string, ascending bool) error { ci, err := ix.Table.ColumnIndex(column) if err != nil { log.Println(err) @@ -260,94 +248,28 @@ func (ix *IndexView) SortStableColumnName(column string, ascending bool) error { // SortStableColumn sorts the indexes into our Table according to values in // given column index, using either ascending or descending order. // Only valid for 1-dimensional columns. -func (ix *IndexView) SortStableColumn(colIndex int, ascending bool) { +func (ix *Indexed) SortStableColumn(colIndex int, ascending bool) { cl := ix.Table.Columns[colIndex] if cl.IsString() { - ix.SortStable(func(et *Table, i, j int) bool { - if ascending { - return cl.String1D(i) < cl.String1D(j) - } else { - return cl.String1D(i) > cl.String1D(j) - } + ix.SortStableFunc(func(dt *Table, i, j int) int { + return tensor.CompareStrings(cl.String1D(i), cl.String1D(j), ascending) }) } else { - ix.SortStable(func(et *Table, i, j int) bool { - if ascending { - return cl.Float1D(i) < cl.Float1D(j) - } else { - return cl.Float1D(i) > cl.Float1D(j) - } + ix.SortStableFunc(func(dt *Table, i, j int) int { + return tensor.CompareNumbers(cl.Float1D(i), cl.Float1D(j), ascending) }) } } -// SortStableColumnNames sorts the indexes into our Table according to values in -// given column names, using either ascending or descending order. -// Only valid for 1-dimensional columns. -// Returns error if column name not found. -func (ix *IndexView) SortStableColumnNames(columns []string, ascending bool) error { - nc := len(columns) - if nc == 0 { - return fmt.Errorf("table.IndexView.SortStableColumnNames: no column names provided") - } - cis := make([]int, nc) - for i, cn := range columns { - ci, err := ix.Table.ColumnIndex(cn) - if err != nil { - log.Println(err) - return err - } - cis[i] = ci - } - ix.SortStableColumns(cis, ascending) - return nil -} - -// SortStableColumns sorts the indexes into our Table according to values in -// given list of column indexes, using either ascending or descending order for -// all of the columns. Only valid for 1-dimensional columns. -func (ix *IndexView) SortStableColumns(colIndexes []int, ascending bool) { - ix.SortStable(func(et *Table, i, j int) bool { - for _, ci := range colIndexes { - cl := ix.Table.Columns[ci] - if cl.IsString() { - if ascending { - if cl.String1D(i) < cl.String1D(j) { - return true - } else if cl.String1D(i) > cl.String1D(j) { - return false - } // if equal, fallthrough to next col - } else { - if cl.String1D(i) > cl.String1D(j) { - return true - } else if cl.String1D(i) < cl.String1D(j) { - return false - } // if equal, fallthrough to next col - } - } else { - if ascending { - if cl.Float1D(i) < cl.Float1D(j) { - return true - } else if cl.Float1D(i) > cl.Float1D(j) { - return false - } // if equal, fallthrough to next col - } else { - if cl.Float1D(i) > cl.Float1D(j) { - return true - } else if cl.Float1D(i) < cl.Float1D(j) { - return false - } // if equal, fallthrough to next col - } - } - } - return false - }) -} +// FilterFunc is a function used for filtering that returns +// true if Table row should be included in the current filtered +// view of the table, and false if it should be removed. +type FilterFunc func(dt *Table, row int) bool // Filter filters the indexes into our Table using given Filter function. // The Filter function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. -func (ix *IndexView) Filter(filterer func(et *Table, row int) bool) { +func (ix *Indexed) Filter(filterer func(dt *Table, row int) bool) { sz := len(ix.Indexes) for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering if !filterer(ix.Table, ix.Indexes[i]) { // delete @@ -363,7 +285,7 @@ func (ix *IndexView) Filter(filterer func(et *Table, row int) bool) { // Use named args for greater clarity. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *IndexView) FilterColumnName(column string, str string, exclude, contains, ignoreCase bool) error { //types:add +func (ix *Indexed) FilterColumnName(column string, str string, exclude, contains, ignoreCase bool) error { //types:add ci, err := ix.Table.ColumnIndex(column) if err != nil { log.Println(err) @@ -379,10 +301,10 @@ func (ix *IndexView) FilterColumnName(column string, str string, exclude, contai // If contains, only checks if row contains string; if ignoreCase, ignores case. // Use named args for greater clarity. // Only valid for 1-dimensional columns. -func (ix *IndexView) FilterColumn(colIndex int, str string, exclude, contains, ignoreCase bool) { +func (ix *Indexed) FilterColumn(colIndex int, str string, exclude, contains, ignoreCase bool) { col := ix.Table.Columns[colIndex] lowstr := strings.ToLower(str) - ix.Filter(func(et *Table, row int) bool { + ix.Filter(func(dt *Table, row int) bool { val := col.String1D(row) has := false switch { @@ -404,7 +326,7 @@ func (ix *IndexView) FilterColumn(colIndex int, str string, exclude, contains, i // NewTable returns a new table with column data organized according to // the indexes -func (ix *IndexView) NewTable() *Table { +func (ix *Indexed) NewTable() *Table { rows := len(ix.Indexes) nt := ix.Table.Clone() nt.SetNumRows(rows) @@ -423,20 +345,20 @@ func (ix *IndexView) NewTable() *Table { } // Clone returns a copy of the current index view with its own index memory -func (ix *IndexView) Clone() *IndexView { - nix := &IndexView{} +func (ix *Indexed) Clone() *Indexed { + nix := &Indexed{} nix.CopyFrom(ix) return nix } -// CopyFrom copies from given other IndexView (we have our own unique copy of indexes) -func (ix *IndexView) CopyFrom(oix *IndexView) { +// CopyFrom copies from given other Indexed (we have our own unique copy of indexes) +func (ix *Indexed) CopyFrom(oix *Indexed) { ix.Table = oix.Table ix.Indexes = slices.Clone(oix.Indexes) } // AddRows adds n rows to end of underlying Table, and to the indexes in this view -func (ix *IndexView) AddRows(n int) { //types:add +func (ix *Indexed) AddRows(n int) { //types:add stidx := ix.Table.Rows ix.Table.SetNumRows(stidx + n) for i := stidx; i < stidx+n; i++ { @@ -446,7 +368,7 @@ func (ix *IndexView) AddRows(n int) { //types:add // InsertRows adds n rows to end of underlying Table, and to the indexes starting at // given index in this view -func (ix *IndexView) InsertRows(at, n int) { +func (ix *Indexed) InsertRows(at, n int) { stidx := ix.Table.Rows ix.Table.SetNumRows(stidx + n) nw := make([]int, n, n+len(ix.Indexes)-at) @@ -457,7 +379,7 @@ func (ix *IndexView) InsertRows(at, n int) { } // DeleteRows deletes n rows of indexes starting at given index in the list of indexes -func (ix *IndexView) DeleteRows(at, n int) { +func (ix *Indexed) DeleteRows(at, n int) { ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) } @@ -465,11 +387,11 @@ func (ix *IndexView) DeleteRows(at, n int) { // given string value in given column index (de-reference our indexes to get actual row). // if contains, only checks if row contains string; if ignoreCase, ignores case. // Use named args for greater clarity. -func (ix *IndexView) RowsByStringIndex(colIndex int, str string, contains, ignoreCase bool) []int { +func (ix *Indexed) RowsByStringIndex(colIndex int, str string, contains, ignoreCase bool) []int { dt := ix.Table col := dt.Columns[colIndex] lowstr := strings.ToLower(str) - var idxs []int + var indexes []int for idx, srw := range ix.Indexes { val := col.String1D(srw) has := false @@ -484,10 +406,10 @@ func (ix *IndexView) RowsByStringIndex(colIndex int, str string, contains, ignor has = (val == str) } if has { - idxs = append(idxs, idx) + indexes = append(indexes, idx) } } - return idxs + return indexes } // RowsByString returns the list of *our indexes* whose row in the table has @@ -495,7 +417,7 @@ func (ix *IndexView) RowsByStringIndex(colIndex int, str string, contains, ignor // if contains, only checks if row contains string; if ignoreCase, ignores case. // returns error message for invalid column name. // Use named args for greater clarity. -func (ix *IndexView) RowsByString(column string, str string, contains, ignoreCase bool) ([]int, error) { +func (ix *Indexed) RowsByString(column string, str string, contains, ignoreCase bool) ([]int, error) { dt := ix.Table ci, err := dt.ColumnIndex(column) if err != nil { @@ -505,16 +427,11 @@ func (ix *IndexView) RowsByString(column string, str string, contains, ignoreCas } // Len returns the length of the index list -func (ix *IndexView) Len() int { +func (ix *Indexed) Len() int { return len(ix.Indexes) } -// Less calls the LessFunc for sorting -func (ix *IndexView) Less(i, j int) bool { - return ix.lessFunc(ix.Table, ix.Indexes[i], ix.Indexes[j]) -} - // Swap switches the indexes for i and j -func (ix *IndexView) Swap(i, j int) { +func (ix *Indexed) Swap(i, j int) { ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] } diff --git a/tensor/table/io.go b/tensor/table/io.go index c5e0395cfd..d8470882f4 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -85,7 +85,7 @@ func (dt *Table) SaveCSV(filename core.Filename, delim Delims, headers bool) err // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (ix *IndexView) SaveCSV(filename core.Filename, delim Delims, headers bool) error { //types:add +func (ix *Indexed) SaveCSV(filename core.Filename, delim Delims, headers bool) error { //types:add fp, err := os.Create(string(filename)) defer fp.Close() if err != nil { @@ -135,14 +135,14 @@ func (dt *Table) OpenFS(fsys fs.FS, filename string, delim Delims) error { // information for tensor type and dimensionality. // If the table DOES have existing columns, then those are used robustly // for whatever information fits from each row of the file. -func (ix *IndexView) OpenCSV(filename core.Filename, delim Delims) error { //types:add +func (ix *Indexed) OpenCSV(filename core.Filename, delim Delims) error { //types:add err := ix.Table.OpenCSV(filename, delim) ix.Sequential() return err } -// OpenFS is the version of [IndexView.OpenCSV] that uses an [fs.FS] filesystem. -func (ix *IndexView) OpenFS(fsys fs.FS, filename string, delim Delims) error { +// OpenFS is the version of [Indexed.OpenCSV] that uses an [fs.FS] filesystem. +func (ix *Indexed) OpenFS(fsys fs.FS, filename string, delim Delims) error { err := ix.Table.OpenFS(fsys, filename, delim) ix.Sequential() return err @@ -422,7 +422,7 @@ func (dt *Table) WriteCSV(w io.Writer, delim Delims, headers bool) error { // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (ix *IndexView) WriteCSV(w io.Writer, delim Delims, headers bool) error { +func (ix *Indexed) WriteCSV(w io.Writer, delim Delims, headers bool) error { ncol := 0 var err error if headers { diff --git a/tensor/table/splits.go b/tensor/table/splits.go index faa8ec71c3..9f81c65941 100644 --- a/tensor/table/splits.go +++ b/tensor/table/splits.go @@ -32,7 +32,7 @@ type SplitAgg struct { // It is functionally equivalent to the MultiIndex in python's pandas: it has multiple // levels of indexes as listed in the Levels field, which then have corresponding // Values for each split. These index levels can be re-ordered, and new Splits or -// IndexViews's can be created from subsets of the existing levels. The Values are +// Indexeds's can be created from subsets of the existing levels. The Values are // stored simply as string values, as this is the most general type and often // index values are labels etc. // @@ -58,7 +58,7 @@ type SplitAgg struct { type Splits struct { // the list of index views for each split - Splits []*IndexView + Splits []*Indexed // levels of indexes used to organize the splits -- each split contains the full outer product across these index levels. for example, if the split was generated by grouping over column values, then these are the column names in order of grouping. the splits are not automatically sorted hierarchically by these levels but e.g., the GroupBy method produces that result -- use the Sort methods to explicitly sort. Levels []string @@ -95,9 +95,9 @@ func (spl *Splits) Table() *Table { // values, which are copied before saving into Values list, and any number of rows // from the table associated with this split (also copied). // Any existing Aggs are deleted by this. -func (spl *Splits) New(dt *Table, values []string, rows ...int) *IndexView { +func (spl *Splits) New(dt *Table, values []string, rows ...int) *Indexed { spl.Aggs = nil - ix := &IndexView{Table: dt} + ix := &Indexed{Table: dt} spl.Splits = append(spl.Splits, ix) if len(rows) > 0 { ix.Indexes = append(ix.Indexes, slices.Clone(rows)...) @@ -239,7 +239,7 @@ func (spl *Splits) ExtractLevels(levels []int) (*Splits, error) { // now just do the grouping by levels values lstValues := make([]string, nlv) curValues := make([]string, nlv) - var curIx *IndexView + var curIx *Indexed nsp := len(ss.Splits) for si := nsp - 1; si >= 0; si-- { diff := false @@ -295,7 +295,7 @@ func (spl *Splits) Clone() *Splits { // CopyFrom copies from other Splits -- we get our own unique copy of everything func (spl *Splits) CopyFrom(osp *Splits) { - spl.Splits = make([]*IndexView, len(osp.Splits)) + spl.Splits = make([]*Indexed, len(osp.Splits)) spl.Values = make([][]string, len(osp.Values)) for si := range osp.Splits { spl.Splits[si] = osp.Splits[si].Clone() diff --git a/tensor/table/typegen.go b/tensor/table/typegen.go index 92eec5f9c5..64c43d7067 100644 --- a/tensor/table/typegen.go +++ b/tensor/table/typegen.go @@ -6,6 +6,6 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.IndexView", IDName: "index-view", Doc: "IndexView is an indexed wrapper around an table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexView views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}, {Name: "lessFunc", Doc: "current Less function used in sorting"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Indexed", IDName: "index-view", Doc: "Indexed is an indexed wrapper around an table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}, {Name: "lessFunc", Doc: "current Less function used in sorting"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of data, with columns of tensors,\neach with the same number of Rows (outer-most dimension).", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to each of the columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns\nif rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "columns of data, as tensor.Tensor tensors"}, {Name: "ColumnNames", Doc: "the names of the columns"}, {Name: "Rows", Doc: "number of rows, which is enforced to be the size of the outer-most dimension of the column tensors"}, {Name: "ColumnNameMap", Doc: "the map of column names to column numbers"}, {Name: "MetaData", Doc: "misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column"}}}) diff --git a/tensor/tensor.go b/tensor/tensor.go index d0953711a0..54f6150595 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -10,6 +10,7 @@ import ( "fmt" "reflect" + "cogentcore.org/core/base/metadata" "gonum.org/v1/gonum/mat" ) @@ -29,9 +30,15 @@ type Tensor interface { fmt.Stringer mat.Matrix - // Shape returns a pointer to the shape that fully parametrizes the tensor shape + // Shape returns a pointer to the Shape that fully parametrizes + // the tensor shape. Shape() *Shape + // SetShape sets the sizes parameters of the tensor, and resizes + // backing storage appropriately. + // Existing names will be preserved if not presented. + SetShape(sizes []int, names ...string) + // Len returns the number of elements in the tensor (product of shape dimensions). Len() int @@ -106,11 +113,11 @@ type Tensor interface { SetString1D(i int, val string) // StringRowCell returns the value at given row and cell, where row is outer-most dim, - // and cell is 1D index into remaining inner dims. For Table columns + // and cell is 1D index into remaining inner dims. For Table columns. StringRowCell(row, cell int) string // SetStringRowCell sets the value at given row and cell, where row is outer-most dim, - // and cell is 1D index into remaining inner dims. For Table columns + // and cell is 1D index into remaining inner dims. For Table columns. SetStringRowCell(row, cell int, val string) // SubSpace returns a new tensor with innermost subspace at given @@ -126,7 +133,7 @@ type Tensor interface { // Other math operations can be done using gonum/floats package. Range() (min, max float64, minIndex, maxIndex int) - // SetZeros is simple convenience function initialize all values to 0 + // SetZeros is simple convenience function initialize all values to 0. SetZeros() // Clone clones this tensor, creating a duplicate copy of itself with its @@ -150,26 +157,12 @@ type Tensor interface { // of the same type, and otherwise it goes through appropriate standard type. CopyCellsFrom(from Tensor, to, start, n int) - // SetShape sets the sizes parameters of the tensor, and resizes backing storage appropriately. - // existing names will be preserved if not presented. - SetShape(sizes []int, names ...string) - // SetNumRows sets the number of rows (outer-most dimension). SetNumRows(rows int) - // SetMetaData sets a key=value meta data (stored as a map[string]string). - // For TensorGrid display: top-zero=+/-, odd-row=+/-, image=+/-, - // min, max set fixed min / max values, background=color - SetMetaData(key, val string) - - // MetaData retrieves value of given key, bool = false if not set - MetaData(key string) (string, bool) - - // MetaDataMap returns the underlying map used for meta data - MetaDataMap() map[string]string - - // CopyMetaData copies meta data from given source tensor - CopyMetaData(from Tensor) + // Metadata returns the metadata for this tensor, which can be used + // to encode plotting options, etc. + Metadata() *metadata.Data } // New returns a new n-dimensional tensor of given value type diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 2efa6c1ba4..f2c7ca3bd2 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -8,6 +8,7 @@ import ( "reflect" "testing" + "cogentcore.org/core/base/metadata" "github.com/stretchr/testify/assert" ) @@ -55,12 +56,12 @@ func TestTensorString(t *testing.T) { tsr.SetNumRows(5) assert.Equal(t, 20, tsr.Len()) - tsr.SetMetaData("name", "test") - nm, has := tsr.MetaData("name") + tsr.Metadata().Set("name", "test") + nm, err := metadata.Get[string](*tsr.Metadata(), "name") assert.Equal(t, "test", nm) - assert.Equal(t, true, has) - _, has = tsr.MetaData("type") - assert.Equal(t, false, has) + assert.NoError(t, err) + _, err = metadata.Get[string](*tsr.Metadata(), "type") + assert.Error(t, err) var flt []float64 cln.SetString1D(0, "3.14") @@ -115,13 +116,6 @@ func TestTensorFloat64(t *testing.T) { tsr.SetNumRows(5) assert.Equal(t, 20, tsr.Len()) - tsr.SetMetaData("name", "test") - nm, has := tsr.MetaData("name") - assert.Equal(t, "test", nm) - assert.Equal(t, true, has) - _, has = tsr.MetaData("type") - assert.Equal(t, false, has) - var flt []float64 cln.SetString1D(0, "3.14") assert.Equal(t, 3.14, cln.Float1D(0)) diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 27ab0998fa..69dfbe7161 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -35,7 +35,7 @@ type Table struct { core.ListBase // the idx view of the table that we're a view of - Table *table.IndexView `set:"-"` + Table *table.Indexed `set:"-"` // overall display options for tensor display TensorDisplay TensorDisplay `set:"-"` @@ -131,7 +131,7 @@ func (tb *Table) StyleValue(w core.Widget, s *styles.Style, row, col int) { s.SetTextWrap(false) } -// SetTable sets the source table that we are viewing, using a sequential IndexView +// SetTable sets the source table that we are viewing, using a sequential Indexed // and then configures the display func (tb *Table) SetTable(et *table.Table) *Table { if et == nil { @@ -139,7 +139,7 @@ func (tb *Table) SetTable(et *table.Table) *Table { } tb.SetSliceBase() - tb.Table = table.NewIndexView(et) + tb.Table = table.NewIndexed(et) tb.This.(core.Lister).UpdateSliceSize() tb.Update() return tb @@ -161,9 +161,9 @@ func (tb *Table) AsyncUpdateTable() { tb.AsyncUnlock() } -// SetIndexView sets the source IndexView of a table (using a copy so original is not modified) +// SetIndexed sets the source Indexed of a table (using a copy so original is not modified) // and then configures the display -func (tb *Table) SetIndexView(ix *table.IndexView) *Table { +func (tb *Table) SetIndexed(ix *table.Indexed) *Table { if ix == nil { return tb } @@ -669,7 +669,7 @@ func (tb *Table) CopySelectToMime() mimedata.Mimes { if nitms == 0 { return nil } - ix := &table.IndexView{} + ix := &table.Indexed{} ix.Table = tb.Table.Table idx := tb.SelectedIndexesList(false) // ascending iidx := make([]int, len(idx)) diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-table.go b/yaegicore/symbols/cogentcore_org-core-tensor-table.go index 1fd38dbb65..751886cc5a 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-table.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-table.go @@ -3,8 +3,9 @@ package symbols import ( - "cogentcore.org/core/tensor/table" "reflect" + + "cogentcore.org/core/tensor/table" ) func init() { @@ -27,7 +28,7 @@ func init() { "Headers": reflect.ValueOf(table.Headers), "IgnoreCase": reflect.ValueOf(table.IgnoreCase), "InferDataType": reflect.ValueOf(table.InferDataType), - "NewIndexView": reflect.ValueOf(table.NewIndexView), + "NewIndexed": reflect.ValueOf(table.NewIndexed), "NewSliceTable": reflect.ValueOf(table.NewSliceTable), "NewTable": reflect.ValueOf(table.NewTable), "NoHeaders": reflect.ValueOf(table.NoHeaders), @@ -43,7 +44,7 @@ func init() { // type definitions "Delims": reflect.ValueOf((*table.Delims)(nil)), "Filterer": reflect.ValueOf((*table.Filterer)(nil)), - "IndexView": reflect.ValueOf((*table.IndexView)(nil)), + "Indexed": reflect.ValueOf((*table.Indexed)(nil)), "LessFunc": reflect.ValueOf((*table.LessFunc)(nil)), "SplitAgg": reflect.ValueOf((*table.SplitAgg)(nil)), "Splits": reflect.ValueOf((*table.Splits)(nil)), From 79210aa677acdf06c1fbcd3a528c44d91dcf815c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 8 Sep 2024 16:47:14 -0700 Subject: [PATCH 002/311] table IndexedColumn method --- tensor/table/indexed.go | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tensor/table/indexed.go b/tensor/table/indexed.go index 4690150d08..064642ba24 100644 --- a/tensor/table/indexed.go +++ b/tensor/table/indexed.go @@ -5,14 +5,13 @@ package table import ( - "errors" "fmt" - "log" "math/rand" "slices" "sort" "strings" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -95,6 +94,23 @@ func (ix *Indexed) AddIndex(idx int) { ix.Indexes = append(ix.Indexes, idx) } +// IndexedColumnName returns a tensor.Indexed view of given column name, +// using our current indexes. +func (ix *Indexed) IndexedColumnName(column string) (*tensor.Indexed, error) { //types:add + ci, err := ix.Table.ColumnIndex(column) + if errors.Log(err) != nil { + return nil, err + } + return ix.IndexedColumn(ci), nil +} + +// IndexedColumnName returns a tensor.Indexed view of given column, +// using our current indexes. +func (ix *Indexed) IndexedColumn(colIndex int) *tensor.Indexed { + cl := ix.Table.Columns[colIndex] + return tensor.NewIndexed(cl, ix.Indexes) +} + // SortFunc sorts the indexes into our Table using given compare function. // The compare function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. @@ -119,8 +135,7 @@ func (ix *Indexed) SortIndexes() { // Returns error if column name not found. func (ix *Indexed) SortColumnName(column string, ascending bool) error { //types:add ci, err := ix.Table.ColumnIndex(column) - if err != nil { - log.Println(err) + if errors.Log(err) != nil { return err } ix.SortColumn(ci, ascending) @@ -160,8 +175,7 @@ func (ix *Indexed) SortColumnNames(columns []string, ascending, stable bool) err cis := make([]int, nc) for i, cn := range columns { ci, err := ix.Table.ColumnIndex(cn) - if err != nil { - log.Println(err) + if errors.Log(err) != nil { return err } cis[i] = ci @@ -237,8 +251,7 @@ func (ix *Indexed) SortStableFunc(cmp func(dt *Table, i, j int) int) { // Returns error if column name not found. func (ix *Indexed) SortStableColumnName(column string, ascending bool) error { ci, err := ix.Table.ColumnIndex(column) - if err != nil { - log.Println(err) + if errors.Log(err) != nil { return err } ix.SortStableColumn(ci, ascending) @@ -287,8 +300,7 @@ func (ix *Indexed) Filter(filterer func(dt *Table, row int) bool) { // Returns error if column name not found. func (ix *Indexed) FilterColumnName(column string, str string, exclude, contains, ignoreCase bool) error { //types:add ci, err := ix.Table.ColumnIndex(column) - if err != nil { - log.Println(err) + if errors.Log(err) != nil { return err } ix.FilterColumn(ci, str, exclude, contains, ignoreCase) @@ -420,7 +432,7 @@ func (ix *Indexed) RowsByStringIndex(colIndex int, str string, contains, ignoreC func (ix *Indexed) RowsByString(column string, str string, contains, ignoreCase bool) ([]int, error) { dt := ix.Table ci, err := dt.ColumnIndex(column) - if err != nil { + if errors.Log(err) != nil { return nil, err } return ix.RowsByStringIndex(ci, str, contains, ignoreCase), nil From 95e09f966d446ab6eb4eea7e97ee0527cdda5b93 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 8 Sep 2024 17:49:00 -0700 Subject: [PATCH 003/311] fix metadata refs --- tensor/stats/simat/simat.go | 8 ++++---- tensor/tensorcore/table.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensor/stats/simat/simat.go b/tensor/stats/simat/simat.go index b668e74d85..150a77a730 100644 --- a/tensor/stats/simat/simat.go +++ b/tensor/stats/simat/simat.go @@ -34,7 +34,7 @@ func NewSimMat() *SimMat { // Init initializes SimMat with default Matrix and nil rows, cols func (smat *SimMat) Init() { smat.Mat = &tensor.Float64{} - smat.Mat.SetMetaData("grid-fill", "1") // best for sim mats -- can override later if need to + smat.Mat.Metadata().Set("grid-fill", float32(1)) // best for sim mats -- can override later if need to smat.Rows = nil smat.Columns = nil } @@ -105,12 +105,12 @@ func (smat *SimMat) TableColumn(ix *table.Indexed, column, labNm string, blankRe } if nm, has := ix.Table.MetaData["name"]; has { - sm.SetMetaData("name", nm+"_"+column) + sm.Metadata().Set("name", nm+"_"+column) } else { - sm.SetMetaData("name", column) + sm.Metadata().Set("name", column) } if ds, has := ix.Table.MetaData["desc"]; has { - sm.SetMetaData("desc", ds) + sm.Metadata().Set("desc", ds) } if labNm == "" { diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 69dfbe7161..7083d81f89 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -396,7 +396,7 @@ func (tb *Table) GetColumnTensorDisplay(col int) *TensorDisplay { } if tb.Table != nil { cl := tb.Table.Table.Columns[col] - if len(cl.MetaDataMap()) > 0 { + if len(*cl.Metadata()) > 0 { return tb.SetColumnTensorDisplay(col) } } From bdab27cf2a5ca435582795c4966a8cede06a532d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 9 Sep 2024 02:13:59 -0700 Subject: [PATCH 004/311] vectorize function working, seems like it will work in general. so much cleaner! added new "FromSlice" wrapper constructors so any slice literal can be easily operated on by the same funcs. --- tensor/indexed.go | 10 +- tensor/number.go | 12 + tensor/stats/stats/desc.go | 134 ++--- tensor/stats/stats/floats.go | 730 ------------------------- tensor/stats/stats/floats_test.go | 70 --- tensor/stats/stats/funcs.go | 112 ++-- tensor/stats/stats/funcs_test.go | 57 ++ tensor/stats/stats/if.go | 7 +- tensor/stats/stats/indexview.go | 761 --------------------------- tensor/stats/stats/indexview_test.go | 13 +- tensor/stats/stats/tensor.go | 7 +- tensor/stats/stats/tensor_test.go | 11 +- tensor/string.go | 12 + tensor/vectorize.go | 80 +++ 14 files changed, 319 insertions(+), 1697 deletions(-) delete mode 100644 tensor/stats/stats/floats.go delete mode 100644 tensor/stats/stats/floats_test.go create mode 100644 tensor/stats/stats/funcs_test.go delete mode 100644 tensor/stats/stats/indexview.go create mode 100644 tensor/vectorize.go diff --git a/tensor/indexed.go b/tensor/indexed.go index 9125dbb23a..22d132cd10 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -52,6 +52,11 @@ func (ix *Indexed) SetTensor(tsr Tensor) { ix.Sequential() } +// Len returns the length of the index list +func (ix *Indexed) Len() int { + return len(ix.Indexes) +} + // DeleteInvalid deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. func (ix *Indexed) DeleteInvalid() { @@ -264,11 +269,6 @@ func (ix *Indexed) DeleteRows(at, n int) { ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) } -// Len returns the length of the index list -func (ix *Indexed) Len() int { - return len(ix.Indexes) -} - // Swap switches the indexes for i and j func (ix *Indexed) Swap(i, j int) { ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] diff --git a/tensor/number.go b/tensor/number.go index da84cc1a51..244a3ae970 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -84,6 +84,18 @@ func NewNumberShape[T num.Number](shape *Shape) *Number[T] { return tsr } +// NewNumberFromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewNumberFromSlice[T num.Number](vals []T) Tensor { + n := len(vals) + sizes := []int{n} + tsr := &Number[T]{} + tsr.Values = vals + tsr.SetShape(sizes) + return tsr +} + func (tsr *Number[T]) IsString() bool { return false } diff --git a/tensor/stats/stats/desc.go b/tensor/stats/stats/desc.go index d26869af9a..d0a4ee1607 100644 --- a/tensor/stats/stats/desc.go +++ b/tensor/stats/stats/desc.go @@ -18,85 +18,91 @@ var DescStatsND = []Stats{Count, Mean, Std, Sem, Min, Max} // all numeric columns in given table, operating over all non-Null, non-NaN elements // in each column. func DescAll(ix *table.Indexed) *table.Table { - st := ix.Table - nAgg := len(DescStats) - dt := table.NewTable().SetNumRows(nAgg) - dt.AddStringColumn("Stat") - for ci := range st.Columns { - col := st.Columns[ci] - if col.IsString() { - continue + /* + st := ix.Table + nAgg := len(DescStats) + dt := table.NewTable().SetNumRows(nAgg) + dt.AddStringColumn("Stat") + for ci := range st.Columns { + col := st.Columns[ci] + if col.IsString() { + continue + } + dt.AddFloat64TensorColumn(st.ColumnNames[ci], col.Shape().Sizes[1:], col.Shape().Names[1:]...) } - dt.AddFloat64TensorColumn(st.ColumnNames[ci], col.Shape().Sizes[1:], col.Shape().Names[1:]...) - } - dtnm := dt.Columns[0] - dtci := 1 - qs := []float64{.25, .5, .75} - sq := len(DescStatsND) - for ci := range st.Columns { - col := st.Columns[ci] - if col.IsString() { - continue + dtnm := dt.Columns[0] + dtci := 1 + qs := []float64{.25, .5, .75} + sq := len(DescStatsND) + for ci := range st.Columns { + col := st.Columns[ci] + if col.IsString() { + continue + } + _, csz := col.RowCellSize() + dtst := dt.Columns[dtci] + for i, styp := range DescStatsND { + ag := StatIndex(ix, ci, styp) + ag := 0.0 + si := i * csz + for j := 0; j < csz; j++ { + dtst.SetFloat1D(si+j, ag[j]) + } + if dtci == 1 { + dtnm.SetString1D(i, styp.String()) + } + } + if col.NumDims() == 1 { + qvs := QuantilesIndex(ix, ci, qs) + for i, qv := range qvs { + dtst.SetFloat1D(sq+i, qv) + dtnm.SetString1D(sq+i, DescStats[sq+i].String()) + } + } + dtci++ } + */ + return ix.Table // dt +} + +// DescIndex returns a table of standard descriptive aggregates +// of non-Null, non-NaN elements in given Indexed indexed view of an +// table.Table, for given column index. +func DescIndex(ix *table.Indexed, colIndex int) *table.Table { + /* + st := ix.Table + col := st.Columns[colIndex] + stats := DescStats + if col.NumDims() > 1 { // nd cannot do qiles + stats = DescStatsND + } + nAgg := len(stats) + dt := table.NewTable().SetNumRows(nAgg) + dt.AddStringColumn("Stat") + dt.AddFloat64TensorColumn(st.ColumnNames[colIndex], col.Shape().Sizes[1:], col.Shape().Names[1:]...) + dtnm := dt.Columns[0] + dtst := dt.Columns[1] _, csz := col.RowCellSize() - dtst := dt.Columns[dtci] for i, styp := range DescStatsND { - ag := StatIndex(ix, ci, styp) + // ag := StatIndex(ix, colIndex, styp) + ag := 0.0 si := i * csz for j := 0; j < csz; j++ { dtst.SetFloat1D(si+j, ag[j]) } - if dtci == 1 { - dtnm.SetString1D(i, styp.String()) - } + dtnm.SetString1D(i, styp.String()) } if col.NumDims() == 1 { - qvs := QuantilesIndex(ix, ci, qs) + sq := len(DescStatsND) + qs := []float64{.25, .5, .75} + qvs := QuantilesIndex(ix, colIndex, qs) for i, qv := range qvs { dtst.SetFloat1D(sq+i, qv) dtnm.SetString1D(sq+i, DescStats[sq+i].String()) } } - dtci++ - } - return dt -} - -// DescIndex returns a table of standard descriptive aggregates -// of non-Null, non-NaN elements in given Indexed indexed view of an -// table.Table, for given column index. -func DescIndex(ix *table.Indexed, colIndex int) *table.Table { - st := ix.Table - col := st.Columns[colIndex] - stats := DescStats - if col.NumDims() > 1 { // nd cannot do qiles - stats = DescStatsND - } - nAgg := len(stats) - dt := table.NewTable().SetNumRows(nAgg) - dt.AddStringColumn("Stat") - dt.AddFloat64TensorColumn(st.ColumnNames[colIndex], col.Shape().Sizes[1:], col.Shape().Names[1:]...) - dtnm := dt.Columns[0] - dtst := dt.Columns[1] - _, csz := col.RowCellSize() - for i, styp := range DescStatsND { - ag := StatIndex(ix, colIndex, styp) - si := i * csz - for j := 0; j < csz; j++ { - dtst.SetFloat1D(si+j, ag[j]) - } - dtnm.SetString1D(i, styp.String()) - } - if col.NumDims() == 1 { - sq := len(DescStatsND) - qs := []float64{.25, .5, .75} - qvs := QuantilesIndex(ix, colIndex, qs) - for i, qv := range qvs { - dtst.SetFloat1D(sq+i, qv) - dtnm.SetString1D(sq+i, DescStats[sq+i].String()) - } - } - return dt + */ + return ix.Table // dt } // DescColumn returns a table of standard descriptive stats diff --git a/tensor/stats/stats/floats.go b/tensor/stats/stats/floats.go deleted file mode 100644 index dff87d1152..0000000000 --- a/tensor/stats/stats/floats.go +++ /dev/null @@ -1,730 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -import ( - "math" - - "cogentcore.org/core/math32" -) - -// Stat32 returns statistic according to given Stats type applied -// to all non-NaN elements in given slice of float32 -func Stat32(a []float32, stat Stats) float32 { - switch stat { - case Count: - return Count32(a) - case Sum: - return Sum32(a) - case Prod: - return Prod32(a) - case Min: - return Min32(a) - case Max: - return Max32(a) - case MinAbs: - return MinAbs32(a) - case MaxAbs: - return MaxAbs32(a) - case Mean: - return Mean32(a) - case Var: - return Var32(a) - case Std: - return Std32(a) - case Sem: - return Sem32(a) - case L1Norm: - return L1Norm32(a) - case SumSq: - return SumSq32(a) - case L2Norm: - return L2Norm32(a) - case VarPop: - return VarPop32(a) - case StdPop: - return StdPop32(a) - case SemPop: - return SemPop32(a) - // case Median: - // return Median32(a) - // case Q1: - // return Q132(a) - // case Q3: - // return Q332(a) - } - return 0 -} - -// Stat64 returns statistic according to given Stats type applied -// to all non-NaN elements in given slice of float64 -func Stat64(a []float64, stat Stats) float64 { - switch stat { - case Count: - return Count64(a) - case Sum: - return Sum64(a) - case Prod: - return Prod64(a) - case Min: - return Min64(a) - case Max: - return Max64(a) - case MinAbs: - return MinAbs64(a) - case MaxAbs: - return MaxAbs64(a) - case Mean: - return Mean64(a) - case Var: - return Var64(a) - case Std: - return Std64(a) - case Sem: - return Sem64(a) - case L1Norm: - return L1Norm64(a) - case SumSq: - return SumSq64(a) - case L2Norm: - return L2Norm64(a) - case VarPop: - return VarPop64(a) - case StdPop: - return StdPop64(a) - case SemPop: - return SemPop64(a) - // case Median: - // return Median64(a) - // case Q1: - // return Q164(a) - // case Q3: - // return Q364(a) - } - return 0 -} - -/////////////////////////////////////////// -// Count - -// Count32 computes the number of non-NaN vector values. -// Skips NaN's -func Count32(a []float32) float32 { - n := 0 - for _, av := range a { - if math32.IsNaN(av) { - continue - } - n++ - } - return float32(n) -} - -// Count64 computes the number of non-NaN vector values. -// Skips NaN's -func Count64(a []float64) float64 { - n := 0 - for _, av := range a { - if math.IsNaN(av) { - continue - } - n++ - } - return float64(n) -} - -/////////////////////////////////////////// -// Sum - -// Sum32 computes the sum of vector values. -// Skips NaN's -func Sum32(a []float32) float32 { - s := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - s += av - } - return s -} - -// Sum64 computes the sum of vector values. -// Skips NaN's -func Sum64(a []float64) float64 { - s := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - s += av - } - return s -} - -/////////////////////////////////////////// -// Prod - -// Prod32 computes the product of vector values. -// Skips NaN's -func Prod32(a []float32) float32 { - s := float32(1) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - s *= av - } - return s -} - -// Prod64 computes the product of vector values. -// Skips NaN's -func Prod64(a []float64) float64 { - s := float64(1) - for _, av := range a { - if math.IsNaN(av) { - continue - } - s *= av - } - return s -} - -/////////////////////////////////////////// -// Min - -// Min32 computes the max over vector values. -// Skips NaN's -func Min32(a []float32) float32 { - m := float32(math.MaxFloat32) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - m = math32.Min(m, av) - } - return m -} - -// MinIndex32 computes the min over vector values, and returns index of min as well -// Skips NaN's -func MinIndex32(a []float32) (float32, int) { - m := float32(math.MaxFloat32) - mi := -1 - for i, av := range a { - if math32.IsNaN(av) { - continue - } - if av < m { - m = av - mi = i - } - } - return m, mi -} - -// Min64 computes the max over vector values. -// Skips NaN's -func Min64(a []float64) float64 { - m := float64(math.MaxFloat64) - for _, av := range a { - if math.IsNaN(av) { - continue - } - m = math.Min(m, av) - } - return m -} - -// MinIndex64 computes the min over vector values, and returns index of min as well -// Skips NaN's -func MinIndex64(a []float64) (float64, int) { - m := float64(math.MaxFloat64) - mi := -1 - for i, av := range a { - if math.IsNaN(av) { - continue - } - if av < m { - m = av - mi = i - } - } - return m, mi -} - -/////////////////////////////////////////// -// Max - -// Max32 computes the max over vector values. -// Skips NaN's -func Max32(a []float32) float32 { - m := float32(-math.MaxFloat32) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - m = math32.Max(m, av) - } - return m -} - -// MaxIndex32 computes the max over vector values, and returns index of max as well -// Skips NaN's -func MaxIndex32(a []float32) (float32, int) { - m := float32(-math.MaxFloat32) - mi := -1 - for i, av := range a { - if math32.IsNaN(av) { - continue - } - if av > m { - m = av - mi = i - } - } - return m, mi -} - -// Max64 computes the max over vector values. -// Skips NaN's -func Max64(a []float64) float64 { - m := float64(-math.MaxFloat64) - for _, av := range a { - if math.IsNaN(av) { - continue - } - m = math.Max(m, av) - } - return m -} - -// MaxIndex64 computes the max over vector values, and returns index of max as well -// Skips NaN's -func MaxIndex64(a []float64) (float64, int) { - m := float64(-math.MaxFloat64) - mi := -1 - for i, av := range a { - if math.IsNaN(av) { - continue - } - if av > m { - m = av - mi = i - } - } - return m, mi -} - -/////////////////////////////////////////// -// MinAbs - -// MinAbs32 computes the max of absolute value over vector values. -// Skips NaN's -func MinAbs32(a []float32) float32 { - m := float32(math.MaxFloat32) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - m = math32.Min(m, math32.Abs(av)) - } - return m -} - -// MinAbs64 computes the max over vector values. -// Skips NaN's -func MinAbs64(a []float64) float64 { - m := float64(math.MaxFloat64) - for _, av := range a { - if math.IsNaN(av) { - continue - } - m = math.Min(m, math.Abs(av)) - } - return m -} - -/////////////////////////////////////////// -// MaxAbs - -// MaxAbs32 computes the max of absolute value over vector values. -// Skips NaN's -func MaxAbs32(a []float32) float32 { - m := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - m = math32.Max(m, math32.Abs(av)) - } - return m -} - -// MaxAbs64 computes the max over vector values. -// Skips NaN's -func MaxAbs64(a []float64) float64 { - m := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - m = math.Max(m, math.Abs(av)) - } - return m -} - -/////////////////////////////////////////// -// Mean - -// Mean32 computes the mean of the vector (sum / N). -// Skips NaN's -func Mean32(a []float32) float32 { - s := float32(0) - n := 0 - for _, av := range a { - if math32.IsNaN(av) { - continue - } - s += av - n++ - } - if n > 0 { - s /= float32(n) - } - return s -} - -// Mean64 computes the mean of the vector (sum / N). -// Skips NaN's -func Mean64(a []float64) float64 { - s := float64(0) - n := 0 - for _, av := range a { - if math.IsNaN(av) { - continue - } - s += av - n++ - } - if n > 0 { - s /= float64(n) - } - return s -} - -/////////////////////////////////////////// -// Var - -// Var32 returns the sample variance of non-NaN elements. -func Var32(a []float32) float32 { - mean := Mean32(a) - n := 0 - s := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 1 { - s /= float32(n - 1) - } - return s -} - -// Var64 returns the sample variance of non-NaN elements. -func Var64(a []float64) float64 { - mean := Mean64(a) - n := 0 - s := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 1 { - s /= float64(n - 1) - } - return s -} - -/////////////////////////////////////////// -// Std - -// Std32 returns the sample standard deviation of non-NaN elements in vector. -func Std32(a []float32) float32 { - return math32.Sqrt(Var32(a)) -} - -// Std64 returns the sample standard deviation of non-NaN elements in vector. -func Std64(a []float64) float64 { - return math.Sqrt(Var64(a)) -} - -/////////////////////////////////////////// -// Sem - -// Sem32 returns the sample standard error of the mean of non-NaN elements in vector. -func Sem32(a []float32) float32 { - cnt := Count32(a) - if cnt < 2 { - return 0 - } - return Std32(a) / math32.Sqrt(cnt) -} - -// Sem64 returns the sample standard error of the mean of non-NaN elements in vector. -func Sem64(a []float64) float64 { - cnt := Count64(a) - if cnt < 2 { - return 0 - } - return Std64(a) / math.Sqrt(cnt) -} - -/////////////////////////////////////////// -// L1Norm - -// L1Norm32 computes the sum of absolute values (L1 Norm). -// Skips NaN's -func L1Norm32(a []float32) float32 { - ss := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - ss += math32.Abs(av) - } - return ss -} - -// L1Norm64 computes the sum of absolute values (L1 Norm). -// Skips NaN's -func L1Norm64(a []float64) float64 { - ss := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - ss += math.Abs(av) - } - return ss -} - -/////////////////////////////////////////// -// SumSquares - -// SumSq32 computes the sum-of-squares of vector. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func SumSq32(a []float32) float32 { - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0]) - } - return 0 - } - var ( - scale float32 = 0 - sumSquares float32 = 1 - ) - for _, v := range a { - if v == 0 || math32.IsNaN(v) { - continue - } - absxi := math32.Abs(v) - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * scale * sumSquares -} - -// SumSq64 computes the sum-of-squares of vector. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func SumSq64(a []float64) float64 { - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0]) - } - return 0 - } - var ( - scale float64 = 0 - ss float64 = 1 - ) - for _, v := range a { - if v == 0 || math.IsNaN(v) { - continue - } - absxi := math.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * scale * ss -} - -/////////////////////////////////////////// -// L2Norm - -// L2Norm32 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func L2Norm32(a []float32) float32 { - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0]) - } - return 0 - } - var ( - scale float32 = 0 - ss float32 = 1 - ) - for _, v := range a { - if v == 0 || math32.IsNaN(v) { - continue - } - absxi := math32.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * math32.Sqrt(ss) -} - -// L2Norm64 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func L2Norm64(a []float64) float64 { - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0]) - } - return 0 - } - var ( - scale float64 = 0 - ss float64 = 1 - ) - for _, v := range a { - if v == 0 || math.IsNaN(v) { - continue - } - absxi := math.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * math.Sqrt(ss) -} - -/////////////////////////////////////////// -// VarPop - -// VarPop32 returns the population variance of non-NaN elements. -func VarPop32(a []float32) float32 { - mean := Mean32(a) - n := 0 - s := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 0 { - s /= float32(n) - } - return s -} - -// VarPop64 returns the population variance of non-NaN elements. -func VarPop64(a []float64) float64 { - mean := Mean64(a) - n := 0 - s := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 0 { - s /= float64(n) - } - return s -} - -/////////////////////////////////////////// -// StdPop - -// StdPop32 returns the population standard deviation of non-NaN elements in vector. -func StdPop32(a []float32) float32 { - return math32.Sqrt(VarPop32(a)) -} - -// StdPop64 returns the population standard deviation of non-NaN elements in vector. -func StdPop64(a []float64) float64 { - return math.Sqrt(VarPop64(a)) -} - -/////////////////////////////////////////// -// SemPop - -// SemPop32 returns the population standard error of the mean of non-NaN elements in vector. -func SemPop32(a []float32) float32 { - cnt := Count32(a) - if cnt < 2 { - return 0 - } - return StdPop32(a) / math32.Sqrt(cnt) -} - -// SemPop64 returns the population standard error of the mean of non-NaN elements in vector. -func SemPop64(a []float64) float64 { - cnt := Count64(a) - if cnt < 2 { - return 0 - } - return StdPop64(a) / math.Sqrt(cnt) -} diff --git a/tensor/stats/stats/floats_test.go b/tensor/stats/stats/floats_test.go deleted file mode 100644 index 2f8dc16d47..0000000000 --- a/tensor/stats/stats/floats_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -import ( - "math" - "testing" - - "cogentcore.org/core/base/tolassert" - "cogentcore.org/core/math32" - "github.com/stretchr/testify/assert" -) - -func TestStats32(t *testing.T) { - vals := []float32{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - - results := []float32{11, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math32.Sqrt(0.11), math32.Sqrt(0.11) / math32.Sqrt(11), 5.5, 3.85, math32.Sqrt(3.85), 0.1, math32.Sqrt(0.1), math32.Sqrt(0.1) / math32.Sqrt(11)} - - assert.Equal(t, results[Count], Count32(vals)) - assert.Equal(t, results[Sum], Sum32(vals)) - assert.Equal(t, results[Prod], Prod32(vals)) - assert.Equal(t, results[Min], Min32(vals)) - assert.Equal(t, results[Max], Max32(vals)) - assert.Equal(t, results[MinAbs], MinAbs32(vals)) - assert.Equal(t, results[MaxAbs], MaxAbs32(vals)) - assert.Equal(t, results[Mean], Mean32(vals)) - assert.Equal(t, results[Var], Var32(vals)) - assert.Equal(t, results[Std], Std32(vals)) - assert.Equal(t, results[Sem], Sem32(vals)) - assert.Equal(t, results[L1Norm], L1Norm32(vals)) - tolassert.EqualTol(t, results[SumSq], SumSq32(vals), 1.0e-6) - tolassert.EqualTol(t, results[L2Norm], L2Norm32(vals), 1.0e-6) - assert.Equal(t, results[VarPop], VarPop32(vals)) - assert.Equal(t, results[StdPop], StdPop32(vals)) - assert.Equal(t, results[SemPop], SemPop32(vals)) - - for stat := Count; stat <= SemPop; stat++ { - tolassert.EqualTol(t, results[stat], Stat32(vals, stat), 1.0e-6) - } -} - -func TestStats64(t *testing.T) { - vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - - results := []float64{11, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 5.5, 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11)} - - assert.Equal(t, results[Count], Count64(vals)) - assert.Equal(t, results[Sum], Sum64(vals)) - assert.Equal(t, results[Prod], Prod64(vals)) - assert.Equal(t, results[Min], Min64(vals)) - assert.Equal(t, results[Max], Max64(vals)) - assert.Equal(t, results[MinAbs], MinAbs64(vals)) - assert.Equal(t, results[MaxAbs], MaxAbs64(vals)) - assert.Equal(t, results[Mean], Mean64(vals)) - tolassert.EqualTol(t, results[Var], Var64(vals), 1.0e-8) - tolassert.EqualTol(t, results[Std], Std64(vals), 1.0e-8) - tolassert.EqualTol(t, results[Sem], Sem64(vals), 1.0e-8) - assert.Equal(t, results[L1Norm], L1Norm64(vals)) - tolassert.EqualTol(t, results[SumSq], SumSq64(vals), 1.0e-8) - tolassert.EqualTol(t, results[L2Norm], L2Norm64(vals), 1.0e-8) - assert.Equal(t, results[VarPop], VarPop64(vals)) - assert.Equal(t, results[StdPop], StdPop64(vals)) - assert.Equal(t, results[SemPop], SemPop64(vals)) - - for stat := Count; stat <= SemPop; stat++ { - tolassert.EqualTol(t, results[stat], Stat64(vals, stat), 1.0e-8) - } -} diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 76f23dacef..3f819187f9 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -4,62 +4,98 @@ package stats -import "math" +import ( + "math" -// These are standard StatFunc functions that can operate on tensor.Tensor -// or table.Table, using float64 values + "cogentcore.org/core/tensor" +) -// StatFunc is an statistic function that incrementally updates agg -// aggregation value from each element in the tensor in turn. -// Returns new agg value that will be passed into next item as agg. -type StatFunc func(idx int, val float64, agg float64) float64 +// NFunc is the nfun for stats functions, return the length of the +// first tensor, and initializing the second one to hold the output +// as the SubSpace of the first tensor. +func NFunc(tsr ...*tensor.Indexed) int { + if len(tsr) != 2 { + return 0 + } + n := tsr[0].Len() + sh := tensor.Shape{} + sh.CopyShape(tsr[0].Tensor.Shape()) + sh.Sizes[0] = 1 + tsr[1].Tensor.SetShape(sh.Sizes, sh.Names...) + tsr[1].Indexes = []int{0} + return n +} -// CountFunc is an StatFunc that computes number of elements (non-Null, non-NaN) -// Use 0 as initial value. -func CountFunc(idx int, val float64, agg float64) float64 { - return agg + 1 +// StatFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// It also skips over NaN missing values. +func StatFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { + nsub := out.Len() + for i := 0; i < nsub; i++ { + if idx == 0 { + out.Tensor.SetFloat1D(i, ini) + } + val := in.Tensor.FloatRowCell(in.Indexes[idx], i) + if math.IsNaN(val) { + continue + } + out.Tensor.SetFloat1D(i, fun(val, out.Tensor.Float1D(i))) + } } -// SumFunc is an StatFunc that computes a sum aggregate. -// use 0 as initial value. -func SumFunc(idx int, val float64, agg float64) float64 { - return agg + val +// SumFunc is a Vectorize function for computing the sum +func SumFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + val + }) } -// Prodfunc is an StatFunc that computes a product aggregate. -// use 1 as initial value. -func ProdFunc(idx int, val float64, agg float64) float64 { - return agg * val +// CountFunc is a Vectorize function for computing the count +func CountFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + 1 + }) } -// MinFunc is an StatFunc that computes a min aggregate. -// use math.MaxFloat64 for initial agg value. -func MinFunc(idx int, val float64, agg float64) float64 { - return math.Min(agg, val) +// ProdFunc is a Vectorize function for computing the product +func ProdFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { + return agg * val + }) } -// MaxFunc is an StatFunc that computes a max aggregate. -// use -math.MaxFloat64 for initial agg value. -func MaxFunc(idx int, val float64, agg float64) float64 { - return math.Max(agg, val) +// MinFunc is a Vectorize function for computing the min +func MinFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, val) + }) } -// MinAbsFunc is an StatFunc that computes a min aggregate. -// use math.MaxFloat64 for initial agg value. -func MinAbsFunc(idx int, val float64, agg float64) float64 { - return math.Min(agg, math.Abs(val)) +// MaxFunc is a Vectorize function for computing the min +func MaxFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, val) + }) } -// MaxAbsFunc is an StatFunc that computes a max aggregate. -// use -math.MaxFloat64 for initial agg value. -func MaxAbsFunc(idx int, val float64, agg float64) float64 { - return math.Max(agg, math.Abs(val)) +// MinAbsFunc is a Vectorize function for computing the min of abs values +func MinAbsFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, math.Abs(val)) + }) +} + +// MaxAbsFunc is a Vectorize function for computing the max of abs values +func MaxAbsFunc(idx int, tsr ...*tensor.Indexed) { + StatFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, math.Abs(val)) + }) } // L1NormFunc is an StatFunc that computes the L1 norm: sum of absolute values // use 0 as initial value. -func L1NormFunc(idx int, val float64, agg float64) float64 { - return agg + math.Abs(val) -} +// func L1NormFunc(idx int, val float64, agg float64) float64 { +// return agg + math.Abs(val) +// } // Note: SumSq is not numerically stable for large N in simple func form. diff --git a/tensor/stats/stats/funcs_test.go b/tensor/stats/stats/funcs_test.go new file mode 100644 index 0000000000..79beb996d0 --- /dev/null +++ b/tensor/stats/stats/funcs_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "math" + "testing" + + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" +) + +func TestFuncs(t *testing.T) { + vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} + tsr := tensor.NewNumberFromSlice(vals) + ix := tensor.NewIndexed(tsr) + out := tensor.NewFloat64([]int{1}) + oix := tensor.NewIndexed(out) + + results := []float64{11, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 5.5, 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11)} + + // assert.Equal(t, results[Count], Count64(vals)) + tensor.Vectorize(NFunc, SumFunc, ix, oix) + assert.Equal(t, results[Sum], out.Values[0]) + + tensor.Vectorize(NFunc, ProdFunc, ix, oix) + assert.Equal(t, results[Prod], out.Values[0]) + + tensor.Vectorize(NFunc, MinFunc, ix, oix) + assert.Equal(t, results[Min], out.Values[0]) + + tensor.Vectorize(NFunc, MaxFunc, ix, oix) + assert.Equal(t, results[Max], out.Values[0]) + + tensor.Vectorize(NFunc, MinAbsFunc, ix, oix) + assert.Equal(t, results[MinAbs], out.Values[0]) + + tensor.Vectorize(NFunc, MaxAbsFunc, ix, oix) + assert.Equal(t, results[MaxAbs], out.Values[0]) + + // assert.Equal(t, results[Mean], Mean64(vals)) + // tolassert.EqualTol(t, results[Var], Var64(vals), 1.0e-8) + // tolassert.EqualTol(t, results[Std], Std64(vals), 1.0e-8) + // tolassert.EqualTol(t, results[Sem], Sem64(vals), 1.0e-8) + // assert.Equal(t, results[L1Norm], L1Norm64(vals)) + // tolassert.EqualTol(t, results[SumSq], SumSq64(vals), 1.0e-8) + // tolassert.EqualTol(t, results[L2Norm], L2Norm64(vals), 1.0e-8) + // assert.Equal(t, results[VarPop], VarPop64(vals)) + // assert.Equal(t, results[StdPop], StdPop64(vals)) + // assert.Equal(t, results[SemPop], SemPop64(vals)) + // + // for stat := Count; stat <= SemPop; stat++ { + // tolassert.EqualTol(t, results[stat], Stat64(vals, stat), 1.0e-8) + // } +} diff --git a/tensor/stats/stats/if.go b/tensor/stats/stats/if.go index 1673784b16..3293ca7bbc 100644 --- a/tensor/stats/stats/if.go +++ b/tensor/stats/stats/if.go @@ -4,10 +4,7 @@ package stats -import ( - "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor/table" -) +/* // IfFunc is used for the *If aggregators -- counted if it returns true type IfFunc func(idx int, val float64) bool @@ -114,3 +111,5 @@ func PctIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { } return PctIfIndex(ix, colIndex, iffun) } + +*/ diff --git a/tensor/stats/stats/indexview.go b/tensor/stats/stats/indexview.go deleted file mode 100644 index 18cd995f52..0000000000 --- a/tensor/stats/stats/indexview.go +++ /dev/null @@ -1,761 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -import ( - "math" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor/table" -) - -// Every Indexed Stats method in this file follows one of these signatures: - -// IndexedFuncIndex is a stats function operating on Indexed, taking a column index arg -type IndexedFuncIndex func(ix *table.Indexed, colIndex int) []float64 - -// IndexedFuncColumn is a stats function operating on Indexed, taking a column name arg -type IndexedFuncColumn func(ix *table.Indexed, column string) []float64 - -// StatIndex returns Indexed statistic according to given Stats type applied -// to all non-NaN elements in given Indexed indexed view of -// an table.Table, for given column index. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func StatIndex(ix *table.Indexed, colIndex int, stat Stats) []float64 { - switch stat { - case Count: - return CountIndex(ix, colIndex) - case Sum: - return SumIndex(ix, colIndex) - case Prod: - return ProdIndex(ix, colIndex) - case Min: - return MinIndex(ix, colIndex) - case Max: - return MaxIndex(ix, colIndex) - case MinAbs: - return MinAbsIndex(ix, colIndex) - case MaxAbs: - return MaxAbsIndex(ix, colIndex) - case Mean: - return MeanIndex(ix, colIndex) - case Var: - return VarIndex(ix, colIndex) - case Std: - return StdIndex(ix, colIndex) - case Sem: - return SemIndex(ix, colIndex) - case L1Norm: - return L1NormIndex(ix, colIndex) - case SumSq: - return SumSqIndex(ix, colIndex) - case L2Norm: - return L2NormIndex(ix, colIndex) - case VarPop: - return VarPopIndex(ix, colIndex) - case StdPop: - return StdPopIndex(ix, colIndex) - case SemPop: - return SemPopIndex(ix, colIndex) - case Median: - return MedianIndex(ix, colIndex) - case Q1: - return Q1Index(ix, colIndex) - case Q3: - return Q3Index(ix, colIndex) - } - return nil -} - -// StatColumn returns Indexed statistic according to given Stats type applied -// to all non-NaN elements in given Indexed indexed view of -// an table.Table, for given column name. -// If name not found, returns error message. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func StatColumn(ix *table.Indexed, column string, stat Stats) ([]float64, error) { - colIndex, err := ix.Table.ColumnIndex(column) - if err != nil { - return nil, err - } - rv := StatIndex(ix, colIndex, stat) - return rv, nil -} - -// StatIndexFunc applies given StatFunc function to each element in the given column, -// using float64 conversions of the values. ini is the initial value for the agg variable. -// Operates independently over each cell on n-dimensional columns and returns the result as a slice -// of values per cell. -func StatIndexFunc(ix *table.Indexed, colIndex int, ini float64, fun StatFunc) []float64 { - cl := ix.Table.Columns[colIndex] - _, csz := cl.RowCellSize() - - ag := make([]float64, csz) - for i := range ag { - ag[i] = ini - } - if csz == 1 { - for _, srw := range ix.Indexes { - val := cl.Float1D(srw) - if !math.IsNaN(val) { - ag[0] = fun(srw, val, ag[0]) - } - } - } else { - for _, srw := range ix.Indexes { - si := srw * csz - for j := range ag { - val := cl.Float1D(si + j) - if !math.IsNaN(val) { - ag[j] = fun(si+j, val, ag[j]) - } - } - } - } - return ag -} - -/////////////////////////////////////////////////// -// Count - -// CountIndex returns the count of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func CountIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, 0, CountFunc) -} - -// CountColumn returns the count of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func CountColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return CountIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Sum - -// SumIndex returns the sum of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SumIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, 0, SumFunc) -} - -// SumColumn returns the sum of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SumColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return SumIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Prod - -// ProdIndex returns the product of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func ProdIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, 1, ProdFunc) -} - -// ProdColumn returns the product of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func ProdColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return ProdIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Min - -// MinIndex returns the minimum of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MinIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, math.MaxFloat64, MinFunc) -} - -// MinColumn returns the minimum of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MinColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return MinIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Max - -// MaxIndex returns the maximum of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MaxIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, -math.MaxFloat64, MaxFunc) -} - -// MaxColumn returns the maximum of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MaxColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return MaxIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// MinAbs - -// MinAbsIndex returns the minimum of abs of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MinAbsIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, math.MaxFloat64, MinAbsFunc) -} - -// MinAbsColumn returns the minimum of abs of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MinAbsColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return MinAbsIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// MaxAbs - -// MaxAbsIndex returns the maximum of abs of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MaxAbsIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, -math.MaxFloat64, MaxAbsFunc) -} - -// MaxAbsColumn returns the maximum of abs of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MaxAbsColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return MaxAbsIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Mean - -// MeanIndex returns the mean of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MeanIndex(ix *table.Indexed, colIndex int) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - mean := SumIndex(ix, colIndex) - for i := range mean { - if cnt[i] > 0 { - mean[i] /= cnt[i] - } - } - return mean -} - -// MeanColumn returns the mean of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MeanColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return MeanIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Var - -// VarIndex returns the sample variance of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Sample variance is normalized by 1/(n-1) -- see VarPop version for 1/n normalization. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func VarIndex(ix *table.Indexed, colIndex int) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - mean := SumIndex(ix, colIndex) - for i := range mean { - if cnt[i] > 0 { - mean[i] /= cnt[i] - } - } - col := ix.Table.Columns[colIndex] - _, csz := col.RowCellSize() - vr := StatIndexFunc(ix, colIndex, 0, func(idx int, val float64, agg float64) float64 { - cidx := idx % csz - dv := val - mean[cidx] - return agg + dv*dv - }) - for i := range vr { - if cnt[i] > 1 { - vr[i] /= (cnt[i] - 1) - } - } - return vr -} - -// VarColumn returns the sample variance of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// Sample variance is normalized by 1/(n-1) -- see VarPop version for 1/n normalization. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func VarColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return VarIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Std - -// StdIndex returns the sample std deviation of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Sample std deviation is normalized by 1/(n-1) -- see StdPop version for 1/n normalization. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func StdIndex(ix *table.Indexed, colIndex int) []float64 { - std := VarIndex(ix, colIndex) - for i := range std { - std[i] = math.Sqrt(std[i]) - } - return std -} - -// StdColumn returns the sample std deviation of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// Sample std deviation is normalized by 1/(n-1) -- see StdPop version for 1/n normalization. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func StdColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return StdIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Sem - -// SemIndex returns the sample standard error of the mean of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Sample sem is normalized by 1/(n-1) -- see SemPop version for 1/n normalization. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SemIndex(ix *table.Indexed, colIndex int) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - sem := StdIndex(ix, colIndex) - for i := range sem { - if cnt[i] > 0 { - sem[i] /= math.Sqrt(cnt[i]) - } - } - return sem -} - -// SemColumn returns the sample standard error of the mean of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// Sample sem is normalized by 1/(n-1) -- see SemPop version for 1/n normalization. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SemColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return SemIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// L1Norm - -// L1NormIndex returns the L1 norm (sum abs values) of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func L1NormIndex(ix *table.Indexed, colIndex int) []float64 { - return StatIndexFunc(ix, colIndex, 0, L1NormFunc) -} - -// L1NormColumn returns the L1 norm (sum abs values) of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func L1NormColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return L1NormIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// SumSq - -// SumSqIndex returns the sum-of-squares of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SumSqIndex(ix *table.Indexed, colIndex int) []float64 { - cl := ix.Table.Columns[colIndex] - _, csz := cl.RowCellSize() - - scale := make([]float64, csz) - ss := make([]float64, csz) - for i := range ss { - ss[i] = 1 - } - n := len(ix.Indexes) - if csz == 1 { - if n < 2 { - if n == 1 { - ss[0] = math.Abs(cl.Float1D(ix.Indexes[0])) - return ss - } - return scale // all 0s - } - for _, srw := range ix.Indexes { - v := cl.Float1D(srw) - absxi := math.Abs(v) - if scale[0] < absxi { - ss[0] = 1 + ss[0]*(scale[0]/absxi)*(scale[0]/absxi) - scale[0] = absxi - } else { - ss[0] = ss[0] + (absxi/scale[0])*(absxi/scale[0]) - } - } - if math.IsInf(scale[0], 1) { - ss[0] = math.Inf(1) - } else { - ss[0] = scale[0] * scale[0] * ss[0] - } - } else { - if n < 2 { - if n == 1 { - si := csz * ix.Indexes[0] - for j := range csz { - ss[j] = math.Abs(cl.Float1D(si + j)) - } - return ss - } - return scale // all 0s - } - for _, srw := range ix.Indexes { - si := srw * csz - for j := range ss { - v := cl.Float1D(si + j) - absxi := math.Abs(v) - if scale[j] < absxi { - ss[j] = 1 + ss[j]*(scale[j]/absxi)*(scale[j]/absxi) - scale[j] = absxi - } else { - ss[j] = ss[j] + (absxi/scale[j])*(absxi/scale[j]) - } - } - } - for j := range ss { - if math.IsInf(scale[j], 1) { - ss[j] = math.Inf(1) - } else { - ss[j] = scale[j] * scale[j] * ss[j] - } - } - } - return ss -} - -// SumSqColumn returns the sum-of-squares of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SumSqColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return SumSqIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// L2Norm - -// L2NormIndex returns the L2 norm (square root of sum-of-squares) of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func L2NormIndex(ix *table.Indexed, colIndex int) []float64 { - ss := SumSqIndex(ix, colIndex) - for i := range ss { - ss[i] = math.Sqrt(ss[i]) - } - return ss -} - -// L2NormColumn returns the L2 norm (square root of sum-of-squares) of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func L2NormColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return L2NormIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// VarPop - -// VarPopIndex returns the population variance of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// population variance is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func VarPopIndex(ix *table.Indexed, colIndex int) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - mean := SumIndex(ix, colIndex) - for i := range mean { - if cnt[i] > 0 { - mean[i] /= cnt[i] - } - } - col := ix.Table.Columns[colIndex] - _, csz := col.RowCellSize() - vr := StatIndexFunc(ix, colIndex, 0, func(idx int, val float64, agg float64) float64 { - cidx := idx % csz - dv := val - mean[cidx] - return agg + dv*dv - }) - for i := range vr { - if cnt[i] > 0 { - vr[i] /= cnt[i] - } - } - return vr -} - -// VarPopColumn returns the population variance of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// population variance is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func VarPopColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return VarPopIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// StdPop - -// StdPopIndex returns the population std deviation of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// population std dev is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func StdPopIndex(ix *table.Indexed, colIndex int) []float64 { - std := VarPopIndex(ix, colIndex) - for i := range std { - std[i] = math.Sqrt(std[i]) - } - return std -} - -// StdPopColumn returns the population std deviation of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// population std dev is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func StdPopColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return StdPopIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// SemPop - -// SemPopIndex returns the population standard error of the mean of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// population sem is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SemPopIndex(ix *table.Indexed, colIndex int) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - sem := StdPopIndex(ix, colIndex) - for i := range sem { - if cnt[i] > 0 { - sem[i] /= math.Sqrt(cnt[i]) - } - } - return sem -} - -// SemPopColumn returns the standard error of the mean of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// population sem is normalized by 1/n -- see Var version for 1/(n-1) sample normalization. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func SemPopColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return SemPopIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Median - -// MedianIndex returns the median of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MedianIndex(ix *table.Indexed, colIndex int) []float64 { - return QuantilesIndex(ix, colIndex, []float64{.5}) -} - -// MedianColumn returns the median of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func MedianColumn(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return MedianIndex(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Q1 - -// Q1Index returns the first quartile of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func Q1Index(ix *table.Indexed, colIndex int) []float64 { - return QuantilesIndex(ix, colIndex, []float64{.25}) -} - -// Q1Column returns the first quartile of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func Q1Column(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return Q1Index(ix, colIndex) -} - -/////////////////////////////////////////////////// -// Q3 - -// Q3Index returns the third quartile of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func Q3Index(ix *table.Indexed, colIndex int) []float64 { - return QuantilesIndex(ix, colIndex, []float64{.75}) -} - -// Q3Column returns the third quartile of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func Q3Column(ix *table.Indexed, column string) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return Q3Index(ix, colIndex) -} diff --git a/tensor/stats/stats/indexview_test.go b/tensor/stats/stats/indexview_test.go index 66c89f4afc..26433f937a 100644 --- a/tensor/stats/stats/indexview_test.go +++ b/tensor/stats/stats/indexview_test.go @@ -4,17 +4,7 @@ package stats -import ( - "math" - "testing" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/tolassert" - "cogentcore.org/core/tensor/table" - - "github.com/stretchr/testify/assert" -) - +/* func TestIndexed(t *testing.T) { dt := table.NewTable().SetNumRows(5) dt.AddFloat64Column("data") @@ -86,3 +76,4 @@ func TestIndexed(t *testing.T) { }) assert.Equal(t, []float64{0.6}, props) } +*/ diff --git a/tensor/stats/stats/tensor.go b/tensor/stats/stats/tensor.go index bc33041d8e..f51ac42493 100644 --- a/tensor/stats/stats/tensor.go +++ b/tensor/stats/stats/tensor.go @@ -4,11 +4,7 @@ package stats -import ( - "math" - - "cogentcore.org/core/tensor" -) +/* // StatTensor returns Tensor statistic according to given Stats type applied // to all non-NaN elements in given Tensor @@ -214,3 +210,4 @@ func SemPopTensor(tsr tensor.Tensor) float64 { } return StdPopTensor(tsr) / math.Sqrt(cnt) } +*/ diff --git a/tensor/stats/stats/tensor_test.go b/tensor/stats/stats/tensor_test.go index 663ff43f50..328b9bf338 100644 --- a/tensor/stats/stats/tensor_test.go +++ b/tensor/stats/stats/tensor_test.go @@ -4,15 +4,7 @@ package stats -import ( - "math" - "testing" - - "cogentcore.org/core/base/tolassert" - "cogentcore.org/core/tensor" - "github.com/stretchr/testify/assert" -) - +/* func TestTsrAgg(t *testing.T) { tsr := tensor.New[float64]([]int{5}).(*tensor.Float64) tsr.Values = []float64{1, 2, 3, 4, 5} @@ -42,3 +34,4 @@ func TestTsrAgg(t *testing.T) { tolassert.EqualTol(t, results[stat], StatTensor(tsr, stat), 1.0e-8) } } +*/ diff --git a/tensor/string.go b/tensor/string.go index 9a353ccd2b..2cf04fb963 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -38,6 +38,18 @@ func NewStringShape(shape *Shape) *String { return tsr } +// NewStringFromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewStringFromSlice(vals []string) Tensor { + n := len(vals) + sizes := []int{n} + tsr := &String{} + tsr.Values = vals + tsr.SetShape(sizes) + return tsr +} + // StringToFloat64 converts string value to float64 using strconv, // returning 0 if any error func StringToFloat64(str string) float64 { diff --git a/tensor/vectorize.go b/tensor/vectorize.go new file mode 100644 index 0000000000..f4dbabb7f9 --- /dev/null +++ b/tensor/vectorize.go @@ -0,0 +1,80 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "math" + "sync" +) + +// Vectorize applies given function 'fun' to tensor elements indexed +// by given index, with the 'nfun' providing the number of indexes +// to vectorize over, and initializing any output vectors. +// Thus the nfun is often specific to a particular class of functions. +// Both functions are called with the same set +// of Indexed Tensors passed as the final argument(s). +// The role of each tensor is function-dependent: there could be multiple +// inputs and outputs, and the output could be effectively scalar, +// as in a sum operation. The interpretation of the index is +// function dependent as well, but often is used to iterate over +// the outer-most row dimension of the tensor. +// This version runs purely sequentially on on this go routine. +// See VectorizeThreaded and VectorizeGPU for other versions. +func Vectorize(nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { + n := nfun(tsr...) + if n <= 0 { + return + } + for idx := range n { + fun(idx, tsr...) + } +} + +func VectorizeThreaded(threads int, nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { + n := nfun(tsr...) + if n <= 0 { + return + } + nper := int(math.Ceil(float64(n) / float64(threads))) + wait := sync.WaitGroup{} + for start := 0; start < n; start += nper { + end := start + nper + if end > n { + end = n + } + wait.Add(1) // todo: move out of loop + go func() { + for idx := start; idx < end; idx++ { + fun(idx, tsr...) + } + wait.Done() + }() + } + wait.Wait() +} + +// NFirst is an N function for Vectorize that returns the number of +// indexes of the first tensor. +func NFirst(tsr ...*Indexed) int { + if len(tsr) == 0 { + return 0 + } + return tsr[0].Len() +} + +// NMinNotLast is an N function for Vectorize that returns the min number of +// indexes of all but the last tensor. This is used when the last tensor is +// the output of the function, operating on the prior vector(s). +func NMinNotLast(tsr ...*Indexed) int { + nt := len(tsr) + if nt < 2 { + return 0 + } + n := tsr[0].Len() + for i := 1; i < nt-1; i++ { + n = min(n, tsr[0].Len()) + } + return n +} From 35208139c57be0b0e30134421dd99641c9a0c69b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 9 Sep 2024 02:48:17 -0700 Subject: [PATCH 005/311] actual stats functions need to be complete funcs, which can use vectorized calls but also need post-processing steps. these are registered in stats. --- tensor/stats/stats/funcs.go | 457 ++++++++++++++++++++++++++----- tensor/stats/stats/funcs_test.go | 16 +- tensor/stats/stats/stats.go | 24 ++ tensor/stats/stats/vecfuncs.go | 101 +++++++ 4 files changed, 520 insertions(+), 78 deletions(-) create mode 100644 tensor/stats/stats/vecfuncs.go diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 3f819187f9..344cbb215f 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -4,98 +4,413 @@ package stats -import ( - "math" +import "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor" -) +// StatsFunc is the function signature for a stats function, +// where the output has the same shape as the input but with +// an outer-most row dimension size of 1, and contains +// the stat value(s) for the "cells" in higher-dimensional tensors, +// and a single scalar value for a 1D input tensor. +// All stats functions skip over NaN's, as a not-present value. +// Stats functions cannot be computed in parallel, +// e.g., using VectorizeThreaded or GPU, due to shared writing +// to the same output values. Special implementations are required +// if that is needed. +type StatsFunc func(in, out *tensor.Indexed) -// NFunc is the nfun for stats functions, return the length of the -// first tensor, and initializing the second one to hold the output -// as the SubSpace of the first tensor. -func NFunc(tsr ...*tensor.Indexed) int { - if len(tsr) != 2 { +// CountFunc computes the count of non-NaN tensor values. +// See [StatsFunc] for general information. +func CountFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, CountVecFunc, in, out) +} + +// SumFunc computes the sum of tensor values. +// See [StatsFunc] for general information. +func SumFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, SumVecFunc, in, out) +} + +// ProdFunc computes the product of tensor values. +// See [StatsFunc] for general information. +func ProdFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, ProdVecFunc, in, out) +} + +// MinFunc computes the min of tensor values. +// See [StatsFunc] for general information. +func MinFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, MinVecFunc, in, out) +} + +// MaxFunc computes the max of tensor values. +// See [StatsFunc] for general information. +func MaxFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, MaxVecFunc, in, out) +} + +// MinAbsFunc computes the min of absolute-value-of tensor values. +// See [StatsFunc] for general information. +func MinAbsFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, MinAbsVecFunc, in, out) +} + +// MaxAbsFunc computes the max of absolute-value-of tensor values. +// See [StatsFunc] for general information. +func MaxAbsFunc(in, out *tensor.Indexed) { + tensor.Vectorize(NFunc, MaxAbsVecFunc, in, out) +} + +/* +/////////////////////////////////////////// +// Mean + +// Mean32 computes the mean of the vector (sum / N). +// Skips NaN's +func Mean32(a []float32) float32 { + s := float32(0) + n := 0 + for _, av := range a { + if math32.IsNaN(av) { + continue + } + s += av + n++ + } + if n > 0 { + s /= float32(n) + } + return s +} + +// Mean64 computes the mean of the vector (sum / N). +// Skips NaN's +func Mean64(a []float64) float64 { + s := float64(0) + n := 0 + for _, av := range a { + if math.IsNaN(av) { + continue + } + s += av + n++ + } + if n > 0 { + s /= float64(n) + } + return s +} + +/////////////////////////////////////////// +// Var + +// Var32 returns the sample variance of non-NaN elements. +func Var32(a []float32) float32 { + mean := Mean32(a) + n := 0 + s := float32(0) + for _, av := range a { + if math32.IsNaN(av) { + continue + } + dv := av - mean + s += dv * dv + n++ + } + if n > 1 { + s /= float32(n - 1) + } + return s +} + +// Var64 returns the sample variance of non-NaN elements. +func Var64(a []float64) float64 { + mean := Mean64(a) + n := 0 + s := float64(0) + for _, av := range a { + if math.IsNaN(av) { + continue + } + dv := av - mean + s += dv * dv + n++ + } + if n > 1 { + s /= float64(n - 1) + } + return s +} + +/////////////////////////////////////////// +// Std + +// Std32 returns the sample standard deviation of non-NaN elements in vector. +func Std32(a []float32) float32 { + return math32.Sqrt(Var32(a)) +} + +// Std64 returns the sample standard deviation of non-NaN elements in vector. +func Std64(a []float64) float64 { + return math.Sqrt(Var64(a)) +} + +/////////////////////////////////////////// +// Sem + +// Sem32 returns the sample standard error of the mean of non-NaN elements in vector. +func Sem32(a []float32) float32 { + cnt := Count32(a) + if cnt < 2 { + return 0 + } + return Std32(a) / math32.Sqrt(cnt) +} + +// Sem64 returns the sample standard error of the mean of non-NaN elements in vector. +func Sem64(a []float64) float64 { + cnt := Count64(a) + if cnt < 2 { + return 0 + } + return Std64(a) / math.Sqrt(cnt) +} + +/////////////////////////////////////////// +// L1Norm + +// L1Norm32 computes the sum of absolute values (L1 Norm). +// Skips NaN's +func L1Norm32(a []float32) float32 { + ss := float32(0) + for _, av := range a { + if math32.IsNaN(av) { + continue + } + ss += math32.Abs(av) + } + return ss +} + +// L1Norm64 computes the sum of absolute values (L1 Norm). +// Skips NaN's +func L1Norm64(a []float64) float64 { + ss := float64(0) + for _, av := range a { + if math.IsNaN(av) { + continue + } + ss += math.Abs(av) + } + return ss +} + +/////////////////////////////////////////// +// SumSquares + +// SumSq32 computes the sum-of-squares of vector. +// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. +func SumSq32(a []float32) float32 { + n := len(a) + if n < 2 { + if n == 1 { + return math32.Abs(a[0]) + } + return 0 + } + var ( + scale float32 = 0 + sumSquares float32 = 1 + ) + for _, v := range a { + if v == 0 || math32.IsNaN(v) { + continue + } + absxi := math32.Abs(v) + if scale < absxi { + sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + sumSquares = sumSquares + (absxi/scale)*(absxi/scale) + } + } + if math32.IsInf(scale, 1) { + return math32.Inf(1) + } + return scale * scale * sumSquares +} + +// SumSq64 computes the sum-of-squares of vector. +// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. +func SumSq64(a []float64) float64 { + n := len(a) + if n < 2 { + if n == 1 { + return math.Abs(a[0]) + } return 0 } - n := tsr[0].Len() - sh := tensor.Shape{} - sh.CopyShape(tsr[0].Tensor.Shape()) - sh.Sizes[0] = 1 - tsr[1].Tensor.SetShape(sh.Sizes, sh.Names...) - tsr[1].Indexes = []int{0} - return n -} - -// StatFunc is a helper function for stats functions, dealing with iterating over -// the Cell subspace per row and initializing the aggregation values for first index. -// It also skips over NaN missing values. -func StatFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { - nsub := out.Len() - for i := 0; i < nsub; i++ { - if idx == 0 { - out.Tensor.SetFloat1D(i, ini) - } - val := in.Tensor.FloatRowCell(in.Indexes[idx], i) - if math.IsNaN(val) { + var ( + scale float64 = 0 + ss float64 = 1 + ) + for _, v := range a { + if v == 0 || math.IsNaN(v) { continue } - out.Tensor.SetFloat1D(i, fun(val, out.Tensor.Float1D(i))) + absxi := math.Abs(v) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } + } + if math.IsInf(scale, 1) { + return math.Inf(1) } + return scale * scale * ss } -// SumFunc is a Vectorize function for computing the sum -func SumFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + val - }) +/////////////////////////////////////////// +// L2Norm + +// L2Norm32 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. +// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. +func L2Norm32(a []float32) float32 { + n := len(a) + if n < 2 { + if n == 1 { + return math32.Abs(a[0]) + } + return 0 + } + var ( + scale float32 = 0 + ss float32 = 1 + ) + for _, v := range a { + if v == 0 || math32.IsNaN(v) { + continue + } + absxi := math32.Abs(v) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } + } + if math32.IsInf(scale, 1) { + return math32.Inf(1) + } + return scale * math32.Sqrt(ss) } -// CountFunc is a Vectorize function for computing the count -func CountFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + 1 - }) +// L2Norm64 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. +// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. +func L2Norm64(a []float64) float64 { + n := len(a) + if n < 2 { + if n == 1 { + return math.Abs(a[0]) + } + return 0 + } + var ( + scale float64 = 0 + ss float64 = 1 + ) + for _, v := range a { + if v == 0 || math.IsNaN(v) { + continue + } + absxi := math.Abs(v) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } + } + if math.IsInf(scale, 1) { + return math.Inf(1) + } + return scale * math.Sqrt(ss) } -// ProdFunc is a Vectorize function for computing the product -func ProdFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { - return agg * val - }) +/////////////////////////////////////////// +// VarPop + +// VarPop32 returns the population variance of non-NaN elements. +func VarPop32(a []float32) float32 { + mean := Mean32(a) + n := 0 + s := float32(0) + for _, av := range a { + if math32.IsNaN(av) { + continue + } + dv := av - mean + s += dv * dv + n++ + } + if n > 0 { + s /= float32(n) + } + return s } -// MinFunc is a Vectorize function for computing the min -func MinFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { - return math.Min(agg, val) - }) +// VarPop64 returns the population variance of non-NaN elements. +func VarPop64(a []float64) float64 { + mean := Mean64(a) + n := 0 + s := float64(0) + for _, av := range a { + if math.IsNaN(av) { + continue + } + dv := av - mean + s += dv * dv + n++ + } + if n > 0 { + s /= float64(n) + } + return s } -// MaxFunc is a Vectorize function for computing the min -func MaxFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { - return math.Max(agg, val) - }) +/////////////////////////////////////////// +// StdPop + +// StdPop32 returns the population standard deviation of non-NaN elements in vector. +func StdPop32(a []float32) float32 { + return math32.Sqrt(VarPop32(a)) } -// MinAbsFunc is a Vectorize function for computing the min of abs values -func MinAbsFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { - return math.Min(agg, math.Abs(val)) - }) +// StdPop64 returns the population standard deviation of non-NaN elements in vector. +func StdPop64(a []float64) float64 { + return math.Sqrt(VarPop64(a)) } -// MaxAbsFunc is a Vectorize function for computing the max of abs values -func MaxAbsFunc(idx int, tsr ...*tensor.Indexed) { - StatFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { - return math.Max(agg, math.Abs(val)) - }) +/////////////////////////////////////////// +// SemPop + +// SemPop32 returns the population standard error of the mean of non-NaN elements in vector. +func SemPop32(a []float32) float32 { + cnt := Count32(a) + if cnt < 2 { + return 0 + } + return StdPop32(a) / math32.Sqrt(cnt) } -// L1NormFunc is an StatFunc that computes the L1 norm: sum of absolute values -// use 0 as initial value. -// func L1NormFunc(idx int, val float64, agg float64) float64 { -// return agg + math.Abs(val) -// } +// SemPop64 returns the population standard error of the mean of non-NaN elements in vector. +func SemPop64(a []float64) float64 { + cnt := Count64(a) + if cnt < 2 { + return 0 + } + return StdPop64(a) / math.Sqrt(cnt) +} -// Note: SumSq is not numerically stable for large N in simple func form. +*/ diff --git a/tensor/stats/stats/funcs_test.go b/tensor/stats/stats/funcs_test.go index 79beb996d0..ddd1fa1a48 100644 --- a/tensor/stats/stats/funcs_test.go +++ b/tensor/stats/stats/funcs_test.go @@ -21,23 +21,25 @@ func TestFuncs(t *testing.T) { results := []float64{11, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 5.5, 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11)} - // assert.Equal(t, results[Count], Count64(vals)) - tensor.Vectorize(NFunc, SumFunc, ix, oix) + CountFunc(ix, oix) + assert.Equal(t, results[Count], out.Values[0]) + + SumFunc(ix, oix) assert.Equal(t, results[Sum], out.Values[0]) - tensor.Vectorize(NFunc, ProdFunc, ix, oix) + ProdFunc(ix, oix) assert.Equal(t, results[Prod], out.Values[0]) - tensor.Vectorize(NFunc, MinFunc, ix, oix) + MinFunc(ix, oix) assert.Equal(t, results[Min], out.Values[0]) - tensor.Vectorize(NFunc, MaxFunc, ix, oix) + MaxFunc(ix, oix) assert.Equal(t, results[Max], out.Values[0]) - tensor.Vectorize(NFunc, MinAbsFunc, ix, oix) + MinAbsFunc(ix, oix) assert.Equal(t, results[MinAbs], out.Values[0]) - tensor.Vectorize(NFunc, MaxAbsFunc, ix, oix) + MaxAbsFunc(ix, oix) assert.Equal(t, results[MaxAbs], out.Values[0]) // assert.Equal(t, results[Mean], Mean64(vals)) diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index a77a2a15a8..36bdc3ac84 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -4,8 +4,32 @@ package stats +import "cogentcore.org/core/tensor" + //go:generate core generate +// StatsFuncs is a registry of named stats functions, +// which can then be called by standard enum or +// string name for custom functions. +var StatsFuncs map[string]StatsFunc + +func init() { + StatsFuncs = make(map[string]StatsFunc) + StatsFuncs[Count.String()] = CountFunc + StatsFuncs[Sum.String()] = SumFunc + StatsFuncs[Prod.String()] = ProdFunc + StatsFuncs[Min.String()] = MinFunc + StatsFuncs[Max.String()] = MaxFunc + StatsFuncs[MinAbs.String()] = MinAbsFunc + StatsFuncs[MaxAbs.String()] = MaxAbsFunc +} + +// Standard calls a standard stats function on given tensors. +// Output results are in the out tensor. +func Standard(stat Stats, in, out *tensor.Indexed) { + StatsFuncs[stat.String()](in, out) +} + // Stats is a list of different standard aggregation functions, which can be used // to choose an aggregation function type Stats int32 //enums:enum diff --git a/tensor/stats/stats/vecfuncs.go b/tensor/stats/stats/vecfuncs.go new file mode 100644 index 0000000000..fe0709eaf6 --- /dev/null +++ b/tensor/stats/stats/vecfuncs.go @@ -0,0 +1,101 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "math" + + "cogentcore.org/core/tensor" +) + +// NFunc is the nfun for stats functions, return the length of the +// first tensor, and initializing the second one to hold the output +// as the SubSpace of the first tensor. +func NFunc(tsr ...*tensor.Indexed) int { + if len(tsr) != 2 { + return 0 + } + n := tsr[0].Len() + sh := tensor.Shape{} + sh.CopyShape(tsr[0].Tensor.Shape()) + sh.Sizes[0] = 1 + tsr[1].Tensor.SetShape(sh.Sizes, sh.Names...) + tsr[1].Indexes = []int{0} + return n +} + +// StatVecFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// It also skips over NaN missing values. +func StatVecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { + nsub := out.Len() + for i := 0; i < nsub; i++ { + if idx == 0 { + out.Tensor.SetFloat1D(i, ini) + } + val := in.Tensor.FloatRowCell(in.Indexes[idx], i) + if math.IsNaN(val) { + continue + } + out.Tensor.SetFloat1D(i, fun(val, out.Tensor.Float1D(i))) + } +} + +// SumVecFunc is a Vectorize function for computing the sum +func SumVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + val + }) +} + +// CountVecFunc is a Vectorize function for computing the count +func CountVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + 1 + }) +} + +// ProdVecFunc is a Vectorize function for computing the product +func ProdVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { + return agg * val + }) +} + +// MinVecFunc is a Vectorize function for computing the min +func MinVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, val) + }) +} + +// MaxVecFunc is a Vectorize function for computing the min +func MaxVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, val) + }) +} + +// MinAbsVecFunc is a Vectorize function for computing the min of abs values +func MinAbsVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, math.Abs(val)) + }) +} + +// MaxAbsVecFunc is a Vectorize function for computing the max of abs values +func MaxAbsVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, math.Abs(val)) + }) +} + +// L1NormFunc is an StatFunc that computes the L1 norm: sum of absolute values +// use 0 as initial value. +// func L1NormFunc(idx int, val float64, agg float64) float64 { +// return agg + math.Abs(val) +// } + +// Note: SumSq is not numerically stable for large N in simple func form. From 97c9b0653b654b789601e081656893f8232ae7d4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 9 Sep 2024 14:55:20 -0700 Subject: [PATCH 006/311] all vectorize-based functions now in place. framework seems very powerful and elegant so far. elminated so much redundancy here. --- tensor/base.go | 6 +- tensor/indexed.go | 63 +++- tensor/stats/stats/README.md | 25 +- tensor/stats/stats/enumgen.go | 10 +- tensor/stats/stats/funcs.go | 451 ++++++++++----------------- tensor/stats/stats/funcs_test.go | 91 +++++- tensor/stats/stats/indexview_test.go | 79 ----- tensor/stats/stats/stats.go | 58 ++-- tensor/stats/stats/tensor.go | 213 ------------- tensor/stats/stats/tensor_test.go | 37 --- tensor/stats/stats/vecfuncs.go | 185 +++++++++-- 11 files changed, 516 insertions(+), 702 deletions(-) delete mode 100644 tensor/stats/stats/indexview_test.go delete mode 100644 tensor/stats/stats/tensor.go delete mode 100644 tensor/stats/stats/tensor_test.go diff --git a/tensor/base.go b/tensor/base.go index 28434dcf31..c3573e75af 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -15,14 +15,14 @@ import ( "cogentcore.org/core/base/slicesx" ) -// Base is an n-dim array of float64s. +// Base is the base Tensor implementation for given type. type Base[T any] struct { shape Shape Values []T Meta metadata.Data } -// Shape returns a pointer to the shape that fully parametrizes the tensor shape +// Shape returns a pointer to the shape that fully parametrizes the tensor shape. func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } // Metadata returns the metadata for this tensor, which can be used @@ -35,7 +35,7 @@ func (tsr *Base[T]) Len() int { return tsr.shape.Len() } // NumDims returns the total number of dimensions. func (tsr *Base[T]) NumDims() int { return tsr.shape.NumDims() } -// DimSize returns size of given dimension +// DimSize returns size of given dimension. func (tsr *Base[T]) DimSize(dim int) int { return tsr.shape.DimSize(dim) } // RowCellSize returns the size of the outer-most Row shape dimension, diff --git a/tensor/indexed.go b/tensor/indexed.go index 22d132cd10..c3a5f43dca 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -7,6 +7,7 @@ package tensor import ( "cmp" "errors" + "math" "math/rand" "slices" "sort" @@ -15,19 +16,21 @@ import ( // Indexed is an indexed wrapper around a tensor.Tensor that provides a // specific view onto the Tensor defined by the set of indexes, which -// apply to the outer-most ("row") dimension. +// apply to the outer-most row dimension (with default row-major indexing). // This provides an efficient way of sorting and filtering a tensor by only // updating the indexes while doing nothing to the Tensor itself. // To produce a tensor that has data actually organized according to the // indexed order, call the NewTensor method. -// Indexed views on a tensor can also be organized together as Splits -// of the tensor rows, e.g., by grouping values along a given column. +// Use the [Set]FloatRowCell methods wherever possible, for the most efficient +// and natural indirection through the indexes. The 1D methods on underlying +// tensor data do not indirect through the indexes and must be called directly +// on the [Tensor]. type Indexed struct { //types:add - // Tensor that we are an indexed view onto + // Tensor that we are an indexed view onto. Tensor Tensor - // current indexes into Tensor + // Indexes are the indexes into Tensor rows. Indexes []int } @@ -84,6 +87,24 @@ func (ix *Indexed) Sequential() { //types:add } } +// ExcludeMissing1D deletes indexes for a 1D tensor (only) where +// the values are missing, as indicated by NaN. +func (ix *Indexed) ExcludeMissing1D() { //types:add + if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + ix.Indexes = nil + return + } + if ix.Tensor.NumDims() > 1 { + return + } + ni := ix.Len() + for i := ni - 1; i >= 0; i-- { + if math.IsNaN(ix.Tensor.Float1D(ix.Indexes[i])) { + ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + } + } +} + // Permuted sets indexes to a permuted order -- if indexes already exist // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated @@ -273,3 +294,35 @@ func (ix *Indexed) DeleteRows(at, n int) { func (ix *Indexed) Swap(i, j int) { ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] } + +// Float returns the value of given index as a float64. +// The first index value is indirected through the indexes. +func (ix *Indexed) Float(i []int) float64 { + ic := slices.Clone(i) + ic[0] = ix.Indexes[ic[0]] + return ix.Tensor.Float(ic) +} + +// SetFloat sets the value of given index as a float64 +// The first index value is indirected through the [Indexes]. +func (ix *Indexed) SetFloat(i []int, val float64) { + ic := slices.Clone(i) + ic[0] = ix.Indexes[ic[0]] + ix.Tensor.SetFloat(ic, val) +} + +// FloatRowCell returns the value at given row and cell, +// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Indexes]. +// This is the preferred interface for all Indexed operations. +func (ix *Indexed) FloatRowCell(row, cell int) float64 { + return ix.Tensor.FloatRowCell(ix.Indexes[row], cell) +} + +// SetFloatRowCell sets the value at given row and cell, +// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Indexes]. +// This is the preferred interface for all Indexed operations. +func (ix *Indexed) SetFloatRowCell(row, cell int, val float64) { + ix.Tensor.SetFloatRowCell(ix.Indexes[row], cell, val) +} diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index fa7d0e2ebf..4e07484bf5 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -1,12 +1,13 @@ # stats -The `stats` package provides standard statistic computations operating over floating-point data (both 32 and 64 bit) in the following formats. Each statistic returns a single scalar value summarizing the data in a different way. Some formats also support multi-dimensional tensor data, returning a summary stat for each tensor value, using the outer-most ("row-wise") dimension to summarize over. +The `stats` package provides standard statistic computations operating on the `tensor.Indexed` standard data representation, using this standard function: +```Go +type StatsFunc func(in, out *tensor.Indexed) +``` -* `[]float32` and `[]float64` slices, as e.g., `Mean32` and `Mean64`, skipping any `NaN` values as missing data. +For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outer-most row dimension set to 1. -* `tensor.Float32`, `tensor.Float64` using the underlying `Values` slice, and other generic `Tensor` using the `Floats` interface (less efficient). - -* `table.Indexed` indexed views of `table.Table` data, with `*Column` functions (e.g., `MeanColumn`) using names to specify columns, and `*Index` versions operating on column indexes. Also available for this type are `CountIf*`, `PctIf*`, `PropIf*` functions that return count, percentage, or propoprtion of values according to given function. +There is a `StatsFuncs` map of named stats funcs, which is initialized with the standard Stats per below, and any additional user-defined functions can be added to. ## Stats @@ -14,16 +15,17 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Count`: count of number of elements * `Sum`: sum of elements +* `SumAbs`: sum of absolute-value-of elements (same as L1Norm) +* `L1Norm`: L1 Norm: sum of absolute values (same as SumAbs) * `Prod`: product of elements * `Min`: minimum value -* `Max`: max maximum value -* `MinAbs`: minimum absolute value -* `MaxAbs`: maximum absolute value -* `Mean`: mean mean value +* `Max`: maximum value +* `MinAbs`: minimum of absolute values +* `MaxAbs`: maximum of absolute values +* `Mean`: mean value * `Var`: sample variance (squared diffs from mean, divided by n-1) * `Std`: sample standard deviation (sqrt of Var) * `Sem`: sample standard error of the mean (Std divided by sqrt(n)) -* `L1Norm`: L1 Norm: sum of absolute values * `SumSq`: sum of squared element values * `L2Norm`: L2 Norm: square-root of sum-of-squares * `VarPop`: population variance (squared diffs from mean, divided by n) @@ -33,4 +35,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Q1`: Q1 first quartile = 25%ile value = .25 quantile value (only for Indexed) * `Q3`: Q3 third quartile = 75%ile value = .75 quantile value (only for Indexed) +## Vectorize functions + +See [vecfuncs.go](vecfuncs.go) for corresponding `tensor.Vectorize` functions that are used in performing the computations. These cannot be parallelized directly due to shared writing to output accumulators, and other ordering constraints. If needed, special atomic-locking or other such techniques would be required. diff --git a/tensor/stats/stats/enumgen.go b/tensor/stats/stats/enumgen.go index 7a760c668e..6e2cc87399 100644 --- a/tensor/stats/stats/enumgen.go +++ b/tensor/stats/stats/enumgen.go @@ -6,16 +6,16 @@ import ( "cogentcore.org/core/enums" ) -var _StatsValues = []Stats{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19} +var _StatsValues = []Stats{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20} // StatsN is the highest valid value for type Stats, plus one. -const StatsN Stats = 20 +const StatsN Stats = 21 -var _StatsValueMap = map[string]Stats{`Count`: 0, `Sum`: 1, `Prod`: 2, `Min`: 3, `Max`: 4, `MinAbs`: 5, `MaxAbs`: 6, `Mean`: 7, `Var`: 8, `Std`: 9, `Sem`: 10, `L1Norm`: 11, `SumSq`: 12, `L2Norm`: 13, `VarPop`: 14, `StdPop`: 15, `SemPop`: 16, `Median`: 17, `Q1`: 18, `Q3`: 19} +var _StatsValueMap = map[string]Stats{`Count`: 0, `Sum`: 1, `SumAbs`: 2, `L1Norm`: 3, `Prod`: 4, `Min`: 5, `Max`: 6, `MinAbs`: 7, `MaxAbs`: 8, `Mean`: 9, `Var`: 10, `Std`: 11, `Sem`: 12, `SumSq`: 13, `L2Norm`: 14, `VarPop`: 15, `StdPop`: 16, `SemPop`: 17, `Median`: 18, `Q1`: 19, `Q3`: 20} -var _StatsDescMap = map[Stats]string{0: `count of number of elements`, 1: `sum of elements`, 2: `product of elements`, 3: `minimum value`, 4: `max maximum value`, 5: `minimum absolute value`, 6: `maximum absolute value`, 7: `mean mean value`, 8: `sample variance (squared diffs from mean, divided by n-1)`, 9: `sample standard deviation (sqrt of Var)`, 10: `sample standard error of the mean (Std divided by sqrt(n))`, 11: `L1 Norm: sum of absolute values`, 12: `sum of squared values`, 13: `L2 Norm: square-root of sum-of-squares`, 14: `population variance (squared diffs from mean, divided by n)`, 15: `population standard deviation (sqrt of VarPop)`, 16: `population standard error of the mean (StdPop divided by sqrt(n))`, 17: `middle value in sorted ordering`, 18: `Q1 first quartile = 25%ile value = .25 quantile value`, 19: `Q3 third quartile = 75%ile value = .75 quantile value`} +var _StatsDescMap = map[Stats]string{0: `count of number of elements.`, 1: `sum of elements.`, 2: `sum of absolute-value-of elements (= L1Norm).`, 3: `L1 Norm: sum of absolute values (= SumAbs).`, 4: `product of elements.`, 5: `minimum value.`, 6: `maximum value.`, 7: `minimum of absolute values.`, 8: `maximum of absolute values.`, 9: `mean value = sum / count.`, 10: `sample variance (squared deviations from mean, divided by n-1).`, 11: `sample standard deviation (sqrt of Var).`, 12: `sample standard error of the mean (Std divided by sqrt(n)).`, 13: `sum of squared values.`, 14: `L2 Norm: square-root of sum-of-squares.`, 15: `population variance (squared diffs from mean, divided by n).`, 16: `population standard deviation (sqrt of VarPop).`, 17: `population standard error of the mean (StdPop divided by sqrt(n)).`, 18: `middle value in sorted ordering.`, 19: `Q1 first quartile = 25%ile value = .25 quantile value.`, 20: `Q3 third quartile = 75%ile value = .75 quantile value.`} -var _StatsMap = map[Stats]string{0: `Count`, 1: `Sum`, 2: `Prod`, 3: `Min`, 4: `Max`, 5: `MinAbs`, 6: `MaxAbs`, 7: `Mean`, 8: `Var`, 9: `Std`, 10: `Sem`, 11: `L1Norm`, 12: `SumSq`, 13: `L2Norm`, 14: `VarPop`, 15: `StdPop`, 16: `SemPop`, 17: `Median`, 18: `Q1`, 19: `Q3`} +var _StatsMap = map[Stats]string{0: `Count`, 1: `Sum`, 2: `SumAbs`, 3: `L1Norm`, 4: `Prod`, 5: `Min`, 6: `Max`, 7: `MinAbs`, 8: `MaxAbs`, 9: `Mean`, 10: `Var`, 11: `Std`, 12: `Sem`, 13: `SumSq`, 14: `L2Norm`, 15: `VarPop`, 16: `StdPop`, 17: `SemPop`, 18: `Median`, 19: `Q1`, 20: `Q3`} // String returns the string representation of this Stats value. func (i Stats) String() string { return enums.String(i, _StatsMap) } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 344cbb215f..0d478afe29 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -4,14 +4,18 @@ package stats -import "cogentcore.org/core/tensor" +import ( + "math" + + "cogentcore.org/core/tensor" +) // StatsFunc is the function signature for a stats function, // where the output has the same shape as the input but with -// an outer-most row dimension size of 1, and contains +// the outer-most row dimension size of 1, and contains // the stat value(s) for the "cells" in higher-dimensional tensors, // and a single scalar value for a 1D input tensor. -// All stats functions skip over NaN's, as a not-present value. +// All stats functions skip over NaN's, as a missing value. // Stats functions cannot be computed in parallel, // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required @@ -21,291 +25,241 @@ type StatsFunc func(in, out *tensor.Indexed) // CountFunc computes the count of non-NaN tensor values. // See [StatsFunc] for general information. func CountFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, CountVecFunc, in, out) + VectorizeOut64(CountVecFunc, in, out) } // SumFunc computes the sum of tensor values. // See [StatsFunc] for general information. func SumFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, SumVecFunc, in, out) + VectorizeOut64(SumVecFunc, in, out) +} + +// SumAbsFunc computes the sum of absolute-value-of tensor values. +// This is also known as the L1 norm. +// See [StatsFunc] for general information. +func SumAbsFunc(in, out *tensor.Indexed) { + VectorizeOut64(SumAbsVecFunc, in, out) } // ProdFunc computes the product of tensor values. // See [StatsFunc] for general information. func ProdFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, ProdVecFunc, in, out) + VectorizeOut64(ProdVecFunc, in, out) } // MinFunc computes the min of tensor values. // See [StatsFunc] for general information. func MinFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, MinVecFunc, in, out) + VectorizeOut64(MinVecFunc, in, out) } // MaxFunc computes the max of tensor values. // See [StatsFunc] for general information. func MaxFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, MaxVecFunc, in, out) + VectorizeOut64(MaxVecFunc, in, out) } // MinAbsFunc computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. func MinAbsFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, MinAbsVecFunc, in, out) + VectorizeOut64(MinAbsVecFunc, in, out) } // MaxAbsFunc computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. func MaxAbsFunc(in, out *tensor.Indexed) { - tensor.Vectorize(NFunc, MaxAbsVecFunc, in, out) + VectorizeOut64(MaxAbsVecFunc, in, out) } -/* -/////////////////////////////////////////// -// Mean - -// Mean32 computes the mean of the vector (sum / N). -// Skips NaN's -func Mean32(a []float32) float32 { - s := float32(0) - n := 0 - for _, av := range a { - if math32.IsNaN(av) { - continue - } - s += av - n++ - } - if n > 0 { - s /= float32(n) - } - return s +// MeanFunc computes the mean of tensor values. +// See [StatsFunc] for general information. +func MeanFunc(in, out *tensor.Indexed) { + MeanFuncOut64(in, out) } -// Mean64 computes the mean of the vector (sum / N). -// Skips NaN's -func Mean64(a []float64) float64 { - s := float64(0) - n := 0 - for _, av := range a { - if math.IsNaN(av) { +// MeanFuncOut64 computes the mean of tensor values, +// and returns the Float64 count and mean output values for +// use in subsequent computations. +func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { + sum64 := VectorizeOut64(SumVecFunc, in, out) + count := tensor.NewIndexed(out.Tensor.Clone()) + count64 = VectorizeOut64(CountVecFunc, in, count) + nsub := out.Tensor.Len() + for i := range nsub { + c := count64.Tensor.Float1D(i) + if c == 0 { continue } - s += av - n++ + sum64.Tensor.SetFloat1D(i, sum64.Tensor.Float1D(i)/c) + out.Tensor.SetFloat1D(i, sum64.Tensor.Float1D(i)) } - if n > 0 { - s /= float64(n) - } - return s + return sum64, count64 } -/////////////////////////////////////////// -// Var - -// Var32 returns the sample variance of non-NaN elements. -func Var32(a []float32) float32 { - mean := Mean32(a) - n := 0 - s := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 1 { - s /= float32(n - 1) - } - return s +// VarFunc computes the sample variance of tensor values. +// Squared deviations from mean, divided by n-1. See also [VarPopFunc]. +// See [StatsFunc] for general information. +func VarFunc(in, out *tensor.Indexed) { + VarFuncOut64(in, out) } -// Var64 returns the sample variance of non-NaN elements. -func Var64(a []float64) float64 { - mean := Mean64(a) - n := 0 - s := float64(0) - for _, av := range a { - if math.IsNaN(av) { +// VarFuncOut64 computes the sample variance of tensor values. +// and returns the Float64 output values for +// use in subsequent computations. +func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Indexed) { + mean64, count64 = MeanFuncOut64(in, out) + var64 = VectorizeOut64(VarVecFunc, in, mean64, out) + nsub := out.Tensor.Len() + for i := range nsub { + c := count64.Tensor.Float1D(i) + if c < 2 { continue } - dv := av - mean - s += dv * dv - n++ - } - if n > 1 { - s /= float64(n - 1) + var64.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)/(c-1)) + out.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)) } - return s -} - -/////////////////////////////////////////// -// Std - -// Std32 returns the sample standard deviation of non-NaN elements in vector. -func Std32(a []float32) float32 { - return math32.Sqrt(Var32(a)) -} - -// Std64 returns the sample standard deviation of non-NaN elements in vector. -func Std64(a []float64) float64 { - return math.Sqrt(Var64(a)) + return } -/////////////////////////////////////////// -// Sem - -// Sem32 returns the sample standard error of the mean of non-NaN elements in vector. -func Sem32(a []float32) float32 { - cnt := Count32(a) - if cnt < 2 { - return 0 +// StdFunc computes the sample standard deviation of tensor values. +// Sqrt of variance from [VarFunc]. See also [StdPopFunc]. +// See [StatsFunc] for general information. +func StdFunc(in, out *tensor.Indexed) { + var64, _, _ := VarFuncOut64(in, out) + nsub := out.Tensor.Len() + for i := range nsub { + out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) } - return Std32(a) / math32.Sqrt(cnt) } -// Sem64 returns the sample standard error of the mean of non-NaN elements in vector. -func Sem64(a []float64) float64 { - cnt := Count64(a) - if cnt < 2 { - return 0 +// SemFunc computes the sample standard error of the mean of tensor values. +// Standard deviation [StdFunc] / sqrt(n). See also [SemPopFunc]. +// See [StatsFunc] for general information. +func SemFunc(in, out *tensor.Indexed) { + var64, _, count64 := VarFuncOut64(in, out) + nsub := out.Tensor.Len() + for i := range nsub { + c := count64.Tensor.Float1D(i) + if c < 2 { + out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) + } else { + out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c)) + } } - return Std64(a) / math.Sqrt(cnt) } -/////////////////////////////////////////// -// L1Norm +// VarPopFunc computes the population variance of tensor values. +// Squared deviations from mean, divided by n. See also [VarFunc]. +// See [StatsFunc] for general information. +func VarPopFunc(in, out *tensor.Indexed) { + VarPopFuncOut64(in, out) +} -// L1Norm32 computes the sum of absolute values (L1 Norm). -// Skips NaN's -func L1Norm32(a []float32) float32 { - ss := float32(0) - for _, av := range a { - if math32.IsNaN(av) { +// VarPopFuncOut64 computes the population variance of tensor values. +// and returns the Float64 output values for +// use in subsequent computations. +func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Indexed) { + mean64, count64 = MeanFuncOut64(in, out) + var64 = VectorizeOut64(VarVecFunc, in, mean64, out) + nsub := out.Tensor.Len() + for i := range nsub { + c := count64.Tensor.Float1D(i) + if c == 0 { continue } - ss += math32.Abs(av) + var64.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)/c) + out.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)) } - return ss + return } -// L1Norm64 computes the sum of absolute values (L1 Norm). -// Skips NaN's -func L1Norm64(a []float64) float64 { - ss := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - ss += math.Abs(av) +// StdPopFunc computes the population standard deviation of tensor values. +// Sqrt of variance from [VarPopFunc]. See also [StdFunc]. +// See [StatsFunc] for general information. +func StdPopFunc(in, out *tensor.Indexed) { + var64, _, _ := VarPopFuncOut64(in, out) + nsub := out.Tensor.Len() + for i := range nsub { + out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) } - return ss } -/////////////////////////////////////////// -// SumSquares - -// SumSq32 computes the sum-of-squares of vector. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func SumSq32(a []float32) float32 { - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0]) - } - return 0 - } - var ( - scale float32 = 0 - sumSquares float32 = 1 - ) - for _, v := range a { - if v == 0 || math32.IsNaN(v) { - continue - } - absxi := math32.Abs(v) - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi +// SemPopFunc computes the population standard error of the mean of tensor values. +// Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. +// See [StatsFunc] for general information. +func SemPopFunc(in, out *tensor.Indexed) { + var64, _, count64 := VarPopFuncOut64(in, out) + nsub := out.Tensor.Len() + for i := range nsub { + c := count64.Tensor.Float1D(i) + if c < 2 { + out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) + out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c)) } } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * scale * sumSquares } -// SumSq64 computes the sum-of-squares of vector. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func SumSq64(a []float64) float64 { - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0]) - } - return 0 - } - var ( - scale float64 = 0 - ss float64 = 1 - ) - for _, v := range a { - if v == 0 || math.IsNaN(v) { - continue - } - absxi := math.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi +// SumSqFunc computes the sum of squares of tensor values, +// See [StatsFunc] for general information. +func SumSqFunc(in, out *tensor.Indexed) { + SumSqFuncOut64(in, out) +} + +// SumSqFuncOut64 computes the sum of squares of tensor values, +// and returns the Float64 output values for +// use in subsequent computations. +func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { + scale64, ss64 := Vectorize2Out64(SumSqVecFunc, in, out) + nsub := out.Tensor.Len() + for i := range nsub { + scale := scale64.Tensor.Float1D(i) + ss := ss64.Tensor.Float1D(i) + v := 0.0 + if math.IsInf(scale, 1) { + v = math.Inf(1) } else { - ss = ss + (absxi/scale)*(absxi/scale) + v = scale * scale * ss } + scale64.Tensor.SetFloat1D(i, v) + out.Tensor.SetFloat1D(i, v) } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * scale * ss + return scale64 } -/////////////////////////////////////////// -// L2Norm +// L2NormFunc computes the square root of the sum of squares of tensor values, +// known as the L2 norm. +// See [StatsFunc] for general information. +func L2NormFunc(in, out *tensor.Indexed) { + L2NormFuncOut64(in, out) +} -// L2Norm32 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func L2Norm32(a []float32) float32 { - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0]) - } - return 0 - } - var ( - scale float32 = 0 - ss float32 = 1 - ) - for _, v := range a { - if v == 0 || math32.IsNaN(v) { - continue - } - absxi := math32.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi +// L2NormFuncOut64 computes the square root of the sum of squares of tensor values, +// known as the L2 norm, and returns the Float64 output values for +// use in subsequent computations. +func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { + scale64, ss64 := Vectorize2Out64(SumSqVecFunc, in, out) + nsub := out.Tensor.Len() + for i := range nsub { + scale := scale64.Tensor.Float1D(i) + ss := ss64.Tensor.Float1D(i) + v := 0.0 + if math.IsInf(scale, 1) { + v = math.Inf(1) } else { - ss = ss + (absxi/scale)*(absxi/scale) + v = scale * math.Sqrt(ss) } + scale64.Tensor.SetFloat1D(i, v) + out.Tensor.SetFloat1D(i, v) } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * math32.Sqrt(ss) + return scale64 } +/* + +/////////////////////////////////////////// +// L2Norm + // L2Norm64 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. // Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. func L2Norm64(a []float64) float64 { @@ -338,79 +292,4 @@ func L2Norm64(a []float64) float64 { return scale * math.Sqrt(ss) } -/////////////////////////////////////////// -// VarPop - -// VarPop32 returns the population variance of non-NaN elements. -func VarPop32(a []float32) float32 { - mean := Mean32(a) - n := 0 - s := float32(0) - for _, av := range a { - if math32.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 0 { - s /= float32(n) - } - return s -} - -// VarPop64 returns the population variance of non-NaN elements. -func VarPop64(a []float64) float64 { - mean := Mean64(a) - n := 0 - s := float64(0) - for _, av := range a { - if math.IsNaN(av) { - continue - } - dv := av - mean - s += dv * dv - n++ - } - if n > 0 { - s /= float64(n) - } - return s -} - -/////////////////////////////////////////// -// StdPop - -// StdPop32 returns the population standard deviation of non-NaN elements in vector. -func StdPop32(a []float32) float32 { - return math32.Sqrt(VarPop32(a)) -} - -// StdPop64 returns the population standard deviation of non-NaN elements in vector. -func StdPop64(a []float64) float64 { - return math.Sqrt(VarPop64(a)) -} - -/////////////////////////////////////////// -// SemPop - -// SemPop32 returns the population standard error of the mean of non-NaN elements in vector. -func SemPop32(a []float32) float32 { - cnt := Count32(a) - if cnt < 2 { - return 0 - } - return StdPop32(a) / math32.Sqrt(cnt) -} - -// SemPop64 returns the population standard error of the mean of non-NaN elements in vector. -func SemPop64(a []float64) float64 { - cnt := Count64(a) - if cnt < 2 { - return 0 - } - return StdPop64(a) / math.Sqrt(cnt) -} - */ diff --git a/tensor/stats/stats/funcs_test.go b/tensor/stats/stats/funcs_test.go index ddd1fa1a48..f3514df7ce 100644 --- a/tensor/stats/stats/funcs_test.go +++ b/tensor/stats/stats/funcs_test.go @@ -8,18 +8,21 @@ import ( "math" "testing" + "cogentcore.org/core/base/tolassert" "cogentcore.org/core/tensor" "github.com/stretchr/testify/assert" ) -func TestFuncs(t *testing.T) { +// todo: add int, tensor cell tests! + +func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} tsr := tensor.NewNumberFromSlice(vals) ix := tensor.NewIndexed(tsr) out := tensor.NewFloat64([]int{1}) oix := tensor.NewIndexed(out) - results := []float64{11, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 5.5, 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11)} + results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11)} CountFunc(ix, oix) assert.Equal(t, results[Count], out.Values[0]) @@ -27,6 +30,9 @@ func TestFuncs(t *testing.T) { SumFunc(ix, oix) assert.Equal(t, results[Sum], out.Values[0]) + SumAbsFunc(ix, oix) + assert.Equal(t, results[SumAbs], out.Values[0]) + ProdFunc(ix, oix) assert.Equal(t, results[Prod], out.Values[0]) @@ -42,18 +48,71 @@ func TestFuncs(t *testing.T) { MaxAbsFunc(ix, oix) assert.Equal(t, results[MaxAbs], out.Values[0]) - // assert.Equal(t, results[Mean], Mean64(vals)) - // tolassert.EqualTol(t, results[Var], Var64(vals), 1.0e-8) - // tolassert.EqualTol(t, results[Std], Std64(vals), 1.0e-8) - // tolassert.EqualTol(t, results[Sem], Sem64(vals), 1.0e-8) - // assert.Equal(t, results[L1Norm], L1Norm64(vals)) - // tolassert.EqualTol(t, results[SumSq], SumSq64(vals), 1.0e-8) - // tolassert.EqualTol(t, results[L2Norm], L2Norm64(vals), 1.0e-8) - // assert.Equal(t, results[VarPop], VarPop64(vals)) - // assert.Equal(t, results[StdPop], StdPop64(vals)) - // assert.Equal(t, results[SemPop], SemPop64(vals)) - // - // for stat := Count; stat <= SemPop; stat++ { - // tolassert.EqualTol(t, results[stat], Stat64(vals, stat), 1.0e-8) - // } + MeanFunc(ix, oix) + assert.Equal(t, results[Mean], out.Values[0]) + + VarFunc(ix, oix) + assert.InDelta(t, results[Var], out.Values[0], 1.0e-8) + + StdFunc(ix, oix) + assert.InDelta(t, results[Std], out.Values[0], 1.0e-8) + + SemFunc(ix, oix) + assert.InDelta(t, results[Sem], out.Values[0], 1.0e-8) + + VarPopFunc(ix, oix) + assert.InDelta(t, results[VarPop], out.Values[0], 1.0e-8) + + StdPopFunc(ix, oix) + assert.InDelta(t, results[StdPop], out.Values[0], 1.0e-8) + + SemPopFunc(ix, oix) + assert.InDelta(t, results[SemPop], out.Values[0], 1.0e-8) + + SumSqFunc(ix, oix) + assert.InDelta(t, results[SumSq], out.Values[0], 1.0e-8) + + L2NormFunc(ix, oix) + assert.InDelta(t, results[L2Norm], out.Values[0], 1.0e-8) + + for stat := Count; stat <= SemPop; stat++ { + Standard(stat, ix, oix) + tolassert.EqualTol(t, results[stat], out.Values[0], 1.0e-8) + } +} + +/* +func TestIndexed(t *testing.T) { + desc := DescAll(ix) + assert.Equal(t, len(DescStats), desc.Rows) + assert.Equal(t, 2, desc.NumColumns()) + + for ri, stat := range DescStats { + dv := desc.Float("data", ri) + // fmt.Println(ri, ag.String(), dv, results[ag]) + assert.Equal(t, results[stat], dv) + } + + desc, err := DescColumn(ix, "data") + if err != nil { + t.Error(err) + } + assert.Equal(t, len(DescStats), desc.Rows) + assert.Equal(t, 2, desc.NumColumns()) + for ri, stat := range DescStats { + dv := desc.Float("data", ri) + // fmt.Println(ri, ag.String(), dv, results[ag]) + assert.Equal(t, results[stat], dv) + } + + pcts := PctIfColumn(ix, "data", func(idx int, val float64) bool { + return val > 2 + }) + assert.Equal(t, []float64{60}, pcts) + + props := PropIfColumn(ix, "data", func(idx int, val float64) bool { + return val > 2 + }) + assert.Equal(t, []float64{0.6}, props) } +*/ diff --git a/tensor/stats/stats/indexview_test.go b/tensor/stats/stats/indexview_test.go deleted file mode 100644 index 26433f937a..0000000000 --- a/tensor/stats/stats/indexview_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -/* -func TestIndexed(t *testing.T) { - dt := table.NewTable().SetNumRows(5) - dt.AddFloat64Column("data") - dt.SetFloat("data", 0, 1) - dt.SetFloat("data", 1, 2) - dt.SetFloat("data", 2, 3) - dt.SetFloat("data", 3, 4) - dt.SetFloat("data", 4, 5) - - ix := table.NewIndexed(dt) - - results := []float64{5, 15, 120, 1, 5, 1, 5, 3, 2.5, math.Sqrt(2.5), math.Sqrt(2.5) / math.Sqrt(5), - 15, 55, math.Sqrt(55), 2, math.Sqrt(2), math.Sqrt(2) / math.Sqrt(5), 3, 2, 4} - - assert.Equal(t, results[Count:Count+1], CountColumn(ix, "data")) - assert.Equal(t, results[Sum:Sum+1], SumColumn(ix, "data")) - assert.Equal(t, results[Prod:Prod+1], ProdColumn(ix, "data")) - assert.Equal(t, results[Min:Min+1], MinColumn(ix, "data")) - assert.Equal(t, results[Max:Max+1], MaxColumn(ix, "data")) - assert.Equal(t, results[MinAbs:MinAbs+1], MinAbsColumn(ix, "data")) - assert.Equal(t, results[MaxAbs:MaxAbs+1], MaxAbsColumn(ix, "data")) - assert.Equal(t, results[Mean:Mean+1], MeanColumn(ix, "data")) - assert.Equal(t, results[Var:Var+1], VarColumn(ix, "data")) - assert.Equal(t, results[Std:Std+1], StdColumn(ix, "data")) - assert.Equal(t, results[Sem:Sem+1], SemColumn(ix, "data")) - assert.Equal(t, results[L1Norm:L1Norm+1], L1NormColumn(ix, "data")) - tolassert.EqualTol(t, results[SumSq], SumSqColumn(ix, "data")[0], 1.0e-8) - tolassert.EqualTol(t, results[L2Norm], L2NormColumn(ix, "data")[0], 1.0e-8) - assert.Equal(t, results[VarPop:VarPop+1], VarPopColumn(ix, "data")) - assert.Equal(t, results[StdPop:StdPop+1], StdPopColumn(ix, "data")) - assert.Equal(t, results[SemPop:SemPop+1], SemPopColumn(ix, "data")) - assert.Equal(t, results[Median:Median+1], MedianColumn(ix, "data")) - assert.Equal(t, results[Q1:Q1+1], Q1Column(ix, "data")) - assert.Equal(t, results[Q3:Q3+1], Q3Column(ix, "data")) - - for _, stat := range StatsValues() { - tolassert.EqualTol(t, results[stat], errors.Log1(StatColumn(ix, "data", stat))[0], 1.0e-8) - } - - desc := DescAll(ix) - assert.Equal(t, len(DescStats), desc.Rows) - assert.Equal(t, 2, desc.NumColumns()) - - for ri, stat := range DescStats { - dv := desc.Float("data", ri) - // fmt.Println(ri, ag.String(), dv, results[ag]) - assert.Equal(t, results[stat], dv) - } - - desc, err := DescColumn(ix, "data") - if err != nil { - t.Error(err) - } - assert.Equal(t, len(DescStats), desc.Rows) - assert.Equal(t, 2, desc.NumColumns()) - for ri, stat := range DescStats { - dv := desc.Float("data", ri) - // fmt.Println(ri, ag.String(), dv, results[ag]) - assert.Equal(t, results[stat], dv) - } - - pcts := PctIfColumn(ix, "data", func(idx int, val float64) bool { - return val > 2 - }) - assert.Equal(t, []float64{60}, pcts) - - props := PropIfColumn(ix, "data", func(idx int, val float64) bool { - return val > 2 - }) - assert.Equal(t, []float64{0.6}, props) -} -*/ diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 36bdc3ac84..0699ad6d98 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -17,11 +17,22 @@ func init() { StatsFuncs = make(map[string]StatsFunc) StatsFuncs[Count.String()] = CountFunc StatsFuncs[Sum.String()] = SumFunc + StatsFuncs[SumAbs.String()] = SumAbsFunc + StatsFuncs[L1Norm.String()] = SumAbsFunc StatsFuncs[Prod.String()] = ProdFunc StatsFuncs[Min.String()] = MinFunc StatsFuncs[Max.String()] = MaxFunc StatsFuncs[MinAbs.String()] = MinAbsFunc StatsFuncs[MaxAbs.String()] = MaxAbsFunc + StatsFuncs[Mean.String()] = MeanFunc + StatsFuncs[Var.String()] = VarFunc + StatsFuncs[Std.String()] = StdFunc + StatsFuncs[Sem.String()] = SemFunc + StatsFuncs[SumSq.String()] = SumSqFunc + StatsFuncs[L2Norm.String()] = L2NormFunc + StatsFuncs[VarPop.String()] = VarPopFunc + StatsFuncs[StdPop.String()] = StdPopFunc + StatsFuncs[SemPop.String()] = SemPopFunc } // Standard calls a standard stats function on given tensors. @@ -35,63 +46,66 @@ func Standard(stat Stats, in, out *tensor.Indexed) { type Stats int32 //enums:enum const ( - // count of number of elements + // count of number of elements. Count Stats = iota - // sum of elements + // sum of elements. Sum - // product of elements + // sum of absolute-value-of elements (= L1Norm). + SumAbs + + // L1 Norm: sum of absolute values (= SumAbs). + L1Norm + + // product of elements. Prod - // minimum value + // minimum value. Min - // max maximum value + // maximum value. Max - // minimum absolute value + // minimum of absolute values. MinAbs - // maximum absolute value + // maximum of absolute values. MaxAbs - // mean mean value + // mean value = sum / count. Mean - // sample variance (squared diffs from mean, divided by n-1) + // sample variance (squared deviations from mean, divided by n-1). Var - // sample standard deviation (sqrt of Var) + // sample standard deviation (sqrt of Var). Std - // sample standard error of the mean (Std divided by sqrt(n)) + // sample standard error of the mean (Std divided by sqrt(n)). Sem - // L1 Norm: sum of absolute values - L1Norm - - // sum of squared values + // sum of squared values. SumSq - // L2 Norm: square-root of sum-of-squares + // L2 Norm: square-root of sum-of-squares. L2Norm - // population variance (squared diffs from mean, divided by n) + // population variance (squared diffs from mean, divided by n). VarPop - // population standard deviation (sqrt of VarPop) + // population standard deviation (sqrt of VarPop). StdPop - // population standard error of the mean (StdPop divided by sqrt(n)) + // population standard error of the mean (StdPop divided by sqrt(n)). SemPop - // middle value in sorted ordering + // middle value in sorted ordering. Median - // Q1 first quartile = 25%ile value = .25 quantile value + // Q1 first quartile = 25%ile value = .25 quantile value. Q1 - // Q3 third quartile = 75%ile value = .75 quantile value + // Q3 third quartile = 75%ile value = .75 quantile value. Q3 ) diff --git a/tensor/stats/stats/tensor.go b/tensor/stats/stats/tensor.go deleted file mode 100644 index f51ac42493..0000000000 --- a/tensor/stats/stats/tensor.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -/* - -// StatTensor returns Tensor statistic according to given Stats type applied -// to all non-NaN elements in given Tensor -func StatTensor(tsr tensor.Tensor, stat Stats) float64 { - switch stat { - case Count: - return CountTensor(tsr) - case Sum: - return SumTensor(tsr) - case Prod: - return ProdTensor(tsr) - case Min: - return MinTensor(tsr) - case Max: - return MaxTensor(tsr) - case MinAbs: - return MinAbsTensor(tsr) - case MaxAbs: - return MaxAbsTensor(tsr) - case Mean: - return MeanTensor(tsr) - case Var: - return VarTensor(tsr) - case Std: - return StdTensor(tsr) - case Sem: - return SemTensor(tsr) - case L1Norm: - return L1NormTensor(tsr) - case SumSq: - return SumSqTensor(tsr) - case L2Norm: - return L2NormTensor(tsr) - case VarPop: - return VarPopTensor(tsr) - case StdPop: - return StdPopTensor(tsr) - case SemPop: - return SemPopTensor(tsr) - // case Median: - // return MedianTensor(tsr) - // case Q1: - // return Q1Tensor(tsr) - // case Q3: - // return Q3Tensor(tsr) - } - return 0 -} - -// TensorStat applies given StatFunc function to each element in the tensor -// (automatically skips NaN elements), using float64 conversions of the values. -// ini is the initial value for the agg variable. returns final aggregate value -func TensorStat(tsr tensor.Tensor, ini float64, fun StatFunc) float64 { - ln := tsr.Len() - agg := ini - for j := 0; j < ln; j++ { - val := tsr.Float1D(j) - if !math.IsNaN(val) { - agg = fun(j, val, agg) - } - } - return agg -} - -// CountTensor returns the count of non-NaN elements in given Tensor. -func CountTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, 0, CountFunc) -} - -// SumTensor returns the sum of non-NaN elements in given Tensor. -func SumTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, 0, SumFunc) -} - -// ProdTensor returns the product of non-NaN elements in given Tensor. -func ProdTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, 1, ProdFunc) -} - -// MinTensor returns the minimum of non-NaN elements in given Tensor. -func MinTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, math.MaxFloat64, MinFunc) -} - -// MaxTensor returns the maximum of non-NaN elements in given Tensor. -func MaxTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, -math.MaxFloat64, MaxFunc) -} - -// MinAbsTensor returns the minimum of non-NaN elements in given Tensor. -func MinAbsTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, math.MaxFloat64, MinAbsFunc) -} - -// MaxAbsTensor returns the maximum of non-NaN elements in given Tensor. -func MaxAbsTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, -math.MaxFloat64, MaxAbsFunc) -} - -// MeanTensor returns the mean of non-NaN elements in given Tensor. -func MeanTensor(tsr tensor.Tensor) float64 { - cnt := CountTensor(tsr) - if cnt == 0 { - return 0 - } - return SumTensor(tsr) / cnt -} - -// VarTensor returns the sample variance of non-NaN elements in given Tensor. -func VarTensor(tsr tensor.Tensor) float64 { - cnt := CountTensor(tsr) - if cnt < 2 { - return 0 - } - mean := SumTensor(tsr) / cnt - vr := TensorStat(tsr, 0, func(idx int, val float64, agg float64) float64 { - dv := val - mean - return agg + dv*dv - }) - return vr / (cnt - 1) -} - -// StdTensor returns the sample standard deviation of non-NaN elements in given Tensor. -func StdTensor(tsr tensor.Tensor) float64 { - return math.Sqrt(VarTensor(tsr)) -} - -// SemTensor returns the sample standard error of the mean of non-NaN elements in given Tensor. -func SemTensor(tsr tensor.Tensor) float64 { - cnt := CountTensor(tsr) - if cnt < 2 { - return 0 - } - return StdTensor(tsr) / math.Sqrt(cnt) -} - -// L1NormTensor returns the L1 norm: sum of absolute values of non-NaN elements in given Tensor. -func L1NormTensor(tsr tensor.Tensor) float64 { - return TensorStat(tsr, 0, L1NormFunc) -} - -// SumSqTensor returns the sum-of-squares of non-NaN elements in given Tensor. -func SumSqTensor(tsr tensor.Tensor) float64 { - n := tsr.Len() - if n < 2 { - if n == 1 { - return math.Abs(tsr.Float1D(0)) - } - return 0 - } - var ( - scale float64 = 0 - ss float64 = 1 - ) - for j := 0; j < n; j++ { - v := tsr.Float1D(j) - if v == 0 || math.IsNaN(v) { - continue - } - absxi := math.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * scale * ss -} - -// L2NormTensor returns the L2 norm: square root of sum-of-squared values of non-NaN elements in given Tensor. -func L2NormTensor(tsr tensor.Tensor) float64 { - return math.Sqrt(SumSqTensor(tsr)) -} - -// VarPopTensor returns the population variance of non-NaN elements in given Tensor. -func VarPopTensor(tsr tensor.Tensor) float64 { - cnt := CountTensor(tsr) - if cnt < 2 { - return 0 - } - mean := SumTensor(tsr) / cnt - vr := TensorStat(tsr, 0, func(idx int, val float64, agg float64) float64 { - dv := val - mean - return agg + dv*dv - }) - return vr / cnt -} - -// StdPopTensor returns the population standard deviation of non-NaN elements in given Tensor. -func StdPopTensor(tsr tensor.Tensor) float64 { - return math.Sqrt(VarPopTensor(tsr)) -} - -// SemPopTensor returns the population standard error of the mean of non-NaN elements in given Tensor. -func SemPopTensor(tsr tensor.Tensor) float64 { - cnt := CountTensor(tsr) - if cnt < 2 { - return 0 - } - return StdPopTensor(tsr) / math.Sqrt(cnt) -} -*/ diff --git a/tensor/stats/stats/tensor_test.go b/tensor/stats/stats/tensor_test.go deleted file mode 100644 index 328b9bf338..0000000000 --- a/tensor/stats/stats/tensor_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -/* -func TestTsrAgg(t *testing.T) { - tsr := tensor.New[float64]([]int{5}).(*tensor.Float64) - tsr.Values = []float64{1, 2, 3, 4, 5} - - results := []float64{5, 15, 120, 1, 5, 1, 5, 3, 2.5, math.Sqrt(2.5), math.Sqrt(2.5) / math.Sqrt(5), - 15, 55, math.Sqrt(55), 2, math.Sqrt(2), math.Sqrt(2) / math.Sqrt(5), 3, 2, 4} - - assert.Equal(t, results[Count], CountTensor(tsr)) - assert.Equal(t, results[Sum], SumTensor(tsr)) - assert.Equal(t, results[Prod], ProdTensor(tsr)) - assert.Equal(t, results[Min], MinTensor(tsr)) - assert.Equal(t, results[Max], MaxTensor(tsr)) - assert.Equal(t, results[MinAbs], MinAbsTensor(tsr)) - assert.Equal(t, results[MaxAbs], MaxAbsTensor(tsr)) - assert.Equal(t, results[Mean], MeanTensor(tsr)) - assert.Equal(t, results[Var], VarTensor(tsr)) - assert.Equal(t, results[Std], StdTensor(tsr)) - assert.Equal(t, results[Sem], SemTensor(tsr)) - assert.Equal(t, results[L1Norm], L1NormTensor(tsr)) - tolassert.EqualTol(t, results[SumSq], SumSqTensor(tsr), 1.0e-8) - tolassert.EqualTol(t, results[L2Norm], L2NormTensor(tsr), 1.0e-8) - assert.Equal(t, results[VarPop], VarPopTensor(tsr)) - assert.Equal(t, results[StdPop], StdPopTensor(tsr)) - assert.Equal(t, results[SemPop], SemPopTensor(tsr)) - - for stat := Count; stat <= SemPop; stat++ { - tolassert.EqualTol(t, results[stat], StatTensor(tsr, stat), 1.0e-8) - } -} -*/ diff --git a/tensor/stats/stats/vecfuncs.go b/tensor/stats/stats/vecfuncs.go index fe0709eaf6..6e157ab4ac 100644 --- a/tensor/stats/stats/vecfuncs.go +++ b/tensor/stats/stats/vecfuncs.go @@ -6,23 +6,81 @@ package stats import ( "math" + "slices" "cogentcore.org/core/tensor" ) -// NFunc is the nfun for stats functions, return the length of the -// first tensor, and initializing the second one to hold the output -// as the SubSpace of the first tensor. +// VectorizeOut64 is a version of the [tensor.Vectorize] function +// for stats, which makes a Float64 output tensor for aggregating +// and computing values, and then copies the results back to the +// original output. This allows stats functions to operate directly +// on integer valued inputs and produce sensible results. +// It automatically calls NFunc for the nfun function, +// and returns the Float64 output tensor for further processing as needed. +// It uses the _last_ tensor as the output, allowing for multiple inputs, +// as in the case of VarVecFun. +func VectorizeOut64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) *tensor.Indexed { + n := NFunc(tsr...) + if n <= 0 { + return nil + } + nt := len(tsr) + out := tsr[nt-1] + o64 := tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + etsr := slices.Clone(tsr) + etsr[nt-1] = o64 + for idx := range n { + fun(idx, etsr...) + } + nsub := out.Tensor.Len() + for i := range nsub { + out.Tensor.SetFloat1D(i, o64.Tensor.Float1D(i)) + } + return o64 +} + +// Vectorize2Out64 is a version of the [tensor.Vectorize] function +// for stats, which makes two Float64 output tensors for aggregating +// and computing values, returning them for final computation. +// It automatically calls NFunc for the nfun function. +func Vectorize2Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2 *tensor.Indexed) { + n := NFunc(tsr...) + if n <= 0 { + return nil, nil + } + nt := len(tsr) + out := tsr[nt-1] + out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + for idx := range n { + fun(idx, tsr[0], out1, out2) + } + return out1, out2 +} + +// OutShape returns the output shape based on given input +// tensor, with outer row dim = 1. +func OutShape(ish *tensor.Shape) *tensor.Shape { + osh := &tensor.Shape{} + osh.CopyShape(ish) + osh.Sizes[0] = 1 + return osh +} + +// NFunc is the nfun for stats functions, returning the length of the +// first tensor, and initializing the _last_ one to hold the output +// with the first, row dimension set to 1. func NFunc(tsr ...*tensor.Indexed) int { - if len(tsr) != 2 { + nt := len(tsr) + if nt < 2 { return 0 } - n := tsr[0].Len() - sh := tensor.Shape{} - sh.CopyShape(tsr[0].Tensor.Shape()) - sh.Sizes[0] = 1 - tsr[1].Tensor.SetShape(sh.Sizes, sh.Names...) - tsr[1].Indexes = []int{0} + in, out := tsr[0], tsr[nt-1] + n := in.Len() + osh := OutShape(in.Tensor.Shape()) + out.Tensor.SetShape(osh.Sizes, osh.Names...) + out.Indexes = []int{0} return n } @@ -30,12 +88,12 @@ func NFunc(tsr ...*tensor.Indexed) int { // the Cell subspace per row and initializing the aggregation values for first index. // It also skips over NaN missing values. func StatVecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { - nsub := out.Len() - for i := 0; i < nsub; i++ { + nsub := out.Tensor.Len() + for i := range nsub { if idx == 0 { out.Tensor.SetFloat1D(i, ini) } - val := in.Tensor.FloatRowCell(in.Indexes[idx], i) + val := in.FloatRowCell(idx, i) if math.IsNaN(val) { continue } @@ -43,59 +101,134 @@ func StatVecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, ag } } -// SumVecFunc is a Vectorize function for computing the sum +// SumVecFunc is a Vectorize function for computing the sum. func SumVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + val }) } -// CountVecFunc is a Vectorize function for computing the count +// SumAbsVecFunc is a Vectorize function for computing the sum of abs values. +// This is also known as the L1 norm. +func SumAbsVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + math.Abs(val) + }) +} + +// CountVecFunc is a Vectorize function for computing the count. func CountVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + 1 }) } -// ProdVecFunc is a Vectorize function for computing the product +// ProdVecFunc is a Vectorize function for computing the product. func ProdVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { return agg * val }) } -// MinVecFunc is a Vectorize function for computing the min +// MinVecFunc is a Vectorize function for computing the min. func MinVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, val) }) } -// MaxVecFunc is a Vectorize function for computing the min +// MaxVecFunc is a Vectorize function for computing the max. func MaxVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, val) }) } -// MinAbsVecFunc is a Vectorize function for computing the min of abs values +// MinAbsVecFunc is a Vectorize function for computing the min of abs. func MinAbsVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, math.Abs(val)) }) } -// MaxAbsVecFunc is a Vectorize function for computing the max of abs values +// MaxAbsVecFunc is a Vectorize function for computing the max of abs. func MaxAbsVecFunc(idx int, tsr ...*tensor.Indexed) { StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, math.Abs(val)) }) } -// L1NormFunc is an StatFunc that computes the L1 norm: sum of absolute values -// use 0 as initial value. -// func L1NormFunc(idx int, val float64, agg float64) float64 { -// return agg + math.Abs(val) -// } +/////////////////////////////////////////////////////\ +// Two input Tensors -// Note: SumSq is not numerically stable for large N in simple func form. +// StatVec2inFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// This version has 2 input vectors, the second input being the output of another stat +// e.g., the mean for Var. +// It also skips over NaN missing values. +func StatVec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(val1, val2, agg float64) float64) { + nsub := out.Tensor.Len() + for i := range nsub { + if idx == 0 { + out.Tensor.SetFloat1D(i, ini) + } + val1 := in1.FloatRowCell(idx, i) + if math.IsNaN(val1) { + continue + } + val2 := in2.Tensor.Float1D(i) + out.Tensor.SetFloat1D(i, fun(val1, val2, out.Tensor.Float1D(i))) + } +} + +// VarVecFunc is a Vectorize function for computing the variance, +// using 3 tensors: in, mean, out. +func VarVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { + dv := val1 - val2 + return agg + dv*dv + }) +} + +/////////////////////////////////////////////////////\ +// Two output Tensors + +// StatVec2outFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// This version has 2 output vectors, for separate integration of scale sum squared +// It also skips over NaN missing values. +func StatVec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(val, agg1, agg2 float64) (float64, float64)) { + nsub := out2.Tensor.Len() + for i := range nsub { + if idx == 0 { + out1.Tensor.SetFloat1D(i, ini1) + out2.Tensor.SetFloat1D(i, ini2) + } + val := in.FloatRowCell(idx, i) + if math.IsNaN(val) { + continue + } + ag1, ag2 := out1.Tensor.Float1D(i), out2.Tensor.Float1D(i) + ag1, ag2 = fun(val, ag1, ag2) + out1.Tensor.SetFloat1D(i, ag1) + out2.Tensor.SetFloat1D(i, ag2) + } +} + +// SumSqVecFunc is a Vectorize function for computing the sum of squares, +// using 2 separate aggregation tensors. +func SumSqVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { + if val == 0 { + return scale, ss + } + absxi := math.Abs(val) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } + return scale, ss + }) +} From a4b99a561db9e5ac34f05d5d162cb173d9a37e7c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 9 Sep 2024 15:56:20 -0700 Subject: [PATCH 007/311] quantiles working --- tensor/bits.go | 4 +- tensor/indexed.go | 3 +- tensor/number.go | 10 +-- tensor/stats/stats/funcs_test.go | 111 +++++++++++++++++++++++++++++-- tensor/stats/stats/quantiles.go | 84 +++++++++++------------ tensor/stats/stats/stats.go | 3 + tensor/string.go | 4 +- tensor/tensor.go | 4 +- tensor/tensor_test.go | 4 +- tensor/tensormpi/tensor.go | 2 +- tensor/typegen.go | 9 +++ 11 files changed, 176 insertions(+), 62 deletions(-) create mode 100644 tensor/typegen.go diff --git a/tensor/bits.go b/tensor/bits.go index ebbce80fc9..8848e686a9 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -249,9 +249,9 @@ func (tsr *Bits) CopyFrom(frm Tensor) { } } -// CopyShapeFrom copies just the shape from given source tensor +// SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). -func (tsr *Bits) CopyShapeFrom(frm Tensor) { +func (tsr *Bits) SetShapeFrom(frm Tensor) { tsr.SetShape(frm.Shape().Sizes, frm.Shape().Names...) } diff --git a/tensor/indexed.go b/tensor/indexed.go index c3a5f43dca..502661ce86 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -251,7 +251,8 @@ func (ix *Indexed) NewTensor() Tensor { return nt } -// Clone returns a copy of the current index view with its own index memory. +// Clone returns a copy of the current Indexed view into the same +// underlying Tensor as the source, with its own index memory. func (ix *Indexed) Clone() *Indexed { nix := &Indexed{} nix.CopyFrom(ix) diff --git a/tensor/number.go b/tensor/number.go index 244a3ae970..c5405c9cee 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -51,19 +51,19 @@ func NewFloat64(sizes []int, names ...string) *Float64 { // NewInt returns a new Int tensor // with the given sizes per dimension (shape), and optional dimension names. func NewInt(sizes []int, names ...string) *Int { - return New[float64](sizes, names...).(*Int) + return New[int](sizes, names...).(*Int) } // NewInt32 returns a new Int32 tensor // with the given sizes per dimension (shape), and optional dimension names. func NewInt32(sizes []int, names ...string) *Int32 { - return New[float64](sizes, names...).(*Int32) + return New[int32](sizes, names...).(*Int32) } // NewByte returns a new Byte tensor // with the given sizes per dimension (shape), and optional dimension names. func NewByte(sizes []int, names ...string) *Byte { - return New[float64](sizes, names...).(*Byte) + return New[uint8](sizes, names...).(*Byte) } // NewNumber returns a new n-dimensional tensor of numerical values @@ -288,9 +288,9 @@ func (tsr *Number[T]) CopyFrom(frm Tensor) { } } -// CopyShapeFrom copies just the shape from given source tensor +// SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). -func (tsr *Number[T]) CopyShapeFrom(frm Tensor) { +func (tsr *Number[T]) SetShapeFrom(frm Tensor) { tsr.SetShape(frm.Shape().Sizes, frm.Shape().Names...) } diff --git a/tensor/stats/stats/funcs_test.go b/tensor/stats/stats/funcs_test.go index f3514df7ce..7fc1f2fd4f 100644 --- a/tensor/stats/stats/funcs_test.go +++ b/tensor/stats/stats/funcs_test.go @@ -8,13 +8,10 @@ import ( "math" "testing" - "cogentcore.org/core/base/tolassert" "cogentcore.org/core/tensor" "github.com/stretchr/testify/assert" ) -// todo: add int, tensor cell tests! - func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} tsr := tensor.NewNumberFromSlice(vals) @@ -22,7 +19,7 @@ func TestFuncs64(t *testing.T) { out := tensor.NewFloat64([]int{1}) oix := tensor.NewIndexed(out) - results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11)} + results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} CountFunc(ix, oix) assert.Equal(t, results[Count], out.Values[0]) @@ -75,9 +72,113 @@ func TestFuncs64(t *testing.T) { L2NormFunc(ix, oix) assert.InDelta(t, results[L2Norm], out.Values[0], 1.0e-8) + MedianFunc(ix, oix) + assert.InDelta(t, results[Median], out.Values[0], 1.0e-8) + + Q1Func(ix, oix) + assert.InDelta(t, results[Q1], out.Values[0], 1.0e-8) + + Q3Func(ix, oix) + assert.InDelta(t, results[Q3], out.Values[0], 1.0e-8) + + for stat := Count; stat <= Q3; stat++ { + Standard(stat, ix, oix) + assert.InDelta(t, results[stat], out.Values[0], 1.0e-8) + } +} + +func TestFuncsInt(t *testing.T) { + vals := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} + tsr := tensor.NewNumberFromSlice(vals) + ix := tensor.NewIndexed(tsr) + out := tensor.NewInt([]int{1}) + oix := tensor.NewIndexed(out) + + results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} + + CountFunc(ix, oix) + assert.Equal(t, results[Count], out.Values[0]) + + SumFunc(ix, oix) + assert.Equal(t, results[Sum], out.Values[0]) + + SumAbsFunc(ix, oix) + assert.Equal(t, results[SumAbs], out.Values[0]) + + ProdFunc(ix, oix) + assert.Equal(t, results[Prod], out.Values[0]) + + MinFunc(ix, oix) + assert.Equal(t, results[Min], out.Values[0]) + + MaxFunc(ix, oix) + assert.Equal(t, results[Max], out.Values[0]) + + MinAbsFunc(ix, oix) + assert.Equal(t, results[MinAbs], out.Values[0]) + + MaxAbsFunc(ix, oix) + assert.Equal(t, results[MaxAbs], out.Values[0]) + + MeanFunc(ix, oix) + assert.Equal(t, results[Mean], out.Values[0]) + + VarFunc(ix, oix) + assert.Equal(t, results[Var], out.Values[0]) + + StdFunc(ix, oix) + assert.Equal(t, results[Std], out.Values[0]) + + SemFunc(ix, oix) + assert.Equal(t, results[Sem], out.Values[0]) + + VarPopFunc(ix, oix) + assert.Equal(t, results[VarPop], out.Values[0]) + + StdPopFunc(ix, oix) + assert.Equal(t, results[StdPop], out.Values[0]) + + SemPopFunc(ix, oix) + assert.Equal(t, results[SemPop], out.Values[0]) + + SumSqFunc(ix, oix) + assert.Equal(t, results[SumSq], out.Values[0]) + + L2NormFunc(ix, oix) + assert.Equal(t, results[L2Norm], out.Values[0]) + for stat := Count; stat <= SemPop; stat++ { Standard(stat, ix, oix) - tolassert.EqualTol(t, results[stat], out.Values[0], 1.0e-8) + assert.Equal(t, results[stat], out.Values[0]) + } +} + +func TestFuncsCell(t *testing.T) { + vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9} + tsr := tensor.NewFloat32([]int{20, 10}) + + for i := range 20 { + for j := range 10 { + tsr.SetFloatRowCell(i, j, vals[j]) + } + } + + ix := tensor.NewIndexed(tsr) + out := tensor.NewFloat32([]int{20, 10}) + oix := tensor.NewIndexed(out) + + CountFunc(ix, oix) + nsub := out.Len() + for i := range nsub { + assert.Equal(t, 20.0, out.FloatRowCell(0, i)) + } + MeanFunc(ix, oix) + for i := range nsub { + assert.InDelta(t, vals[i], out.FloatRowCell(0, i), 1.0e-7) // lower tol, using float32 + } + VarFunc(ix, oix) + for i := range nsub { + assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) } } diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index dc02c330a9..df4e87467b 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -8,66 +8,66 @@ import ( "math" "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor/table" + "cogentcore.org/core/tensor" ) -// QuantilesIndex returns the given quantile(s) of non-NaN elements in given -// Indexed indexed view of an table.Table, for given column index. -// Column must be a 1d Column -- returns nil for n-dimensional columns. -// qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. Uses linear interpolation. +// QuantilesFunc returns the given quantile(s) of non-NaN elements in given +// 1D tensor. Because sorting uses indexes, this only works for 1D case. +// If needed for a sub-space of values, that can be extracted through slicing +// and then used. Returns and logs an error if not 1D. +// qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. +// Uses linear interpolation. // Because this requires a sort, it is more efficient to get as many quantiles // as needed in one pass. -func QuantilesIndex(ix *table.Indexed, colIndex int, qs []float64) []float64 { - nq := len(qs) - if nq == 0 { - return nil +func QuantilesFunc(in, qs, out *tensor.Indexed) error { + if in.Tensor.NumDims() != 1 { + return errors.Log(errors.New("stats.QuantilesFunc: only 1D input tensors allowed")) } - col := ix.Table.Columns[colIndex] - if col.NumDims() > 1 { // only valid for 1D - return nil + if qs.Tensor.NumDims() != 1 { + return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } - rvs := make([]float64, nq) - six := ix.Clone() // leave original indexes intact - six.Filter(func(et *table.Table, row int) bool { // get rid of NaNs in this column - if math.IsNaN(col.Float1D(row)) { - return false - } - return true - }) - six.SortColumn(colIndex, true) - sz := len(six.Indexes) - 1 // length of our own index list + sin := in.Clone() + sin.ExcludeMissing1D() + sin.Sort(tensor.Ascending) + sz := len(sin.Indexes) - 1 // length of our own index list fsz := float64(sz) - for i, q := range qs { + out.Tensor.SetShapeFrom(qs.Tensor) + nq := qs.Tensor.Len() + for i := range nq { + q := qs.Tensor.Float1D(i) val := 0.0 qi := q * fsz lwi := math.Floor(qi) lwii := int(lwi) if lwii >= sz { - val = col.Float1D(six.Indexes[sz]) + val = sin.FloatRowCell(sz, 0) } else if lwii < 0 { - val = col.Float1D(six.Indexes[0]) + val = sin.FloatRowCell(0, 0) } else { phi := qi - lwi - lwv := col.Float1D(six.Indexes[lwii]) - hiv := col.Float1D(six.Indexes[lwii+1]) + lwv := sin.FloatRowCell(lwii, 0) + hiv := sin.FloatRowCell(lwii+1, 0) val = (1-phi)*lwv + phi*hiv } - rvs[i] = val + out.Tensor.SetFloat1D(i, val) } - return rvs + return nil } -// Quantiles returns the given quantile(s) of non-Null, non-NaN elements in given -// Indexed indexed view of an table.Table, for given column name. -// If name not found, nil is returned -- use Try version for error message. -// Column must be a 1d Column -- returns nil for n-dimensional columns. -// qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. Uses linear interpolation. -// Because this requires a sort, it is more efficient to get as many quantiles -// as needed in one pass. -func Quantiles(ix *table.Indexed, column string, qs []float64) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return QuantilesIndex(ix, colIndex, qs) +// MedianFunc computes the median (50% quantile) of tensor values. +// See [StatsFunc] for general information. +func MedianFunc(in, out *tensor.Indexed) { + QuantilesFunc(in, tensor.NewIndexed(tensor.NewNumberFromSlice([]float64{.5})), out) +} + +// Q1Func computes the first quantile (25%) of tensor values. +// See [StatsFunc] for general information. +func Q1Func(in, out *tensor.Indexed) { + QuantilesFunc(in, tensor.NewIndexed(tensor.NewNumberFromSlice([]float64{.25})), out) +} + +// Q3Func computes the third quantile (75%) of tensor values. +// See [StatsFunc] for general information. +func Q3Func(in, out *tensor.Indexed) { + QuantilesFunc(in, tensor.NewIndexed(tensor.NewNumberFromSlice([]float64{.75})), out) } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 0699ad6d98..86b81cce5a 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -33,6 +33,9 @@ func init() { StatsFuncs[VarPop.String()] = VarPopFunc StatsFuncs[StdPop.String()] = StdPopFunc StatsFuncs[SemPop.String()] = SemPopFunc + StatsFuncs[Median.String()] = MedianFunc + StatsFuncs[Q1.String()] = Q1Func + StatsFuncs[Q3.String()] = Q3Func } // Standard calls a standard stats function on given tensors. diff --git a/tensor/string.go b/tensor/string.go index 2cf04fb963..884c810abc 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -243,9 +243,9 @@ func (tsr *String) CopyFrom(frm Tensor) { } } -// CopyShapeFrom copies just the shape from given source tensor +// SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). -func (tsr *String) CopyShapeFrom(frm Tensor) { +func (tsr *String) SetShapeFrom(frm Tensor) { tsr.SetShape(frm.Shape().Sizes, frm.Shape().Names...) } diff --git a/tensor/tensor.go b/tensor/tensor.go index 54f6150595..fae9440733 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -146,9 +146,9 @@ type Tensor interface { // otherwise it goes through appropriate standard type. CopyFrom(from Tensor) - // CopyShapeFrom copies just the shape from given source tensor + // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). - CopyShapeFrom(from Tensor) + SetShapeFrom(from Tensor) // CopyCellsFrom copies given range of values from other tensor into this tensor, // using flat 1D indexes: to = starting index in this Tensor to start copying into, diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index f2c7ca3bd2..1f0992514d 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -46,7 +46,7 @@ func TestTensorString(t *testing.T) { assert.Equal(t, "testing", tsr.StringValue([]int{1, 1})) cln.SetString1D(5, "ctesting") - cln.CopyShapeFrom(tsr) + cln.SetShapeFrom(tsr) assert.Equal(t, "ctesting", cln.StringValue([]int{1, 1})) cln.CopyCellsFrom(tsr, 5, 4, 2) @@ -106,7 +106,7 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 2.17, tsr.Float([]int{1, 1})) cln.SetFloat1D(5, 9.9) - cln.CopyShapeFrom(tsr) + cln.SetShapeFrom(tsr) assert.Equal(t, 9.9, cln.Float([]int{1, 1})) cln.CopyCellsFrom(tsr, 5, 4, 2) diff --git a/tensor/tensormpi/tensor.go b/tensor/tensormpi/tensor.go index f25e42268c..c9f929c1bc 100644 --- a/tensor/tensormpi/tensor.go +++ b/tensor/tensormpi/tensor.go @@ -119,7 +119,7 @@ func ReduceTensor(dest, src tensor.Tensor, comm *mpi.Comm, op mpi.Op) error { } slen := src.Len() if slen != dest.Len() { - dest.CopyShapeFrom(src) + dest.SetShapeFrom(src) } var err error switch dt { diff --git a/tensor/typegen.go b/tensor/typegen.go new file mode 100644 index 0000000000..2158c0bfde --- /dev/null +++ b/tensor/typegen.go @@ -0,0 +1,9 @@ +// Code generated by "core generate"; DO NOT EDIT. + +package tensor + +import ( + "cogentcore.org/core/types" +) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a tensor.Tensor that provides a\nspecific view onto the Tensor defined by the set of indexes, which\napply to the outer-most row dimension (with default row-major indexing).\nThis provides an efficient way of sorting and filtering a tensor by only\nupdating the indexes while doing nothing to the Tensor itself.\nTo produce a tensor that has data actually organized according to the\nindexed order, call the NewTensor method.\nUse the [Set]FloatRowCell methods wherever possible, for the most efficient\nand natural indirection through the indexes. The 1D methods on underlying\ntensor data do not indirect through the indexes and must be called directly\non the [Tensor].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows."}}}) From 3bf95d22d5fa8500a9983bffc93529518cb04fa0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 9 Sep 2024 17:18:08 -0700 Subject: [PATCH 008/311] updated readme with big picture vision of tensor data framework. --- tensor/README.md | 18 +++++-- tensor/indexed.go | 10 ++-- tensor/stats/stats/desc.go | 102 ++++++++++++++++-------------------- tensor/stats/stats/stats.go | 2 +- 4 files changed, 69 insertions(+), 63 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index 31448b1344..fea4086d38 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -2,13 +2,25 @@ Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used `numpy` and `pandas` libraries in python, and the commercial MATLAB framework. -* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common row dimension as the outer-most dimension of each tensor. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `Indexed` that provides an indexed view into the rows of the table for highly efficient filtering and sorting of data. +The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. + +The [cosl](../cosl) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. + +The indexes provide a specific view onto the underlying [Tensor] data, applying to the outer-most _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. + +Use the `[Set]FloatRowCell` methods wherever possible, for the most efficient and natural indirection through the indexes. The 1D methods on underlying tensor data do not indirect through the indexes and must be called directly on the [Tensor]. + +* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `Indexed` that provides an indexed view into the rows of the table for highly efficient filtering and sorting of data. Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including the plotcore. -* [tensorcore](tensorcore) provides core widgets for the `Tensor` and `Table` data. +* [datafs](datafs) provides a virtual filesystem (FS) for organizing arbitrary collections of data, supporting interactive, ad-hoc (notebook style) as well as systematic data processing. Interactive [cosl](../cosl) shell commands (`cd`, `ls`, `mkdir` etc) can be used to navigate the data space, with numerical expressions immediately available to operate on the data and save results back to the filesystem. Furthermore, the data can be directly copied to / from the OS filesystem to persist it, and `cosl` can transparently access data on remote systems through ssh. Furthermore, the [databrowser](databrowser) provides a fully interactive GUI for inspecting and plotting data. + +* [tensorcore](tensorcore) provides core widgets for graphically displaying the `Tensor` and `Table` data, which are used in `datafs`. + +* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data. -* [stats](stats) implements a number of different ways of analyzing tensor and table data. +* [stats](stats) implements a number of different ways of analyzing tensor and table data * [plot/plotcore](../plot/plotcore) supports interactive plotting of `Table` data. diff --git a/tensor/indexed.go b/tensor/indexed.go index 502661ce86..a54654876e 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -17,10 +17,14 @@ import ( // Indexed is an indexed wrapper around a tensor.Tensor that provides a // specific view onto the Tensor defined by the set of indexes, which // apply to the outer-most row dimension (with default row-major indexing). -// This provides an efficient way of sorting and filtering a tensor by only +// This is the universal representation of a homogenous data type in the +// [tensor] package framework, from scalar to vector, matrix, and beyond, +// because it can efficiently represent any kind of element with sufficient +// flexibility to enable a huge range of computations to be elegantly expressed. +// For example, sorting and filtering a tensor only requires // updating the indexes while doing nothing to the Tensor itself. -// To produce a tensor that has data actually organized according to the -// indexed order, call the NewTensor method. +// To produce a new [Tensor] that has its raw data actually organized according +// to the indexed order, call the [NewTensor] method. // Use the [Set]FloatRowCell methods wherever possible, for the most efficient // and natural indirection through the indexes. The 1D methods on underlying // tensor data do not indirect through the indexes and must be called directly diff --git a/tensor/stats/stats/desc.go b/tensor/stats/stats/desc.go index d0a4ee1607..3483a07bd5 100644 --- a/tensor/stats/stats/desc.go +++ b/tensor/stats/stats/desc.go @@ -8,11 +8,52 @@ import ( "cogentcore.org/core/tensor/table" ) -// DescStats are all the standard stats -var DescStats = []Stats{Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3} +// StdStats are the standard descriptive stats computed in StdStatsData, +// for one-dimensional tensors. For higher-dimensional cases, the last 3 +// quartile-based ones are excluded because they are not compatible. +var StdStats = []Stats{Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3} + +// StdStatsData adds standard descriptive statistics for given tensor +// to the given [datafs] directory. +// This is an easy way to provide a comprehensive description of data. +// Stats are in StdStats list: Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3 +func StdStatsData(dir *datafs.Data, tsr *tensor.Indexed) { + + st := ix.Table + col := st.Columns[colIndex] + stats := DescStats + if col.NumDims() > 1 { // nd cannot do qiles + stats = DescStatsND + } + nAgg := len(stats) + dt := table.NewTable().SetNumRows(nAgg) + dt.AddStringColumn("Stat") + dt.AddFloat64TensorColumn(st.ColumnNames[colIndex], col.Shape().Sizes[1:], col.Shape().Names[1:]...) + dtnm := dt.Columns[0] + dtst := dt.Columns[1] + _, csz := col.RowCellSize() + for i, styp := range DescStatsND { + // ag := StatIndex(ix, colIndex, styp) + ag := 0.0 + si := i * csz + for j := 0; j < csz; j++ { + dtst.SetFloat1D(si+j, ag[j]) + } + dtnm.SetString1D(i, styp.String()) + } + if col.NumDims() == 1 { + sq := len(DescStatsND) + qs := []float64{.25, .5, .75} + qvs := QuantilesIndex(ix, colIndex, qs) + for i, qv := range qvs { + dtst.SetFloat1D(sq+i, qv) + dtnm.SetString1D(sq+i, DescStats[sq+i].String()) + } + } + */ + return ix.Table // dt +} -// DescStatsND are all the standard stats for n-dimensional (n > 1) data -- cannot do quantiles -var DescStatsND = []Stats{Count, Mean, Std, Sem, Min, Max} // DescAll returns a table of standard descriptive stats for // all numeric columns in given table, operating over all non-Null, non-NaN elements @@ -65,56 +106,5 @@ func DescAll(ix *table.Indexed) *table.Table { return ix.Table // dt } -// DescIndex returns a table of standard descriptive aggregates -// of non-Null, non-NaN elements in given Indexed indexed view of an -// table.Table, for given column index. -func DescIndex(ix *table.Indexed, colIndex int) *table.Table { - /* - st := ix.Table - col := st.Columns[colIndex] - stats := DescStats - if col.NumDims() > 1 { // nd cannot do qiles - stats = DescStatsND - } - nAgg := len(stats) - dt := table.NewTable().SetNumRows(nAgg) - dt.AddStringColumn("Stat") - dt.AddFloat64TensorColumn(st.ColumnNames[colIndex], col.Shape().Sizes[1:], col.Shape().Names[1:]...) - dtnm := dt.Columns[0] - dtst := dt.Columns[1] - _, csz := col.RowCellSize() - for i, styp := range DescStatsND { - // ag := StatIndex(ix, colIndex, styp) - ag := 0.0 - si := i * csz - for j := 0; j < csz; j++ { - dtst.SetFloat1D(si+j, ag[j]) - } - dtnm.SetString1D(i, styp.String()) - } - if col.NumDims() == 1 { - sq := len(DescStatsND) - qs := []float64{.25, .5, .75} - qvs := QuantilesIndex(ix, colIndex, qs) - for i, qv := range qvs { - dtst.SetFloat1D(sq+i, qv) - dtnm.SetString1D(sq+i, DescStats[sq+i].String()) - } - } - */ - return ix.Table // dt -} -// DescColumn returns a table of standard descriptive stats -// of non-NaN elements in given Indexed indexed view of an -// table.Table, for given column name. -// If name not found, returns error message. -// Return value is size of each column cell -- 1 for scalar 1D columns -// and N for higher-dimensional columns. -func DescColumn(ix *table.Indexed, column string) (*table.Table, error) { - colIndex, err := ix.Table.ColumnIndex(column) - if err != nil { - return nil, err - } - return DescIndex(ix, colIndex), nil -} + diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 86b81cce5a..6a210e5471 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -38,7 +38,7 @@ func init() { StatsFuncs[Q3.String()] = Q3Func } -// Standard calls a standard stats function on given tensors. +// Standard calls a standard Stats enum function on given tensors. // Output results are in the out tensor. func Standard(stat Stats, in, out *tensor.Indexed) { StatsFuncs[stat.String()](in, out) From 6e327ddaf28300821b582af968d8a0be90d17f28 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 10 Sep 2024 04:21:33 -0700 Subject: [PATCH 009/311] most metric functions done --- .../{stats/stats/desc.go => datafs/stats.go} | 14 +- tensor/indexed.go | 170 +++++- tensor/stats/metric/README.md | 2 +- tensor/stats/metric/enumgen.go | 48 +- tensor/stats/metric/funcs.go | 219 +++++++ tensor/stats/metric/metric_test.go | 120 ++-- tensor/stats/metric/metrics.go | 111 ++-- tensor/stats/metric/squares.go | 562 +----------------- tensor/stats/metric/tensor.go | 33 +- tensor/stats/metric/vecfuncs.go | 330 ++++++++++ tensor/stats/stats/funcs.go | 39 -- tensor/stats/stats/if.go | 115 ---- tensor/stats/stats/stats.go | 68 ++- .../stats/{funcs_test.go => stats_test.go} | 3 +- 14 files changed, 857 insertions(+), 977 deletions(-) rename tensor/{stats/stats/desc.go => datafs/stats.go} (92%) create mode 100644 tensor/stats/metric/funcs.go create mode 100644 tensor/stats/metric/vecfuncs.go delete mode 100644 tensor/stats/stats/if.go rename tensor/stats/stats/{funcs_test.go => stats_test.go} (98%) diff --git a/tensor/stats/stats/desc.go b/tensor/datafs/stats.go similarity index 92% rename from tensor/stats/stats/desc.go rename to tensor/datafs/stats.go index 3483a07bd5..e6606692ee 100644 --- a/tensor/stats/stats/desc.go +++ b/tensor/datafs/stats.go @@ -2,23 +2,25 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package stats +package datafs import ( + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/table" ) // StdStats are the standard descriptive stats computed in StdStatsData, -// for one-dimensional tensors. For higher-dimensional cases, the last 3 +// for one-dimensional tensors. For higher-dimensional cases, the last 3 // quartile-based ones are excluded because they are not compatible. -var StdStats = []Stats{Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3} +var StdStats = []Stats{stats.Count, stats.Mean, stats.Std, stats.Sem, stats.Min, stats.Max, stats.Q1, stats.Median, stats.Q3} // StdStatsData adds standard descriptive statistics for given tensor // to the given [datafs] directory. // This is an easy way to provide a comprehensive description of data. // Stats are in StdStats list: Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3 func StdStatsData(dir *datafs.Data, tsr *tensor.Indexed) { - + /* st := ix.Table col := st.Columns[colIndex] stats := DescStats @@ -54,7 +56,6 @@ func StdStatsData(dir *datafs.Data, tsr *tensor.Indexed) { return ix.Table // dt } - // DescAll returns a table of standard descriptive stats for // all numeric columns in given table, operating over all non-Null, non-NaN elements // in each column. @@ -105,6 +106,3 @@ func DescAll(ix *table.Indexed) *table.Table { */ return ix.Table // dt } - - - diff --git a/tensor/indexed.go b/tensor/indexed.go index a54654876e..7f50b71f20 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -20,7 +20,7 @@ import ( // This is the universal representation of a homogenous data type in the // [tensor] package framework, from scalar to vector, matrix, and beyond, // because it can efficiently represent any kind of element with sufficient -// flexibility to enable a huge range of computations to be elegantly expressed. +// flexibility to enable a full range of computations to be elegantly expressed. // For example, sorting and filtering a tensor only requires // updating the indexes while doing nothing to the Tensor itself. // To produce a new [Tensor] that has its raw data actually organized according @@ -35,16 +35,18 @@ type Indexed struct { //types:add Tensor Tensor // Indexes are the indexes into Tensor rows. + // Only set if order is different from default sequential order. + // Use the Index() method for nil-aware logic. Indexes []int } // NewIndexed returns a new Indexed based on given tensor. // If a list of indexes is passed, then our indexes are initialized -// as a copy of those. This is used e.g. from a Indext Table column. -// Otherwise it is initialized with sequential indexes. +// as a copy of those. This is used e.g. from a Indexed Table column. +// Otherwise it is initialized with default sequential indexes. func NewIndexed(tsr Tensor, idxs ...[]int) *Indexed { ix := &Indexed{} - if len(idxs) == 1 { + if len(idxs) == 1 { // indexes were passed ix.Tensor = tsr ix.Indexes = slices.Clone(idxs[0]) } else { @@ -59,15 +61,27 @@ func (ix *Indexed) SetTensor(tsr Tensor) { ix.Sequential() } -// Len returns the length of the index list +// Index returns the actual index into underlying tensor based on given +// index value. If Indexes == nil, index is passed through. +func (ix *Indexed) Index(idx int) int { + if ix.Indexes == nil { + return idx + } + return ix.Indexes[idx] +} + +// Len returns the length of the index list or number of outer rows dimension. func (ix *Indexed) Len() int { + if ix.Indexes == nil { + return ix.Tensor.DimSize(0) + } return len(ix.Indexes) } // DeleteInvalid deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. func (ix *Indexed) DeleteInvalid() { - if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { ix.Indexes = nil return } @@ -79,12 +93,30 @@ func (ix *Indexed) DeleteInvalid() { } } -// Sequential sets indexes to sequential row-wise indexes into tensor. +// Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor. func (ix *Indexed) Sequential() { //types:add + if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { + ix.Indexes = nil + return + } + ix.Indexes = make([]int, ix.Tensor.DimSize(0)) + for i := range ix.Indexes { + ix.Indexes[i] = i + } +} + +// IndexesNeeded is called prior to an operation that needs actual indexes, +// e.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise +// current indexes are left as is. Use Sequential, then IndexesNeeded to ensure +// all rows are represented. +func (ix *Indexed) IndexesNeeded() { //types:add if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return } + if ix.Indexes != nil { + return + } ix.Indexes = make([]int, ix.Tensor.DimSize(0)) for i := range ix.Indexes { ix.Indexes[i] = i @@ -101,6 +133,7 @@ func (ix *Indexed) ExcludeMissing1D() { //types:add if ix.Tensor.NumDims() > 1 { return } + ix.IndexesNeeded() ni := ix.Len() for i := ni - 1; i >= 0; i-- { if math.IsNaN(ix.Tensor.Float1D(ix.Indexes[i])) { @@ -109,7 +142,7 @@ func (ix *Indexed) ExcludeMissing1D() { //types:add } } -// Permuted sets indexes to a permuted order -- if indexes already exist +// Permuted sets indexes to a permuted order. If indexes already exist // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated func (ix *Indexed) Permuted() { @@ -117,7 +150,7 @@ func (ix *Indexed) Permuted() { ix.Indexes = nil return } - if len(ix.Indexes) == 0 { + if ix.Indexes == nil { ix.Indexes = rand.Perm(ix.Tensor.DimSize(0)) } else { rand.Shuffle(len(ix.Indexes), func(i, j int) { @@ -149,6 +182,7 @@ func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) error { if ix.Tensor.NumDims() > 1 { return errors.New("tensor Sorting is only for 1D tensors") } + ix.IndexesNeeded() slices.SortFunc(ix.Indexes, func(a, b int) int { return cmp(ix.Tensor, ix.Indexes[a], ix.Indexes[b]) }) @@ -159,6 +193,9 @@ func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) error { // numerical order, producing the native ordering, while preserving // any filtering that might have occurred. func (ix *Indexed) SortIndexes() { + if ix.Indexes == nil { + return + } sort.Ints(ix.Indexes) } @@ -182,9 +219,6 @@ func CompareNumbers(a, b float64, ascending bool) int { // Sort does default alpha or numeric sort of 1D tensor based on data type. // Returns an error if called on a higher-dimensional tensor. func (ix *Indexed) Sort(ascending bool) error { - if ix.Tensor.NumDims() > 1 { - return errors.New("tensor Sorting is only for 1D tensors") - } if ix.Tensor.IsString() { ix.SortFunc(func(tsr Tensor, i, j int) int { return CompareStrings(tsr.String1D(i), tsr.String1D(j), ascending) @@ -204,19 +238,21 @@ func (ix *Indexed) Sort(ascending bool) error { // number when a > b and zero when a == b. // It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { +func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) error { + if ix.Tensor.NumDims() > 1 { + return errors.New("tensor Sorting is only for 1D tensors") + } + ix.IndexesNeeded() slices.SortStableFunc(ix.Indexes, func(a, b int) int { return cmp(ix.Tensor, ix.Indexes[a], ix.Indexes[b]) }) + return nil } // SortStable does default alpha or numeric stable sort // of 1D tensor based on data type. // Returns an error if called on a higher-dimensional tensor. func (ix *Indexed) SortStable(ascending bool) error { - if ix.Tensor.NumDims() > 1 { - return errors.New("tensor Sorting is only for 1D tensors") - } if ix.Tensor.IsString() { ix.SortStableFunc(func(tsr Tensor, i, j int) int { return CompareStrings(tsr.String1D(i), tsr.String1D(j), ascending) @@ -238,6 +274,7 @@ type FilterFunc func(tsr Tensor, row int) bool // The Filter function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { + ix.IndexesNeeded() sz := len(ix.Indexes) for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering if !filterer(ix.Tensor, ix.Indexes[i]) { // delete @@ -247,16 +284,32 @@ func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { } // NewTensor returns a new tensor with column data organized according to -// the indexes. +// the Indexes. If Indexes are nil, a clone of the current tensor is returned +// but this function is only sensible if there is an indexed view in place. func (ix *Indexed) NewTensor() Tensor { - rows := len(ix.Indexes) nt := ix.Tensor.Clone() + if ix.Indexes == nil { + return nt + } + rows := len(ix.Indexes) nt.SetNumRows(rows) + _, cells := ix.Tensor.RowCellSize() + str := ix.Tensor.IsString() + for r := range rows { + for c := range cells { + if str { + nt.SetStringRowCell(r, c, ix.StringRowCell(r, c)) + } else { + nt.SetFloatRowCell(r, c, ix.FloatRowCell(r, c)) + } + } + } return nt } // Clone returns a copy of the current Indexed view into the same -// underlying Tensor as the source, with its own index memory. +// underlying Tensor as the source, with its own copy of Indexes +// if they were present in source. func (ix *Indexed) Clone() *Indexed { nix := &Indexed{} nix.CopyFrom(ix) @@ -266,15 +319,21 @@ func (ix *Indexed) Clone() *Indexed { // CopyFrom copies from given other Indexed (we have our own unique copy of indexes). func (ix *Indexed) CopyFrom(oix *Indexed) { ix.Tensor = oix.Tensor - ix.Indexes = slices.Clone(oix.Indexes) + if oix.Indexes == nil { + ix.Indexes = nil + } else { + ix.Indexes = slices.Clone(oix.Indexes) + } } // AddRows adds n rows to end of underlying Tensor, and to the indexes in this view func (ix *Indexed) AddRows(n int) { //types:add stidx := ix.Tensor.DimSize(0) ix.Tensor.SetNumRows(stidx + n) - for i := stidx; i < stidx+n; i++ { - ix.Indexes = append(ix.Indexes, i) + if ix.Indexes != nil { + for i := stidx; i < stidx+n; i++ { + ix.Indexes = append(ix.Indexes, i) + } } } @@ -283,26 +342,40 @@ func (ix *Indexed) AddRows(n int) { //types:add func (ix *Indexed) InsertRows(at, n int) { stidx := ix.Tensor.DimSize(0) ix.Tensor.SetNumRows(stidx + n) - nw := make([]int, n, n+len(ix.Indexes)-at) - for i := 0; i < n; i++ { - nw[i] = stidx + i + if ix.Indexes != nil { + nw := make([]int, n, n+len(ix.Indexes)-at) + for i := 0; i < n; i++ { + nw[i] = stidx + i + } + ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) } - ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) } // DeleteRows deletes n rows of indexes starting at given index in the list of indexes func (ix *Indexed) DeleteRows(at, n int) { + if ix.Indexes == nil { + return + } ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) } // Swap switches the indexes for i and j func (ix *Indexed) Swap(i, j int) { + if ix.Indexes == nil { + return + } ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] } +/////////////////////////////////////////////// +// Indexed access + // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. func (ix *Indexed) Float(i []int) float64 { + if ix.Indexes == nil { + return ix.Tensor.Float(i) + } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] return ix.Tensor.Float(ic) @@ -311,6 +384,9 @@ func (ix *Indexed) Float(i []int) float64 { // SetFloat sets the value of given index as a float64 // The first index value is indirected through the [Indexes]. func (ix *Indexed) SetFloat(i []int, val float64) { + if ix.Indexes == nil { + ix.Tensor.SetFloat(i, val) + } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] ix.Tensor.SetFloat(ic, val) @@ -321,7 +397,7 @@ func (ix *Indexed) SetFloat(i []int, val float64) { // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) FloatRowCell(row, cell int) float64 { - return ix.Tensor.FloatRowCell(ix.Indexes[row], cell) + return ix.Tensor.FloatRowCell(ix.Index(row), cell) } // SetFloatRowCell sets the value at given row and cell, @@ -329,5 +405,43 @@ func (ix *Indexed) FloatRowCell(row, cell int) float64 { // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetFloatRowCell(row, cell int, val float64) { - ix.Tensor.SetFloatRowCell(ix.Indexes[row], cell, val) + ix.Tensor.SetFloatRowCell(ix.Index(row), cell, val) +} + +// StringValue returns the value of given index as a string. +// The first index value is indirected through the indexes. +func (ix *Indexed) StringValue(i []int) string { + if ix.Indexes == nil { + return ix.Tensor.StringValue(i) + } + ic := slices.Clone(i) + ic[0] = ix.Indexes[ic[0]] + return ix.Tensor.StringValue(ic) +} + +// SetString sets the value of given index as a string +// The first index value is indirected through the [Indexes]. +func (ix *Indexed) SetString(i []int, val string) { + if ix.Indexes == nil { + ix.Tensor.SetString(i, val) + } + ic := slices.Clone(i) + ic[0] = ix.Indexes[ic[0]] + ix.Tensor.SetString(ic, val) +} + +// StringRowCell returns the value at given row and cell, +// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Indexes]. +// This is the preferred interface for all Indexed operations. +func (ix *Indexed) StringRowCell(row, cell int) string { + return ix.Tensor.StringRowCell(ix.Index(row), cell) +} + +// SetStringRowCell sets the value at given row and cell, +// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Indexes]. +// This is the preferred interface for all Indexed operations. +func (ix *Indexed) SetStringRowCell(row, cell int, val string) { + ix.Tensor.SetStringRowCell(ix.Index(row), cell, val) } diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 3a09f6d033..f100a0460f 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -1,6 +1,6 @@ # metric -`metric` provides various similarity / distance metrics for comparing floating-point vectors. All functions have 32 and 64 bit variants, and skip NaN's (often used for missing) and will panic if the lengths of the two slices are unequal (no error return). +`metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Indexed` standard data representation. The signatures of all such metric functions are identical, captured as types: `metric.Func32` and `metric.Func64` so that other functions that use a metric can take a pointer to any such function. diff --git a/tensor/stats/metric/enumgen.go b/tensor/stats/metric/enumgen.go index e99276abeb..e1743df0cc 100644 --- a/tensor/stats/metric/enumgen.go +++ b/tensor/stats/metric/enumgen.go @@ -6,45 +6,43 @@ import ( "cogentcore.org/core/enums" ) -var _StdMetricsValues = []StdMetrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} +var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} -// StdMetricsN is the highest valid value for type StdMetrics, plus one. -const StdMetricsN StdMetrics = 13 +// MetricsN is the highest valid value for type Metrics, plus one. +const MetricsN Metrics = 13 -var _StdMetricsValueMap = map[string]StdMetrics{`Euclidean`: 0, `SumSquares`: 1, `Abs`: 2, `Hamming`: 3, `EuclideanBinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `InnerProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} +var _MetricsValueMap = map[string]Metrics{`Euclidean`: 0, `SumSquares`: 1, `Abs`: 2, `Hamming`: 3, `EuclideanBinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `InnerProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} -var _StdMetricsDescMap = map[StdMetrics]string{0: ``, 1: ``, 2: ``, 3: ``, 4: ``, 5: ``, 6: `InvCosine is 1-Cosine -- useful to convert into an Increasing metric`, 7: `InvCorrelation is 1-Correlation -- useful to convert into an Increasing metric`, 8: ``, 9: `Everything below here is !Increasing -- larger = closer, not farther`, 10: ``, 11: ``, 12: ``} +var _MetricsDescMap = map[Metrics]string{0: ``, 1: ``, 2: ``, 3: ``, 4: ``, 5: ``, 6: `InvCosine is 1-Cosine -- useful to convert into an Increasing metric`, 7: `InvCorrelation is 1-Correlation -- useful to convert into an Increasing metric`, 8: ``, 9: `Everything below here is !Increasing -- larger = closer, not farther`, 10: ``, 11: ``, 12: ``} -var _StdMetricsMap = map[StdMetrics]string{0: `Euclidean`, 1: `SumSquares`, 2: `Abs`, 3: `Hamming`, 4: `EuclideanBinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `InnerProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} +var _MetricsMap = map[Metrics]string{0: `Euclidean`, 1: `SumSquares`, 2: `Abs`, 3: `Hamming`, 4: `EuclideanBinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `InnerProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} -// String returns the string representation of this StdMetrics value. -func (i StdMetrics) String() string { return enums.String(i, _StdMetricsMap) } +// String returns the string representation of this Metrics value. +func (i Metrics) String() string { return enums.String(i, _MetricsMap) } -// SetString sets the StdMetrics value from its string representation, +// SetString sets the Metrics value from its string representation, // and returns an error if the string is invalid. -func (i *StdMetrics) SetString(s string) error { - return enums.SetString(i, s, _StdMetricsValueMap, "StdMetrics") +func (i *Metrics) SetString(s string) error { + return enums.SetString(i, s, _MetricsValueMap, "Metrics") } -// Int64 returns the StdMetrics value as an int64. -func (i StdMetrics) Int64() int64 { return int64(i) } +// Int64 returns the Metrics value as an int64. +func (i Metrics) Int64() int64 { return int64(i) } -// SetInt64 sets the StdMetrics value from an int64. -func (i *StdMetrics) SetInt64(in int64) { *i = StdMetrics(in) } +// SetInt64 sets the Metrics value from an int64. +func (i *Metrics) SetInt64(in int64) { *i = Metrics(in) } -// Desc returns the description of the StdMetrics value. -func (i StdMetrics) Desc() string { return enums.Desc(i, _StdMetricsDescMap) } +// Desc returns the description of the Metrics value. +func (i Metrics) Desc() string { return enums.Desc(i, _MetricsDescMap) } -// StdMetricsValues returns all possible values for the type StdMetrics. -func StdMetricsValues() []StdMetrics { return _StdMetricsValues } +// MetricsValues returns all possible values for the type Metrics. +func MetricsValues() []Metrics { return _MetricsValues } -// Values returns all possible values for the type StdMetrics. -func (i StdMetrics) Values() []enums.Enum { return enums.Values(_StdMetricsValues) } +// Values returns all possible values for the type Metrics. +func (i Metrics) Values() []enums.Enum { return enums.Values(_MetricsValues) } // MarshalText implements the [encoding.TextMarshaler] interface. -func (i StdMetrics) MarshalText() ([]byte, error) { return []byte(i.String()), nil } +func (i Metrics) MarshalText() ([]byte, error) { return []byte(i.String()), nil } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. -func (i *StdMetrics) UnmarshalText(text []byte) error { - return enums.UnmarshalText(i, text, "StdMetrics") -} +func (i *Metrics) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Metrics") } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go new file mode 100644 index 0000000000..9e1d8d5997 --- /dev/null +++ b/tensor/stats/metric/funcs.go @@ -0,0 +1,219 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metric + +import ( + "math" + + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" +) + +// MetricFunc is the function signature for a metric function, +// where the output has the same shape as the inputs but with +// the outer-most row dimension size of 1, and contains +// the metric value(s) for the "cells" in higher-dimensional tensors, +// and a single scalar value for a 1D input tensor. +// All metric functions skip over NaN's, as a missing value. +// Metric functions cannot be computed in parallel, +// e.g., using VectorizeThreaded or GPU, due to shared writing +// to the same output values. Special implementations are required +// if that is needed. +type MetricFunc func(a, b, out *tensor.Indexed) + +// InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. +// See [MetricFunc] for general information. +func InnerProductFunc(a, b, out *tensor.Indexed) { + VectorizeOut64(InnerProductVecFunc, a, b, out) +} + +// SumSquaresFunc computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func SumSquaresFunc(a, b, out *tensor.Indexed) { + SumSquaresFuncOut64(a, b, out) +} + +// SumSquaresFuncOut64 computes the sum of squares differences between tensor values, +// and returns the Float64 output values for use in subsequent computations. +func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { + scale64, ss64 := Vectorize2Out64(SumSquaresVecFunc, a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + scale := scale64.Tensor.Float1D(i) + ss := ss64.Tensor.Float1D(i) + v := 0.0 + if math.IsInf(scale, 1) { + v = math.Inf(1) + } else { + v = scale * scale * ss + } + scale64.Tensor.SetFloat1D(i, v) + out.Tensor.SetFloat1D(i, v) + } + return scale64 +} + +// EuclideanFunc computes the Euclidean square root of the sum of squares +// differences between tensor values. +func EuclideanFunc(a, b, out *tensor.Indexed) { + scale64, ss64 := Vectorize2Out64(SumSquaresVecFunc, a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + scale := scale64.Tensor.Float1D(i) + ss := ss64.Tensor.Float1D(i) + v := 0.0 + if math.IsInf(scale, 1) { + v = math.Inf(1) + } else { + v = scale * math.Sqrt(ss) + } + scale64.Tensor.SetFloat1D(i, v) + out.Tensor.SetFloat1D(i, v) + } +} + +// EuclideanBinTolFunc computes the Euclidean square root of the sum of squares +// differences between tensor values, with binary tolerance: +// differences < 0.5 are thresholded to 0. +func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { + scale64, ss64 := Vectorize2Out64(SumSquaresBinTolVecFunc, a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + scale := scale64.Tensor.Float1D(i) + ss := ss64.Tensor.Float1D(i) + v := 0.0 + if math.IsInf(scale, 1) { + v = math.Inf(1) + } else { + v = scale * math.Sqrt(ss) + } + scale64.Tensor.SetFloat1D(i, v) + out.Tensor.SetFloat1D(i, v) + } +} + +// SumSquaresBinTolFunc computes the sum of squares differences between tensor values, +// with binary tolerance: differences < 0.5 are thresholded to 0. +func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) *tensor.Indexed { + scale64, ss64 := Vectorize2Out64(SumSquaresBinTolVecFunc, a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + scale := scale64.Tensor.Float1D(i) + ss := ss64.Tensor.Float1D(i) + v := 0.0 + if math.IsInf(scale, 1) { + v = math.Inf(1) + } else { + v = scale * scale * ss + } + scale64.Tensor.SetFloat1D(i, v) + out.Tensor.SetFloat1D(i, v) + } + return scale64 +} + +// CovarianceFunc computes the covariance between two vectors, +// i.e., the mean of the co-product of each vector element minus +// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. +func CovarianceFunc(a, b, out *tensor.Indexed) { + amean, acount := stats.MeanFuncOut64(a, out) + bmean, _ := stats.MeanFuncOut64(b, out) + cov64 := VectorizeOut64(CovarianceVecFunc, a, b, amean, bmean, out) + nsub := out.Tensor.Len() + for i := range nsub { + c := acount.Tensor.Float1D(i) + if c == 0 { + continue + } + cov64.Tensor.SetFloat1D(i, cov64.Tensor.Float1D(i)/c) + out.Tensor.SetFloat1D(i, cov64.Tensor.Float1D(i)) + } +} + +// CorrelationFunc computes the correlation between two vectors, +// in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized +// vectors. +func CorrelationFunc(a, b, out *tensor.Indexed) { + CorrelationFuncOut64(a, b, out) +} + +// CorrelationFuncOut64 computes the correlation between two vectors, +// in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized +// vectors. +func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { + amean, _ := stats.MeanFuncOut64(a, out) + bmean, _ := stats.MeanFuncOut64(b, out) + ss64, avar64, bvar64 := Vectorize2in3Out64(CorrelationVecFunc, a, b, amean, bmean, out) + nsub := out.Tensor.Len() + for i := range nsub { + ss := ss64.Tensor.Float1D(i) + vp := math.Sqrt(avar64.Tensor.Float1D(i) * bvar64.Tensor.Float1D(i)) + if vp > 0 { + ss /= vp + } + ss64.Tensor.SetFloat1D(i, ss) + out.Tensor.SetFloat1D(i, ss) + } + return ss64 +} + +// InvCorrelationFunc computes 1 minus the correlation between two vectors, +// in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized +// vectors. This is useful for a difference measure instead of similarity. +func InvCorrelationFunc(a, b, out *tensor.Indexed) { + cor64 := CorrelationFuncOut64(a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + cor := cor64.Tensor.Float1D(i) + out.Tensor.SetFloat1D(i, 1-cor) + } +} + +// CosineFunc computes the correlation between two vectors, +// in range (-1..1) as the normalized inner product: +// inner product / sqrt(ssA * ssB). +func CosineFunc(a, b, out *tensor.Indexed) { + CosineFuncOut64(a, b, out) +} + +// CosineFuncOut64 computes the correlation between two vectors, +// in range (-1..1) as the normalized inner product: +// inner product / sqrt(ssA * ssB). +func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { + ss64, avar64, bvar64 := Vectorize3Out64(CosineVecFunc, a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + ss := ss64.Tensor.Float1D(i) + vp := math.Sqrt(avar64.Tensor.Float1D(i) * bvar64.Tensor.Float1D(i)) + if vp > 0 { + ss /= vp + } + ss64.Tensor.SetFloat1D(i, ss) + out.Tensor.SetFloat1D(i, ss) + } + return ss64 +} + +// InvCosineFunc computes 1 minus the cosine between two vectors, +// in range (-1..1) as the normalized inner product: +// inner product / sqrt(ssA * ssB). +// This is useful for a difference measure instead of similarity. +func InvCosineFunc(a, b, out *tensor.Indexed) { + cos64 := CosineFuncOut64(a, b, out) + nsub := out.Tensor.Len() + for i := range nsub { + cos := cos64.Tensor.Float1D(i) + out.Tensor.SetFloat1D(i, 1-cos) + } +} diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 6cd11ac98d..c597e0184f 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -8,76 +8,60 @@ import ( "math" "testing" - "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" ) func TestAll(t *testing.T) { a64 := []float64{.5, .2, .1, .7, math.NaN(), .5} - b64 := []float64{.2, .5, .1, .7, 0, .2} - - a32 := []float32{.5, .2, .1, .7, math32.NaN(), .5} - b32 := []float32{.2, .5, .1, .7, 0, .2} - - ss := SumSquares64(a64, b64) - if ss != 0.27 { - t.Errorf("SumSquares64: %g\n", ss) - } - ss32 := SumSquares32(a32, b32) - if ss32 != float32(ss) { - t.Errorf("SumSquares32: %g\n", ss32) - } - - ec := Euclidean64(a64, b64) - if math.Abs(ec-math.Sqrt(0.27)) > 1.0e-10 { - t.Errorf("Euclidean64: %g vs. %g\n", ec, math.Sqrt(0.27)) - } - ec32 := Euclidean32(a32, b32) - if ec32 != float32(ec) { - t.Errorf("Euclidean32: %g\n", ec32) - } - - cv := Covariance64(a64, b64) - if cv != 0.023999999999999994 { - t.Errorf("Covariance64: %g\n", cv) - } - cv32 := Covariance32(a32, b32) - if cv32 != float32(cv) { - t.Errorf("Covariance32: %g\n", cv32) - } - - cr := Correlation64(a64, b64) - if cr != 0.47311118871909136 { - t.Errorf("Correlation64: %g\n", cr) - } - cr32 := Correlation32(a32, b32) - if cr32 != 0.47311115 { - t.Errorf("Correlation32: %g\n", cr32) - } - - cs := Cosine64(a64, b64) - if cs != 0.861061697819235 { - t.Errorf("Cosine64: %g\n", cs) - } - cs32 := Cosine32(a32, b32) - if cs32 != 0.86106175 { - t.Errorf("Cosine32: %g\n", cs32) - } - - ab := Abs64(a64, b64) - if ab != 0.8999999999999999 { - t.Errorf("Abs64: %g\n", ab) - } - ab32 := Abs32(a32, b32) - if ab32 != 0.90000004 { - t.Errorf("Abs32: %g\n", ab32) - } - - hm := Hamming64(a64, b64) - if hm != 3 { - t.Errorf("Hamming64: %g\n", hm) - } - hm32 := Hamming32(a32, b32) - if hm32 != 3 { - t.Errorf("Hamming32: %g\n", hm32) - } + b64 := []float64{.2, .9, .1, .7, 0, .2} + + results := []float64{math.Sqrt(0.67), 0.67, 0.9, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 0, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} + + atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64)) + btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64)) + out := tensor.NewFloat64([]int{1}) + oix := tensor.NewIndexed(out) + + EuclideanFunc(atsr, btsr, oix) + assert.InDelta(t, results[Euclidean], out.Values[0], 1.0e-8) + + SumSquaresFunc(atsr, btsr, oix) + assert.InDelta(t, results[SumSquares], out.Values[0], 1.0e-8) + + EuclideanBinTolFunc(atsr, btsr, oix) + assert.Equal(t, results[EuclideanBinTol], out.Values[0]) + + SumSquaresBinTolFunc(atsr, btsr, oix) + assert.InDelta(t, results[SumSquaresBinTol], out.Values[0], 1.0e-8) + + CovarianceFunc(atsr, btsr, oix) + assert.InDelta(t, results[Covariance], out.Values[0], 1.0e-8) + + CorrelationFunc(atsr, btsr, oix) + assert.Equal(t, results[Correlation], out.Values[0]) + + InvCorrelationFunc(atsr, btsr, oix) + assert.Equal(t, results[InvCorrelation], out.Values[0]) + + CosineFunc(atsr, btsr, oix) + assert.Equal(t, results[Cosine], out.Values[0]) + + InvCosineFunc(atsr, btsr, oix) + assert.Equal(t, results[InvCosine], out.Values[0]) + + InnerProductFunc(atsr, btsr, oix) + assert.Equal(t, results[InnerProduct], out.Values[0]) + + /* + ab := Abs64(a64, b64) + if ab != 0.8999999999999999 { + t.Errorf("Abs64: %g\n", ab) + } + + hm := Hamming64(a64, b64) + if hm != 3 { + t.Errorf("Hamming64: %g\n", hm) + } + */ } diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 876be155c8..0d3248bd0a 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -6,17 +6,44 @@ package metric -// Func32 is a distance / similarity metric operating on slices of float32 numbers -type Func32 func(a, b []float32) float32 +import ( + "fmt" -// Func64 is a distance / similarity metric operating on slices of float64 numbers -type Func64 func(a, b []float64) float64 + "cogentcore.org/core/tensor" +) + +// Funcs is a registry of named metric functions, +// which can then be called by standard enum or +// string name for custom functions. +var Funcs map[string]MetricFunc + +func init() { + Funcs = make(map[string]MetricFunc) +} + +// Standard calls a standard Metrics enum function on given tensors. +// Output results are in the out tensor. +func Standard(metric Metrics, a, b, out *tensor.Indexed) { + Funcs[metric.String()](a, b, out) +} + +// Call calls a registered stats function on given tensors. +// Output results are in the out tensor. Returns an +// error if name not found. +func Call(name string, a, b, out *tensor.Indexed) error { + f, ok := Funcs[name] + if !ok { + return fmt.Errorf("metric.Call: function %q not registered", name) + } + f(a, b, out) + return nil +} -// StdMetrics are standard metric functions -type StdMetrics int32 //enums:enum +// Metrics are standard metric functions +type Metrics int32 //enums:enum const ( - Euclidean StdMetrics = iota + Euclidean Metrics = iota SumSquares Abs Hamming @@ -43,75 +70,9 @@ const ( // values increase as a function of distance (e.g., Euclidean) // and false if metric values decrease as a function of distance // (e.g., Cosine, Correlation) -func Increasing(std StdMetrics) bool { - if std >= InnerProduct { +func (m Metrics) Increasing() bool { + if m >= InnerProduct { return false } return true } - -// StdFunc32 returns a standard metric function as specified -func StdFunc32(std StdMetrics) Func32 { - switch std { - case Euclidean: - return Euclidean32 - case SumSquares: - return SumSquares32 - case Abs: - return Abs32 - case Hamming: - return Hamming32 - case EuclideanBinTol: - return EuclideanBinTol32 - case SumSquaresBinTol: - return SumSquaresBinTol32 - case InvCorrelation: - return InvCorrelation32 - case InvCosine: - return InvCosine32 - case CrossEntropy: - return CrossEntropy32 - case InnerProduct: - return InnerProduct32 - case Covariance: - return Covariance32 - case Correlation: - return Correlation32 - case Cosine: - return Cosine32 - } - return nil -} - -// StdFunc64 returns a standard metric function as specified -func StdFunc64(std StdMetrics) Func64 { - switch std { - case Euclidean: - return Euclidean64 - case SumSquares: - return SumSquares64 - case Abs: - return Abs64 - case Hamming: - return Hamming64 - case EuclideanBinTol: - return EuclideanBinTol64 - case SumSquaresBinTol: - return SumSquaresBinTol64 - case InvCorrelation: - return InvCorrelation64 - case InvCosine: - return InvCosine64 - case CrossEntropy: - return CrossEntropy64 - case InnerProduct: - return InnerProduct64 - case Covariance: - return Covariance64 - case Correlation: - return Correlation64 - case Cosine: - return Cosine64 - } - return nil -} diff --git a/tensor/stats/metric/squares.go b/tensor/stats/metric/squares.go index 61576ba56a..71bb0dd1f0 100644 --- a/tensor/stats/metric/squares.go +++ b/tensor/stats/metric/squares.go @@ -4,538 +4,12 @@ package metric -import ( - "math" - - "cogentcore.org/core/math32" - "cogentcore.org/core/tensor/stats/stats" -) - -/////////////////////////////////////////// -// SumSquares - -// SumSquares32 computes the sum-of-squares distance between two vectors. -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -func SumSquares32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float32 = 0 - sumSquares float32 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - absxi := math32.Abs(av - bv) - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * scale * sumSquares -} - -// SumSquares64 computes the sum-of-squares distance between two vectors. -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -func SumSquares64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float64 = 0 - sumSquares float64 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math.IsNaN(av) || math.IsNaN(bv) { - continue - } - absxi := math.Abs(av - bv) - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * scale * sumSquares -} - -/////////////////////////////////////////// -// SumSquaresBinTol - -// SumSquaresBinTol32 computes the sum-of-squares distance between two vectors. -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -// BinTol version uses binary tolerance for 0-1 valued-vectors where -// abs diff < .5 counts as 0 error (i.e., closer than not). -func SumSquaresBinTol32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float32 = 0 - sumSquares float32 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - absxi := math32.Abs(av - bv) - if absxi < 0.5 { - continue - } - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * scale * sumSquares -} - -// SumSquaresBinTol64 computes the sum-of-squares distance between two vectors. -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -// BinTol version uses binary tolerance for 0-1 valued-vectors where -// abs diff < .5 counts as 0 error (i.e., closer than not). -func SumSquaresBinTol64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float64 = 0 - sumSquares float64 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math.IsNaN(av) || math.IsNaN(bv) { - continue - } - absxi := math.Abs(av - bv) - if absxi < 0.5 { - continue - } - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * scale * sumSquares -} - -/////////////////////////////////////////// -// Euclidean - -// Euclidean32 computes the square-root of sum-of-squares distance -// between two vectors (aka the L2 norm). -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -func Euclidean32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float32 = 0 - sumSquares float32 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - absxi := math32.Abs(av - bv) - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * math32.Sqrt(sumSquares) -} - -// Euclidean64 computes the square-root of sum-of-squares distance -// between two vectors (aka the L2 norm). -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -func Euclidean64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float64 = 0 - sumSquares float64 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math.IsNaN(av) || math.IsNaN(bv) { - continue - } - absxi := math.Abs(av - bv) - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * math.Sqrt(sumSquares) -} - -/////////////////////////////////////////// -// EuclideanBinTol - -// EuclideanBinTol32 computes the square-root of sum-of-squares distance -// between two vectors (aka the L2 norm). -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -// BinTol version uses binary tolerance for 0-1 valued-vectors where -// abs diff < .5 counts as 0 error (i.e., closer than not). -func EuclideanBinTol32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math32.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float32 = 0 - sumSquares float32 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - absxi := math32.Abs(av - bv) - if absxi < 0.5 { - continue - } - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math32.IsInf(scale, 1) { - return math32.Inf(1) - } - return scale * math32.Sqrt(sumSquares) -} - -// EuclideanBinTol64 computes the square-root of sum-of-squares distance -// between two vectors (aka the L2 norm). -// Skips NaN's and panics if lengths are not equal. -// Uses optimized algorithm from BLAS that avoids numerical overflow. -// BinTol version uses binary tolerance for 0-1 valued-vectors where -// abs diff < .5 counts as 0 error (i.e., closer than not). -func EuclideanBinTol64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0] - b[0]) - } - return 0 - } - var ( - scale float64 = 0 - sumSquares float64 = 1 - ) - for i, av := range a { - bv := b[i] - if av == bv || math.IsNaN(av) || math.IsNaN(bv) { - continue - } - absxi := math.Abs(av - bv) - if absxi < 0.5 { - continue - } - if scale < absxi { - sumSquares = 1 + sumSquares*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - sumSquares = sumSquares + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * math.Sqrt(sumSquares) -} - -/////////////////////////////////////////// -// Covariance - -// Covariance32 computes the mean of the co-product of each vector element minus -// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))] -// Skips NaN's and panics if lengths are not equal. -func Covariance32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - am := stats.Mean32(a) - bm := stats.Mean32(b) - n := 0 - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - ss += (av - am) * (bv - bm) - n++ - } - if n > 0 { - ss /= float32(n) - } - return ss -} - -// Covariance64 computes the mean of the co-product of each vector element minus -// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))] -// Skips NaN's and panics if lengths are not equal. -func Covariance64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - am := stats.Mean64(a) - bm := stats.Mean64(b) - n := 0 - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - ss += (av - am) * (bv - bm) - n++ - } - if n > 0 { - ss /= float64(n) - } - return ss -} - -/////////////////////////////////////////// -// Correlation - -// Correlation32 computes the vector similarity in range (-1..1) as the -// mean of the co-product of each vector element minus the mean of that vector, -// normalized by the product of their standard deviations: -// cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance) -- equivalent to the cosine of mean-normalized -// vectors. -// Skips NaN's and panics if lengths are not equal. -func Correlation32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - am := stats.Mean32(a) - bm := stats.Mean32(b) - var avar, bvar float32 - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - ad := av - am - bd := bv - bm - ss += ad * bd // between - avar += ad * ad // within - bvar += bd * bd - } - vp := math32.Sqrt(avar * bvar) - if vp > 0 { - ss /= vp - } - return ss -} - -// Correlation64 computes the vector similarity in range (-1..1) as the -// mean of the co-product of each vector element minus the mean of that vector, -// normalized by the product of their standard deviations: -// cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance) -- equivalent to the cosine of mean-normalized -// vectors. -// Skips NaN's and panics if lengths are not equal. -func Correlation64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - am := stats.Mean64(a) - bm := stats.Mean64(b) - var avar, bvar float64 - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - ad := av - am - bd := bv - bm - ss += ad * bd // between - avar += ad * ad // within - bvar += bd * bd - } - vp := math.Sqrt(avar * bvar) - if vp > 0 { - ss /= vp - } - return ss -} - -/////////////////////////////////////////// -// InnerProduct - -// InnerProduct32 computes the sum of the element-wise product of the two vectors. -// Skips NaN's and panics if lengths are not equal. -func InnerProduct32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - ss += av * bv - } - return ss -} - -// InnerProduct64 computes the mean of the co-product of each vector element minus -// the mean of that vector, normalized by the product of their standard deviations: -// cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance) -- equivalent to the cosine of mean-normalized -// vectors. -// Skips NaN's and panics if lengths are not equal. -func InnerProduct64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - ss += av * bv - } - return ss -} +/* /////////////////////////////////////////// // Cosine -// Cosine32 computes the cosine of the angle between two vectors (-1..1), -// as the normalized inner product: inner product / sqrt(ssA * ssB). -// If vectors are mean-normalized = Correlation. -// Skips NaN's and panics if lengths are not equal. -func Cosine32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - var ass, bss float32 - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - ss += av * bv // between - ass += av * av // within - bss += bv * bv - } - vp := math32.Sqrt(ass * bss) - if vp > 0 { - ss /= vp - } - return ss -} - -// Cosine32 computes the cosine of the angle between two vectors (-1..1), +// Cosine64 computes the cosine of the angle between two vectors (-1..1), // as the normalized inner product: inner product / sqrt(ssA * ssB). // If vectors are mean-normalized = Correlation. // Skips NaN's and panics if lengths are not equal. @@ -564,14 +38,6 @@ func Cosine64(a, b []float64) float64 { /////////////////////////////////////////// // InvCosine -// InvCosine32 computes 1 - cosine of the angle between two vectors (-1..1), -// as the normalized inner product: inner product / sqrt(ssA * ssB). -// If vectors are mean-normalized = Correlation. -// Skips NaN's and panics if lengths are not equal. -func InvCosine32(a, b []float32) float32 { - return 1 - Cosine32(a, b) -} - // InvCosine32 computes 1 - cosine of the angle between two vectors (-1..1), // as the normalized inner product: inner product / sqrt(ssA * ssB). // If vectors are mean-normalized = Correlation. @@ -580,27 +46,5 @@ func InvCosine64(a, b []float64) float64 { return 1 - Cosine64(a, b) } -/////////////////////////////////////////// -// InvCorrelation - -// InvCorrelation32 computes 1 - the vector similarity in range (-1..1) as the -// mean of the co-product of each vector element minus the mean of that vector, -// normalized by the product of their standard deviations: -// cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance) -- equivalent to the cosine of mean-normalized -// vectors. -// Skips NaN's and panics if lengths are not equal. -func InvCorrelation32(a, b []float32) float32 { - return 1 - Correlation32(a, b) -} -// InvCorrelation64 computes 1 - the vector similarity in range (-1..1) as the -// mean of the co-product of each vector element minus the mean of that vector, -// normalized by the product of their standard deviations: -// cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance) -- equivalent to the cosine of mean-normalized -// vectors. -// Skips NaN's and panics if lengths are not equal. -func InvCorrelation64(a, b []float64) float64 { - return 1 - Correlation64(a, b) -} +*/ diff --git a/tensor/stats/metric/tensor.go b/tensor/stats/metric/tensor.go index 6d9ae20455..9e650e18b8 100644 --- a/tensor/stats/metric/tensor.go +++ b/tensor/stats/metric/tensor.go @@ -4,12 +4,7 @@ package metric -import ( - "math" - - "cogentcore.org/core/tensor" -) - +/* // ClosestRow32 returns the closest fit between probe pattern and patterns in // an tensor with float32 data where the outer-most dimension is assumed to be a row // (e.g., as a column in an table), using the given metric function, @@ -65,28 +60,4 @@ func ClosestRow64(probe tensor.Tensor, col tensor.Tensor, mfun Func64) (int, flo } return ci, minv } - -// ClosestRow32Py returns the closest fit between probe pattern and patterns in -// an tensor.Float32 where the outer-most dimension is assumed to be a row -// (e.g., as a column in an table), using the given metric function, -// *which must have the Increasing property* -- i.e., larger = further. -// returns the row and metric value for that row. -// Col cell sizes must match size of probe (panics if not). -// Py version is for Python, returns a slice with row, cor, takes std metric -func ClosestRow32Py(probe tensor.Tensor, col tensor.Tensor, std StdMetrics) []float32 { - row, cor := ClosestRow32(probe, col, StdFunc32(std)) - return []float32{float32(row), cor} -} - -// ClosestRow64Py returns the closest fit between probe pattern and patterns in -// an tensor.Tensor where the outer-most dimension is assumed to be a row -// (e.g., as a column in an table), using the given metric function, -// *which must have the Increasing property* -- i.e., larger = further. -// returns the row and metric value for that row. -// Col cell sizes must match size of probe (panics if not). -// Optimized for tensor.Float64 but works for any tensor. -// Py version is for Python, returns a slice with row, cor, takes std metric -func ClosestRow64Py(probe tensor.Tensor, col tensor.Tensor, std StdMetrics) []float64 { - row, cor := ClosestRow64(probe, col, StdFunc64(std)) - return []float64{float64(row), cor} -} +*/ diff --git a/tensor/stats/metric/vecfuncs.go b/tensor/stats/metric/vecfuncs.go new file mode 100644 index 0000000000..1905db1acb --- /dev/null +++ b/tensor/stats/metric/vecfuncs.go @@ -0,0 +1,330 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metric + +import ( + "math" + "slices" + + "cogentcore.org/core/tensor" +) + +// VectorizeOut64 is a version of the [tensor.Vectorize] function +// for metrics, which makes a Float64 output tensor for aggregating +// and computing values, and then copies the results back to the +// original output. This allows metrics functions to operate directly +// on integer valued inputs and produce sensible results. +// It automatically calls NFunc for the nfun function, +// and returns the Float64 output tensor for further processing as needed. +// It uses the _last_ tensor as the output, allowing for multiple inputs. +func VectorizeOut64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) *tensor.Indexed { + n := NFunc(tsr...) + if n <= 0 { + return nil + } + nt := len(tsr) + out := tsr[nt-1] + o64 := tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + etsr := slices.Clone(tsr) + etsr[nt-1] = o64 + for idx := range n { + fun(idx, etsr...) + } + nsub := out.Tensor.Len() + for i := range nsub { + out.Tensor.SetFloat1D(i, o64.Tensor.Float1D(i)) + } + return o64 +} + +// Vectorize2Out64 is a version of the [tensor.Vectorize] function +// for metrics, which makes two Float64 output tensors for aggregating +// and computing values, returning them for final computation. +// It automatically calls NFunc for the nfun function. +func Vectorize2Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2 *tensor.Indexed) { + n := NFunc(tsr...) + if n <= 0 { + return nil, nil + } + nt := len(tsr) + out := tsr[nt-1] + out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + for idx := range n { + fun(idx, tsr[0], tsr[1], out1, out2) + } + return out1, out2 +} + +// Vectorize2in3Out64 is a version of the [tensor.Vectorize] function +// for metrics, which makes three Float64 output tensors for aggregating +// and computing values, returning them for final computation. +// It automatically calls NFunc for the nfun function. +func Vectorize2in3Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2, out3 *tensor.Indexed) { + n := NFunc(tsr...) + if n <= 0 { + return nil, nil, nil + } + nt := len(tsr) + out := tsr[nt-1] + out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out3 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + for idx := range n { + fun(idx, tsr[0], tsr[1], tsr[2], tsr[3], out1, out2, out3) + } + return out1, out2, out3 +} + +// Vectorize3Out64 is a version of the [tensor.Vectorize] function +// for metrics, which makes three Float64 output tensors for aggregating +// and computing values, returning them for final computation. +// It automatically calls NFunc for the nfun function. +func Vectorize3Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2, out3 *tensor.Indexed) { + n := NFunc(tsr...) + if n <= 0 { + return nil, nil, nil + } + nt := len(tsr) + out := tsr[nt-1] + out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out3 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + for idx := range n { + fun(idx, tsr[0], tsr[1], out1, out2, out3) + } + return out1, out2, out3 +} + +// OutShape returns the output shape based on given input +// tensor, with outer row dim = 1. +func OutShape(ish *tensor.Shape) *tensor.Shape { + osh := &tensor.Shape{} + osh.CopyShape(ish) + osh.Sizes[0] = 1 + return osh +} + +// NFunc is the nfun for metrics functions, returning the min length across the +// two input tensors, and initializing the _last_ one to hold the output +// with the first, row dimension set to 1. +func NFunc(tsr ...*tensor.Indexed) int { + nt := len(tsr) + if nt < 3 { + return 0 + } + a, b, out := tsr[0], tsr[1], tsr[nt-1] + na, nb := a.Len(), b.Len() + osh := OutShape(a.Tensor.Shape()) + out.Tensor.SetShape(osh.Sizes, osh.Names...) + out.Indexes = []int{0} + return min(na, nb) +} + +// MetricVecFunc is a helper function for metrics functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// It also skips over NaN missing values. +func MetricVecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg float64) float64) { + nsub := out.Tensor.Len() + for i := range nsub { + if idx == 0 { + out.Tensor.SetFloat1D(i, ini) + } + av := a.FloatRowCell(idx, i) + if math.IsNaN(av) { + continue + } + bv := b.FloatRowCell(idx, i) + if math.IsNaN(bv) { + continue + } + out.Tensor.SetFloat1D(i, fun(av, bv, out.Tensor.Float1D(i))) + } +} + +// MetricVecSSFunc is a helper function for metric functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// This version does sum-of-squares integration over 2 output vectors, +// It also skips over NaN missing values. +func MetricVecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(a, b float64) float64) { + nsub := out2.Tensor.Len() + for i := range nsub { + if idx == 0 { + out1.Tensor.SetFloat1D(i, ini1) + out2.Tensor.SetFloat1D(i, ini2) + } + av := a.FloatRowCell(idx, i) + if math.IsNaN(av) { + continue + } + bv := b.FloatRowCell(idx, i) + if math.IsNaN(bv) { + continue + } + scale, ss := out1.Tensor.Float1D(i), out2.Tensor.Float1D(i) + d := fun(av, bv) + if d == 0 { + continue + } + absxi := math.Abs(d) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } + out1.Tensor.SetFloat1D(i, scale) + out2.Tensor.SetFloat1D(i, ss) + } +} + +// SumSquaresVecFunc computes the sum-of-squares distance between two vectors. +func SumSquaresVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { + return a - b + }) +} + +// SumSquaresBinTolVecFunc computes the sum-of-squares distance between two vectors, +// with binary tolerance: differences < 0.5 are thresholded to 0. +func SumSquaresBinTolVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { + d := a - b + if math.Abs(d) < 0.5 { + d = 0 + } + return d + }) +} + +// InnerProductVecFunc is a Vectorize function for computing the inner product. +func InnerProductVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { + return agg + a*b + }) +} + +// MetricVec2inFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// This version has 2 input vectors, the second input being the output of another stat +// e.g., the mean. It also skips over NaN missing values. +func MetricVec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun func(a, b, a2, b2, agg float64) float64) { + nsub := out.Tensor.Len() + for i := range nsub { + if idx == 0 { + out.Tensor.SetFloat1D(i, ini) + } + av := a.FloatRowCell(idx, i) + if math.IsNaN(av) { + continue + } + bv := b.FloatRowCell(idx, i) + if math.IsNaN(bv) { + continue + } + av2 := a2.Tensor.Float1D(i) + bv2 := b2.Tensor.Float1D(i) + out.Tensor.SetFloat1D(i, fun(av, bv, av2, bv2, out.Tensor.Float1D(i))) + } +} + +// CovarianceVecFunc is a Vectorize function for computing the covariance between +// two vectors, i.e., the mean of the co-product of each vector element minus +// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))] +func CovarianceVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, a2, b2, agg float64) float64 { + return agg + (a-a2)*(b-b2) + }) +} + +// MetricVec2in3outFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// This version has 2 input, 3 output vectors. The second input being the output of another stat +// e.g., the mean. It also skips over NaN missing values. +func MetricVec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, a2, b2, out1, out2, out3 float64) (float64, float64, float64)) { + nsub := out1.Tensor.Len() + for i := range nsub { + if idx == 0 { + out1.Tensor.SetFloat1D(i, ini) + out2.Tensor.SetFloat1D(i, ini) + out3.Tensor.SetFloat1D(i, ini) + } + av := a.FloatRowCell(idx, i) + if math.IsNaN(av) { + continue + } + bv := b.FloatRowCell(idx, i) + if math.IsNaN(bv) { + continue + } + av2 := a2.Tensor.Float1D(i) + bv2 := b2.Tensor.Float1D(i) + o1 := out1.Tensor.Float1D(i) + o2 := out2.Tensor.Float1D(i) + o3 := out3.Tensor.Float1D(i) + o1, o2, o3 = fun(av, bv, av2, bv2, o1, o2, o3) + out1.Tensor.SetFloat1D(i, o1) + out2.Tensor.SetFloat1D(i, o2) + out3.Tensor.SetFloat1D(i, o3) + } +} + +// CorrelationVecFunc is a Vectorize function for computing the correlation between +// two vectors, in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized +// vectors. +func CorrelationVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { + ad := a - am + bd := b - bm + ss += ad * bd // between + avar += ad * ad // within + bvar += bd * bd + return ss, avar, bvar + }) +} + +// MetricVec3outFunc is a helper function for stats functions, dealing with iterating over +// the Cell subspace per row and initializing the aggregation values for first index. +// This version has 3 output vectors. It also skips over NaN missing values. +func MetricVec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, out1, out2, out3 float64) (float64, float64, float64)) { + nsub := out1.Tensor.Len() + for i := range nsub { + if idx == 0 { + out1.Tensor.SetFloat1D(i, ini) + out2.Tensor.SetFloat1D(i, ini) + out3.Tensor.SetFloat1D(i, ini) + } + av := a.FloatRowCell(idx, i) + if math.IsNaN(av) { + continue + } + bv := b.FloatRowCell(idx, i) + if math.IsNaN(bv) { + continue + } + o1 := out1.Tensor.Float1D(i) + o2 := out2.Tensor.Float1D(i) + o3 := out3.Tensor.Float1D(i) + o1, o2, o3 = fun(av, bv, o1, o2, o3) + out1.Tensor.SetFloat1D(i, o1) + out2.Tensor.SetFloat1D(i, o2) + out3.Tensor.SetFloat1D(i, o3) + } +} + +// CosineVecFunc is a Vectorize function for computing the cosine between +// two vectors, in range (-1..1) as the normalized inner product: +// inner product / sqrt(ssA * ssB). +func CosineVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { + ss += a * b + avar += a * a + bvar += b * b + return ss, avar, bvar + }) +} diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 0d478afe29..45c8754cec 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -254,42 +254,3 @@ func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { } return scale64 } - -/* - -/////////////////////////////////////////// -// L2Norm - -// L2Norm64 computes the square-root of sum-of-squares of vector, i.e., the L2 norm. -// Skips NaN's. Uses optimized algorithm from BLAS that avoids numerical overflow. -func L2Norm64(a []float64) float64 { - n := len(a) - if n < 2 { - if n == 1 { - return math.Abs(a[0]) - } - return 0 - } - var ( - scale float64 = 0 - ss float64 = 1 - ) - for _, v := range a { - if v == 0 || math.IsNaN(v) { - continue - } - absxi := math.Abs(v) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - } - if math.IsInf(scale, 1) { - return math.Inf(1) - } - return scale * math.Sqrt(ss) -} - -*/ diff --git a/tensor/stats/stats/if.go b/tensor/stats/stats/if.go deleted file mode 100644 index 3293ca7bbc..0000000000 --- a/tensor/stats/stats/if.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package stats - -/* - -// IfFunc is used for the *If aggregators -- counted if it returns true -type IfFunc func(idx int, val float64) bool - -/////////////////////////////////////////////////// -// CountIf - -// CountIfIndex returns the count of true return values for given IfFunc on -// non-NaN elements in given Indexed indexed view of an -// table.Table, for given column index. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func CountIfIndex(ix *table.Indexed, colIndex int, iffun IfFunc) []float64 { - return StatIndexFunc(ix, colIndex, 0, func(idx int, val float64, agg float64) float64 { - if iffun(idx, val) { - return agg + 1 - } - return agg - }) -} - -// CountIfColumn returns the count of true return values for given IfFunc on -// non-NaN elements in given Indexed indexed view of an -// table.Table, for given column name. -// If name not found, nil is returned. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func CountIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return CountIfIndex(ix, colIndex, iffun) -} - -/////////////////////////////////////////////////// -// PropIf - -// PropIfIndex returns the proportion (0-1) of true return values for given IfFunc on -// non-Null, non-NaN elements in given Indexed indexed view of an -// table.Table, for given column index. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func PropIfIndex(ix *table.Indexed, colIndex int, iffun IfFunc) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - pif := CountIfIndex(ix, colIndex, iffun) - for i := range pif { - if cnt[i] > 0 { - pif[i] /= cnt[i] - } - } - return pif -} - -// PropIfColumn returns the proportion (0-1) of true return values for given IfFunc on -// non-NaN elements in given Indexed indexed view of an -// table.Table, for given column name. -// If name not found, nil is returned -- use Try version for error message. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func PropIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return PropIfIndex(ix, colIndex, iffun) -} - -/////////////////////////////////////////////////// -// PctIf - -// PctIfIndex returns the percentage (0-100) of true return values for given IfFunc on -// non-Null, non-NaN elements in given Indexed indexed view of an -// table.Table, for given column index. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func PctIfIndex(ix *table.Indexed, colIndex int, iffun IfFunc) []float64 { - cnt := CountIndex(ix, colIndex) - if cnt == nil { - return nil - } - pif := CountIfIndex(ix, colIndex, iffun) - for i := range pif { - if cnt[i] > 0 { - pif[i] = 100.0 * (pif[i] / cnt[i]) - } - } - return pif -} - -// PctIfColumn returns the percentage (0-100) of true return values for given IfFunc on -// non-Null, non-NaN elements in given Indexed indexed view of an -// table.Table, for given column name. -// If name not found, nil is returned -- use Try version for error message. -// Return value(s) is size of column cell: 1 for scalar 1D columns -// and N for higher-dimensional columns. -func PctIfColumn(ix *table.Indexed, column string, iffun IfFunc) []float64 { - colIndex := errors.Log1(ix.Table.ColumnIndex(column)) - if colIndex == -1 { - return nil - } - return PctIfIndex(ix, colIndex, iffun) -} - -*/ diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 6a210e5471..ca50e94294 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -4,44 +4,60 @@ package stats -import "cogentcore.org/core/tensor" +import ( + "fmt" + + "cogentcore.org/core/tensor" +) //go:generate core generate -// StatsFuncs is a registry of named stats functions, +// Funcs is a registry of named stats functions, // which can then be called by standard enum or // string name for custom functions. -var StatsFuncs map[string]StatsFunc +var Funcs map[string]StatsFunc func init() { - StatsFuncs = make(map[string]StatsFunc) - StatsFuncs[Count.String()] = CountFunc - StatsFuncs[Sum.String()] = SumFunc - StatsFuncs[SumAbs.String()] = SumAbsFunc - StatsFuncs[L1Norm.String()] = SumAbsFunc - StatsFuncs[Prod.String()] = ProdFunc - StatsFuncs[Min.String()] = MinFunc - StatsFuncs[Max.String()] = MaxFunc - StatsFuncs[MinAbs.String()] = MinAbsFunc - StatsFuncs[MaxAbs.String()] = MaxAbsFunc - StatsFuncs[Mean.String()] = MeanFunc - StatsFuncs[Var.String()] = VarFunc - StatsFuncs[Std.String()] = StdFunc - StatsFuncs[Sem.String()] = SemFunc - StatsFuncs[SumSq.String()] = SumSqFunc - StatsFuncs[L2Norm.String()] = L2NormFunc - StatsFuncs[VarPop.String()] = VarPopFunc - StatsFuncs[StdPop.String()] = StdPopFunc - StatsFuncs[SemPop.String()] = SemPopFunc - StatsFuncs[Median.String()] = MedianFunc - StatsFuncs[Q1.String()] = Q1Func - StatsFuncs[Q3.String()] = Q3Func + Funcs = make(map[string]StatsFunc) + Funcs[Count.String()] = CountFunc + Funcs[Sum.String()] = SumFunc + Funcs[SumAbs.String()] = SumAbsFunc + Funcs[L1Norm.String()] = SumAbsFunc + Funcs[Prod.String()] = ProdFunc + Funcs[Min.String()] = MinFunc + Funcs[Max.String()] = MaxFunc + Funcs[MinAbs.String()] = MinAbsFunc + Funcs[MaxAbs.String()] = MaxAbsFunc + Funcs[Mean.String()] = MeanFunc + Funcs[Var.String()] = VarFunc + Funcs[Std.String()] = StdFunc + Funcs[Sem.String()] = SemFunc + Funcs[SumSq.String()] = SumSqFunc + Funcs[L2Norm.String()] = L2NormFunc + Funcs[VarPop.String()] = VarPopFunc + Funcs[StdPop.String()] = StdPopFunc + Funcs[SemPop.String()] = SemPopFunc + Funcs[Median.String()] = MedianFunc + Funcs[Q1.String()] = Q1Func + Funcs[Q3.String()] = Q3Func } // Standard calls a standard Stats enum function on given tensors. // Output results are in the out tensor. func Standard(stat Stats, in, out *tensor.Indexed) { - StatsFuncs[stat.String()](in, out) + Funcs[stat.String()](in, out) +} + +// Call calls a registered stats function on given tensors. +// Output results are in the out tensor. Returns an +// error if name not found. +func Call(name string, in, out *tensor.Indexed) error { + f, ok := Funcs[name] + if !ok { + return fmt.Errorf("stats.Call: function %q not registered", name) + } + f(in, out) + return nil } // Stats is a list of different standard aggregation functions, which can be used diff --git a/tensor/stats/stats/funcs_test.go b/tensor/stats/stats/stats_test.go similarity index 98% rename from tensor/stats/stats/funcs_test.go rename to tensor/stats/stats/stats_test.go index 7fc1f2fd4f..4cca3404e6 100644 --- a/tensor/stats/stats/funcs_test.go +++ b/tensor/stats/stats/stats_test.go @@ -14,8 +14,7 @@ import ( func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - tsr := tensor.NewNumberFromSlice(vals) - ix := tensor.NewIndexed(tsr) + ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) out := tensor.NewFloat64([]int{1}) oix := tensor.NewIndexed(out) From d8def0bfd14a96bdde748c61b8f3c1116127e27e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 10 Sep 2024 04:27:17 -0700 Subject: [PATCH 010/311] register std funcs --- tensor/stats/metric/funcs.go | 3 +-- tensor/stats/metric/metrics.go | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 9e1d8d5997..e7219fdfe6 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -96,7 +96,7 @@ func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { // SumSquaresBinTolFunc computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) *tensor.Indexed { +func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { scale64, ss64 := Vectorize2Out64(SumSquaresBinTolVecFunc, a, b, out) nsub := out.Tensor.Len() for i := range nsub { @@ -111,7 +111,6 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) *tensor.Indexed { scale64.Tensor.SetFloat1D(i, v) out.Tensor.SetFloat1D(i, v) } - return scale64 } // CovarianceFunc computes the covariance between two vectors, diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 0d3248bd0a..4cf4b15296 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -19,6 +19,16 @@ var Funcs map[string]MetricFunc func init() { Funcs = make(map[string]MetricFunc) + Funcs[Euclidean.String()] = EuclideanFunc + Funcs[SumSquares.String()] = SumSquaresFunc + Funcs[EuclideanBinTol.String()] = EuclideanBinTolFunc + Funcs[SumSquaresBinTol.String()] = SumSquaresBinTolFunc + Funcs[InvCosine.String()] = InvCosineFunc + Funcs[InvCorrelation.String()] = InvCorrelationFunc + Funcs[InnerProduct.String()] = InnerProductFunc + Funcs[Covariance.String()] = CovarianceFunc + Funcs[Correlation.String()] = CorrelationFunc + Funcs[Cosine.String()] = CosineFunc } // Standard calls a standard Metrics enum function on given tensors. From 3146ff1a2171a03d205585ed3964720b02b18666 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 10 Sep 2024 10:48:09 -0700 Subject: [PATCH 011/311] reorganize stats, metric funcs so the full computation is in the same place in funcs --- tensor/stats/metric/funcs.go | 107 ++++++++++++++++++++----- tensor/stats/metric/squares.go | 50 ------------ tensor/stats/metric/vecfuncs.go | 88 +-------------------- tensor/stats/stats/funcs.go | 134 ++++++++++++++++++++++++++------ tensor/stats/stats/vecfuncs.go | 84 -------------------- 5 files changed, 199 insertions(+), 264 deletions(-) delete mode 100644 tensor/stats/metric/squares.go diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index e7219fdfe6..e54141998b 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -23,10 +23,11 @@ import ( // if that is needed. type MetricFunc func(a, b, out *tensor.Indexed) -// InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. -// See [MetricFunc] for general information. -func InnerProductFunc(a, b, out *tensor.Indexed) { - VectorizeOut64(InnerProductVecFunc, a, b, out) +// SumSquaresVecFunc computes the sum-of-squares distance between two vectors. +func SumSquaresVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { + return a - b + }) } // SumSquaresFunc computes the sum of squares differences between tensor values, @@ -74,6 +75,18 @@ func EuclideanFunc(a, b, out *tensor.Indexed) { } } +// SumSquaresBinTolVecFunc computes the sum-of-squares distance between two vectors, +// with binary tolerance: differences < 0.5 are thresholded to 0. +func SumSquaresBinTolVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { + d := a - b + if math.Abs(d) < 0.5 { + d = 0 + } + return d + }) +} + // EuclideanBinTolFunc computes the Euclidean square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. @@ -113,6 +126,28 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { } } +// InnerProductVecFunc is a Vectorize function for computing the inner product. +func InnerProductVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { + return agg + a*b + }) +} + +// InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. +// See [MetricFunc] for general information. +func InnerProductFunc(a, b, out *tensor.Indexed) { + VectorizeOut64(InnerProductVecFunc, a, b, out) +} + +// CovarianceVecFunc is a Vectorize function for computing the covariance between +// two vectors, i.e., the mean of the co-product of each vector element minus +// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))] +func CovarianceVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, am, bm, agg float64) float64 { + return agg + (a-am)*(b-bm) + }) +} + // CovarianceFunc computes the covariance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. @@ -131,26 +166,33 @@ func CovarianceFunc(a, b, out *tensor.Indexed) { } } -// CorrelationFunc computes the correlation between two vectors, -// in range (-1..1) as the mean of the co-product of each vector +// CorrelationVecFunc is a Vectorize function for computing the correlation between +// two vectors, in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized -// vectors. -func CorrelationFunc(a, b, out *tensor.Indexed) { - CorrelationFuncOut64(a, b, out) +// (i.e., the standardized covariance). +// Equivalent to the cosine of mean-normalized vectors. +func CorrelationVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { + ad := a - am + bd := b - bm + ss += ad * bd // between + avar += ad * ad // within + bvar += bd * bd + return ss, avar, bvar + }) } // CorrelationFuncOut64 computes the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized -// vectors. +// (i.e., the standardized covariance). +// Equivalent to the cosine of mean-normalized vectors. func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { amean, _ := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - ss64, avar64, bvar64 := Vectorize2in3Out64(CorrelationVecFunc, a, b, amean, bmean, out) + ss64, avar64, bvar64 := Vectorize3Out64(CorrelationVecFunc, a, b, amean, bmean, out) nsub := out.Tensor.Len() for i := range nsub { ss := ss64.Tensor.Float1D(i) @@ -164,12 +206,24 @@ func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { return ss64 } +// CorrelationFunc computes the correlation between two vectors, +// in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized covariance). +// Equivalent to the cosine of mean-normalized vectors. +func CorrelationFunc(a, b, out *tensor.Indexed) { + CorrelationFuncOut64(a, b, out) +} + // InvCorrelationFunc computes 1 minus the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized -// vectors. This is useful for a difference measure instead of similarity. +// (i.e., the standardized covariance). +// Equivalent to the cosine of mean-normalized vectors. +// This is useful for a difference measure instead of similarity, +// where more different vectors have larger metric values. func InvCorrelationFunc(a, b, out *tensor.Indexed) { cor64 := CorrelationFuncOut64(a, b, out) nsub := out.Tensor.Len() @@ -179,11 +233,16 @@ func InvCorrelationFunc(a, b, out *tensor.Indexed) { } } -// CosineFunc computes the correlation between two vectors, -// in range (-1..1) as the normalized inner product: +// CosineVecFunc is a Vectorize function for computing the cosine between +// two vectors, in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). -func CosineFunc(a, b, out *tensor.Indexed) { - CosineFuncOut64(a, b, out) +func CosineVecFunc(idx int, tsr ...*tensor.Indexed) { + MetricVec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { + ss += a * b + avar += a * a + bvar += b * b + return ss, avar, bvar + }) } // CosineFuncOut64 computes the correlation between two vectors, @@ -204,10 +263,18 @@ func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { return ss64 } +// CosineFunc computes the correlation between two vectors, +// in range (-1..1) as the normalized inner product: +// inner product / sqrt(ssA * ssB). +func CosineFunc(a, b, out *tensor.Indexed) { + CosineFuncOut64(a, b, out) +} + // InvCosineFunc computes 1 minus the cosine between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). -// This is useful for a difference measure instead of similarity. +// This is useful for a difference measure instead of similarity, +// where more different vectors have larger metric values. func InvCosineFunc(a, b, out *tensor.Indexed) { cos64 := CosineFuncOut64(a, b, out) nsub := out.Tensor.Len() diff --git a/tensor/stats/metric/squares.go b/tensor/stats/metric/squares.go deleted file mode 100644 index 71bb0dd1f0..0000000000 --- a/tensor/stats/metric/squares.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package metric - -/* - -/////////////////////////////////////////// -// Cosine - -// Cosine64 computes the cosine of the angle between two vectors (-1..1), -// as the normalized inner product: inner product / sqrt(ssA * ssB). -// If vectors are mean-normalized = Correlation. -// Skips NaN's and panics if lengths are not equal. -func Cosine64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - var ass, bss float64 - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - ss += av * bv // between - ass += av * av // within - bss += bv * bv - } - vp := math.Sqrt(ass * bss) - if vp > 0 { - ss /= vp - } - return ss -} - -/////////////////////////////////////////// -// InvCosine - -// InvCosine32 computes 1 - cosine of the angle between two vectors (-1..1), -// as the normalized inner product: inner product / sqrt(ssA * ssB). -// If vectors are mean-normalized = Correlation. -// Skips NaN's and panics if lengths are not equal. -func InvCosine64(a, b []float64) float64 { - return 1 - Cosine64(a, b) -} - - -*/ diff --git a/tensor/stats/metric/vecfuncs.go b/tensor/stats/metric/vecfuncs.go index 1905db1acb..26c2bd7df5 100644 --- a/tensor/stats/metric/vecfuncs.go +++ b/tensor/stats/metric/vecfuncs.go @@ -58,26 +58,6 @@ func Vectorize2Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.I return out1, out2 } -// Vectorize2in3Out64 is a version of the [tensor.Vectorize] function -// for metrics, which makes three Float64 output tensors for aggregating -// and computing values, returning them for final computation. -// It automatically calls NFunc for the nfun function. -func Vectorize2in3Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2, out3 *tensor.Indexed) { - n := NFunc(tsr...) - if n <= 0 { - return nil, nil, nil - } - nt := len(tsr) - out := tsr[nt-1] - out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - out3 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - for idx := range n { - fun(idx, tsr[0], tsr[1], tsr[2], tsr[3], out1, out2, out3) - } - return out1, out2, out3 -} - // Vectorize3Out64 is a version of the [tensor.Vectorize] function // for metrics, which makes three Float64 output tensors for aggregating // and computing values, returning them for final computation. @@ -92,8 +72,10 @@ func Vectorize3Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.I out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) out3 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + tsrs := tsr[:nt-1] + tsrs = append(tsrs, out1, out2, out3) for idx := range n { - fun(idx, tsr[0], tsr[1], out1, out2, out3) + fun(idx, tsrs...) } return out1, out2, out3 } @@ -180,32 +162,6 @@ func MetricVecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float } } -// SumSquaresVecFunc computes the sum-of-squares distance between two vectors. -func SumSquaresVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { - return a - b - }) -} - -// SumSquaresBinTolVecFunc computes the sum-of-squares distance between two vectors, -// with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTolVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { - d := a - b - if math.Abs(d) < 0.5 { - d = 0 - } - return d - }) -} - -// InnerProductVecFunc is a Vectorize function for computing the inner product. -func InnerProductVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { - return agg + a*b - }) -} - // MetricVec2inFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input vectors, the second input being the output of another stat @@ -230,15 +186,6 @@ func MetricVec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, f } } -// CovarianceVecFunc is a Vectorize function for computing the covariance between -// two vectors, i.e., the mean of the co-product of each vector element minus -// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))] -func CovarianceVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, a2, b2, agg float64) float64 { - return agg + (a-a2)*(b-b2) - }) -} - // MetricVec2in3outFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input, 3 output vectors. The second input being the output of another stat @@ -271,23 +218,6 @@ func MetricVec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexe } } -// CorrelationVecFunc is a Vectorize function for computing the correlation between -// two vectors, in range (-1..1) as the mean of the co-product of each vector -// element minus the mean of that vector, normalized by the product of their -// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance). Equivalent to the cosine of mean-normalized -// vectors. -func CorrelationVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { - ad := a - am - bd := b - bm - ss += ad * bd // between - avar += ad * ad // within - bvar += bd * bd - return ss, avar, bvar - }) -} - // MetricVec3outFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 3 output vectors. It also skips over NaN missing values. @@ -316,15 +246,3 @@ func MetricVec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini floa out3.Tensor.SetFloat1D(i, o3) } } - -// CosineVecFunc is a Vectorize function for computing the cosine between -// two vectors, in range (-1..1) as the normalized inner product: -// inner product / sqrt(ssA * ssB). -func CosineVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { - ss += a * b - avar += a * a - bvar += b * b - return ss, avar, bvar - }) -} diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 45c8754cec..0ee7792f83 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -22,18 +22,40 @@ import ( // if that is needed. type StatsFunc func(in, out *tensor.Indexed) +// CountVecFunc is a Vectorize function for computing the count. +func CountVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + 1 + }) +} + // CountFunc computes the count of non-NaN tensor values. // See [StatsFunc] for general information. func CountFunc(in, out *tensor.Indexed) { VectorizeOut64(CountVecFunc, in, out) } +// SumVecFunc is a Vectorize function for computing the sum. +func SumVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + val + }) +} + // SumFunc computes the sum of tensor values. // See [StatsFunc] for general information. func SumFunc(in, out *tensor.Indexed) { VectorizeOut64(SumVecFunc, in, out) } +// SumAbsVecFunc is a Vectorize function for computing the sum of abs values. +// This is also known as the L1 norm. +func SumAbsVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + math.Abs(val) + }) +} + // SumAbsFunc computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. @@ -41,42 +63,71 @@ func SumAbsFunc(in, out *tensor.Indexed) { VectorizeOut64(SumAbsVecFunc, in, out) } +// ProdVecFunc is a Vectorize function for computing the product. +func ProdVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { + return agg * val + }) +} + // ProdFunc computes the product of tensor values. // See [StatsFunc] for general information. func ProdFunc(in, out *tensor.Indexed) { VectorizeOut64(ProdVecFunc, in, out) } +// MinVecFunc is a Vectorize function for computing the min. +func MinVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, val) + }) +} + // MinFunc computes the min of tensor values. // See [StatsFunc] for general information. func MinFunc(in, out *tensor.Indexed) { VectorizeOut64(MinVecFunc, in, out) } +// MaxVecFunc is a Vectorize function for computing the max. +func MaxVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, val) + }) +} + // MaxFunc computes the max of tensor values. // See [StatsFunc] for general information. func MaxFunc(in, out *tensor.Indexed) { VectorizeOut64(MaxVecFunc, in, out) } +// MinAbsVecFunc is a Vectorize function for computing the min of abs. +func MinAbsVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, math.Abs(val)) + }) +} + // MinAbsFunc computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. func MinAbsFunc(in, out *tensor.Indexed) { VectorizeOut64(MinAbsVecFunc, in, out) } +// MaxAbsVecFunc is a Vectorize function for computing the max of abs. +func MaxAbsVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, math.Abs(val)) + }) +} + // MaxAbsFunc computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. func MaxAbsFunc(in, out *tensor.Indexed) { VectorizeOut64(MaxAbsVecFunc, in, out) } -// MeanFunc computes the mean of tensor values. -// See [StatsFunc] for general information. -func MeanFunc(in, out *tensor.Indexed) { - MeanFuncOut64(in, out) -} - // MeanFuncOut64 computes the mean of tensor values, // and returns the Float64 count and mean output values for // use in subsequent computations. @@ -96,11 +147,19 @@ func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { return sum64, count64 } -// VarFunc computes the sample variance of tensor values. -// Squared deviations from mean, divided by n-1. See also [VarPopFunc]. +// MeanFunc computes the mean of tensor values. // See [StatsFunc] for general information. -func VarFunc(in, out *tensor.Indexed) { - VarFuncOut64(in, out) +func MeanFunc(in, out *tensor.Indexed) { + MeanFuncOut64(in, out) +} + +// VarVecFunc is a Vectorize function for computing the variance, +// using 3 tensors: in, mean, out. +func VarVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { + dv := val1 - val2 + return agg + dv*dv + }) } // VarFuncOut64 computes the sample variance of tensor values. @@ -121,6 +180,13 @@ func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Index return } +// VarFunc computes the sample variance of tensor values. +// Squared deviations from mean, divided by n-1. See also [VarPopFunc]. +// See [StatsFunc] for general information. +func VarFunc(in, out *tensor.Indexed) { + VarFuncOut64(in, out) +} + // StdFunc computes the sample standard deviation of tensor values. // Sqrt of variance from [VarFunc]. See also [StdPopFunc]. // See [StatsFunc] for general information. @@ -148,13 +214,6 @@ func SemFunc(in, out *tensor.Indexed) { } } -// VarPopFunc computes the population variance of tensor values. -// Squared deviations from mean, divided by n. See also [VarFunc]. -// See [StatsFunc] for general information. -func VarPopFunc(in, out *tensor.Indexed) { - VarPopFuncOut64(in, out) -} - // VarPopFuncOut64 computes the population variance of tensor values. // and returns the Float64 output values for // use in subsequent computations. @@ -173,6 +232,13 @@ func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.In return } +// VarPopFunc computes the population variance of tensor values. +// Squared deviations from mean, divided by n. See also [VarFunc]. +// See [StatsFunc] for general information. +func VarPopFunc(in, out *tensor.Indexed) { + VarPopFuncOut64(in, out) +} + // StdPopFunc computes the population standard deviation of tensor values. // Sqrt of variance from [VarPopFunc]. See also [StdFunc]. // See [StatsFunc] for general information. @@ -200,10 +266,22 @@ func SemPopFunc(in, out *tensor.Indexed) { } } -// SumSqFunc computes the sum of squares of tensor values, -// See [StatsFunc] for general information. -func SumSqFunc(in, out *tensor.Indexed) { - SumSqFuncOut64(in, out) +// SumSqVecFunc is a Vectorize function for computing the sum of squares, +// using 2 separate aggregation tensors. +func SumSqVecFunc(idx int, tsr ...*tensor.Indexed) { + StatVec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { + if val == 0 { + return scale, ss + } + absxi := math.Abs(val) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } + return scale, ss + }) } // SumSqFuncOut64 computes the sum of squares of tensor values, @@ -227,11 +305,10 @@ func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { return scale64 } -// L2NormFunc computes the square root of the sum of squares of tensor values, -// known as the L2 norm. +// SumSqFunc computes the sum of squares of tensor values, // See [StatsFunc] for general information. -func L2NormFunc(in, out *tensor.Indexed) { - L2NormFuncOut64(in, out) +func SumSqFunc(in, out *tensor.Indexed) { + SumSqFuncOut64(in, out) } // L2NormFuncOut64 computes the square root of the sum of squares of tensor values, @@ -254,3 +331,10 @@ func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { } return scale64 } + +// L2NormFunc computes the square root of the sum of squares of tensor values, +// known as the L2 norm. +// See [StatsFunc] for general information. +func L2NormFunc(in, out *tensor.Indexed) { + L2NormFuncOut64(in, out) +} diff --git a/tensor/stats/stats/vecfuncs.go b/tensor/stats/stats/vecfuncs.go index 6e157ab4ac..5aa2da98d3 100644 --- a/tensor/stats/stats/vecfuncs.go +++ b/tensor/stats/stats/vecfuncs.go @@ -101,63 +101,6 @@ func StatVecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, ag } } -// SumVecFunc is a Vectorize function for computing the sum. -func SumVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + val - }) -} - -// SumAbsVecFunc is a Vectorize function for computing the sum of abs values. -// This is also known as the L1 norm. -func SumAbsVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + math.Abs(val) - }) -} - -// CountVecFunc is a Vectorize function for computing the count. -func CountVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + 1 - }) -} - -// ProdVecFunc is a Vectorize function for computing the product. -func ProdVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { - return agg * val - }) -} - -// MinVecFunc is a Vectorize function for computing the min. -func MinVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { - return math.Min(agg, val) - }) -} - -// MaxVecFunc is a Vectorize function for computing the max. -func MaxVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { - return math.Max(agg, val) - }) -} - -// MinAbsVecFunc is a Vectorize function for computing the min of abs. -func MinAbsVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { - return math.Min(agg, math.Abs(val)) - }) -} - -// MaxAbsVecFunc is a Vectorize function for computing the max of abs. -func MaxAbsVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { - return math.Max(agg, math.Abs(val)) - }) -} - /////////////////////////////////////////////////////\ // Two input Tensors @@ -181,15 +124,6 @@ func StatVec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun fun } } -// VarVecFunc is a Vectorize function for computing the variance, -// using 3 tensors: in, mean, out. -func VarVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { - dv := val1 - val2 - return agg + dv*dv - }) -} - /////////////////////////////////////////////////////\ // Two output Tensors @@ -214,21 +148,3 @@ func StatVec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64 out2.Tensor.SetFloat1D(i, ag2) } } - -// SumSqVecFunc is a Vectorize function for computing the sum of squares, -// using 2 separate aggregation tensors. -func SumSqVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { - if val == 0 { - return scale, ss - } - absxi := math.Abs(val) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - return scale, ss - }) -} From 2fac27a413c32a0be77dc93ddb20088fb771ed09 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 10 Sep 2024 11:51:09 -0700 Subject: [PATCH 012/311] major cleanup: put all vec funcs inline -- no need to export those at all -- much cleaner. --- math32/math.go | 11 ++ tensor/stats/metric/funcs.go | 150 ++++++++++++----------- tensor/stats/metric/metric_test.go | 23 ++-- tensor/stats/metric/prob.go | 64 ---------- tensor/stats/metric/vecfuncs.go | 69 ++--------- tensor/stats/stats/funcs.go | 189 ++++++++++++++--------------- tensor/stats/stats/vecfuncs.go | 22 ++-- 7 files changed, 212 insertions(+), 316 deletions(-) delete mode 100644 tensor/stats/metric/prob.go diff --git a/math32/math.go b/math32/math.go index 0f4413e3f8..efce54652a 100644 --- a/math32/math.go +++ b/math32/math.go @@ -804,6 +804,17 @@ func ClampInt(x, a, b int) int { return x } +// Clamp64 clamps x to the provided closed interval [a, b] +func Clamp64(x, a, b float64) float64 { + if x < a { + return a + } + if x > b { + return b + } + return x +} + // MinPos returns the minimum of the two values, excluding any that are <= 0 func MinPos(a, b float32) float32 { if a > 0 && b > 0 { diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index e54141998b..b678679851 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -7,6 +7,7 @@ package metric import ( "math" + "cogentcore.org/core/math32" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/stats/stats" ) @@ -23,23 +24,21 @@ import ( // if that is needed. type MetricFunc func(a, b, out *tensor.Indexed) -// SumSquaresVecFunc computes the sum-of-squares distance between two vectors. -func SumSquaresVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { - return a - b - }) -} - -// SumSquaresFunc computes the sum of squares differences between tensor values, -// See [MetricFunc] for general information. -func SumSquaresFunc(a, b, out *tensor.Indexed) { - SumSquaresFuncOut64(a, b, out) +// SumSquaresScaleFuncOut64 computes the sum of squares differences between tensor values, +// returning scale and ss factors aggregated separately for better numerical stability, per BLAS. +func SumSquaresScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { + scale64, ss64 = stats.Vectorize2Out64(func(idx int, tsr ...*tensor.Indexed) { + VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { + return a - b + }) + }, a, b, out) + return } // SumSquaresFuncOut64 computes the sum of squares differences between tensor values, // and returns the Float64 output values for use in subsequent computations. func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { - scale64, ss64 := Vectorize2Out64(SumSquaresVecFunc, a, b, out) + scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { scale := scale64.Tensor.Float1D(i) @@ -56,10 +55,16 @@ func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { return scale64 } +// SumSquaresFunc computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func SumSquaresFunc(a, b, out *tensor.Indexed) { + SumSquaresFuncOut64(a, b, out) +} + // EuclideanFunc computes the Euclidean square root of the sum of squares // differences between tensor values. func EuclideanFunc(a, b, out *tensor.Indexed) { - scale64, ss64 := Vectorize2Out64(SumSquaresVecFunc, a, b, out) + scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { scale := scale64.Tensor.Float1D(i) @@ -75,23 +80,27 @@ func EuclideanFunc(a, b, out *tensor.Indexed) { } } -// SumSquaresBinTolVecFunc computes the sum-of-squares distance between two vectors, +// SumSquaresBinTolScaleFuncOut64 computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTolVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { - d := a - b - if math.Abs(d) < 0.5 { - d = 0 - } - return d - }) +// returning scale and ss factors aggregated separately for better numerical stability, per BLAS. +func SumSquaresBinTolScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { + scale64, ss64 = stats.Vectorize2Out64(func(idx int, tsr ...*tensor.Indexed) { + VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { + d := a - b + if math.Abs(d) < 0.5 { + d = 0 + } + return d + }) + }, a, b, out) + return } // EuclideanBinTolFunc computes the Euclidean square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { - scale64, ss64 := Vectorize2Out64(SumSquaresBinTolVecFunc, a, b, out) + scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { scale := scale64.Tensor.Float1D(i) @@ -110,7 +119,7 @@ func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { // SumSquaresBinTolFunc computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { - scale64, ss64 := Vectorize2Out64(SumSquaresBinTolVecFunc, a, b, out) + scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { scale := scale64.Tensor.Float1D(i) @@ -126,26 +135,32 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { } } -// InnerProductVecFunc is a Vectorize function for computing the inner product. -func InnerProductVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { - return agg + a*b - }) +// CrossEntropyFunc computes the sum of the co-products of the two on-NaN tensor values. +// See [MetricFunc] for general information. +func CrossEntropyFunc(a, b, out *tensor.Indexed) { + stats.VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { + b = math32.Clamp64(b, 0.000001, 0.999999) + if a >= 1.0 { + agg += -math.Log(b) + } else if a <= 0.0 { + agg += -math.Log(1.0 - b) + } else { + agg += a*math.Log(a/b) + (1-a)*math.Log((1-a)/(1-b)) + } + return agg + }) + }, a, b, out) } // InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. // See [MetricFunc] for general information. func InnerProductFunc(a, b, out *tensor.Indexed) { - VectorizeOut64(InnerProductVecFunc, a, b, out) -} - -// CovarianceVecFunc is a Vectorize function for computing the covariance between -// two vectors, i.e., the mean of the co-product of each vector element minus -// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))] -func CovarianceVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, am, bm, agg float64) float64 { - return agg + (a-am)*(b-bm) - }) + stats.VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { + return agg + a*b + }) + }, a, b, out) } // CovarianceFunc computes the covariance between two vectors, @@ -154,7 +169,11 @@ func CovarianceVecFunc(idx int, tsr ...*tensor.Indexed) { func CovarianceFunc(a, b, out *tensor.Indexed) { amean, acount := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - cov64 := VectorizeOut64(CovarianceVecFunc, a, b, amean, bmean, out) + cov64 := stats.VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, am, bm, agg float64) float64 { + return agg + (a-am)*(b-bm) + }) + }, a, b, amean, bmean, out) nsub := out.Tensor.Len() for i := range nsub { c := acount.Tensor.Float1D(i) @@ -166,33 +185,27 @@ func CovarianceFunc(a, b, out *tensor.Indexed) { } } -// CorrelationVecFunc is a Vectorize function for computing the correlation between -// two vectors, in range (-1..1) as the mean of the co-product of each vector -// element minus the mean of that vector, normalized by the product of their -// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance). -// Equivalent to the cosine of mean-normalized vectors. -func CorrelationVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { - ad := a - am - bd := b - bm - ss += ad * bd // between - avar += ad * ad // within - bvar += bd * bd - return ss, avar, bvar - }) -} - // CorrelationFuncOut64 computes the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized covariance). // Equivalent to the cosine of mean-normalized vectors. +// Returns the Float64 output values for subsequent use. func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { amean, _ := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - ss64, avar64, bvar64 := Vectorize3Out64(CorrelationVecFunc, a, b, amean, bmean, out) + ss64, avar64, bvar64 := Vectorize3Out64(func(idx int, tsr ...*tensor.Indexed) { + Vec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { + ad := a - am + bd := b - bm + ss += ad * bd // between + avar += ad * ad // within + bvar += bd * bd + return ss, avar, bvar + }) + }, a, b, amean, bmean, out) + nsub := out.Tensor.Len() for i := range nsub { ss := ss64.Tensor.Float1D(i) @@ -233,23 +246,18 @@ func InvCorrelationFunc(a, b, out *tensor.Indexed) { } } -// CosineVecFunc is a Vectorize function for computing the cosine between -// two vectors, in range (-1..1) as the normalized inner product: -// inner product / sqrt(ssA * ssB). -func CosineVecFunc(idx int, tsr ...*tensor.Indexed) { - MetricVec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { - ss += a * b - avar += a * a - bvar += b * b - return ss, avar, bvar - }) -} - // CosineFuncOut64 computes the correlation between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { - ss64, avar64, bvar64 := Vectorize3Out64(CosineVecFunc, a, b, out) + ss64, avar64, bvar64 := Vectorize3Out64(func(idx int, tsr ...*tensor.Indexed) { + Vec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { + ss += a * b + avar += a * a + bvar += b * b + return ss, avar, bvar + }) + }, a, b, out) nsub := out.Tensor.Len() for i := range nsub { ss := ss64.Tensor.Float1D(i) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index c597e0184f..c1648d96f3 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -12,11 +12,11 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAll(t *testing.T) { +func TestFuncs(t *testing.T) { a64 := []float64{.5, .2, .1, .7, math.NaN(), .5} b64 := []float64{.2, .9, .1, .7, 0, .2} - results := []float64{math.Sqrt(0.67), 0.67, 0.9, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 0, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} + results := []float64{math.Sqrt(0.67), 0.67, 0.9, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 1.8090248566170337, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64)) btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64)) @@ -30,7 +30,7 @@ func TestAll(t *testing.T) { assert.InDelta(t, results[SumSquares], out.Values[0], 1.0e-8) EuclideanBinTolFunc(atsr, btsr, oix) - assert.Equal(t, results[EuclideanBinTol], out.Values[0]) + assert.InDelta(t, results[EuclideanBinTol], out.Values[0], 1.0e-8) SumSquaresBinTolFunc(atsr, btsr, oix) assert.InDelta(t, results[SumSquaresBinTol], out.Values[0], 1.0e-8) @@ -39,19 +39,22 @@ func TestAll(t *testing.T) { assert.InDelta(t, results[Covariance], out.Values[0], 1.0e-8) CorrelationFunc(atsr, btsr, oix) - assert.Equal(t, results[Correlation], out.Values[0]) + assert.InDelta(t, results[Correlation], out.Values[0], 1.0e-8) InvCorrelationFunc(atsr, btsr, oix) - assert.Equal(t, results[InvCorrelation], out.Values[0]) + assert.InDelta(t, results[InvCorrelation], out.Values[0], 1.0e-8) + + CrossEntropyFunc(atsr, btsr, oix) + assert.InDelta(t, results[CrossEntropy], out.Values[0], 1.0e-8) + + InnerProductFunc(atsr, btsr, oix) + assert.InDelta(t, results[InnerProduct], out.Values[0], 1.0e-8) CosineFunc(atsr, btsr, oix) - assert.Equal(t, results[Cosine], out.Values[0]) + assert.InDelta(t, results[Cosine], out.Values[0], 1.0e-8) InvCosineFunc(atsr, btsr, oix) - assert.Equal(t, results[InvCosine], out.Values[0]) - - InnerProductFunc(atsr, btsr, oix) - assert.Equal(t, results[InnerProduct], out.Values[0]) + assert.InDelta(t, results[InvCosine], out.Values[0], 1.0e-8) /* ab := Abs64(a64, b64) diff --git a/tensor/stats/metric/prob.go b/tensor/stats/metric/prob.go deleted file mode 100644 index 7749a1b1fd..0000000000 --- a/tensor/stats/metric/prob.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package metric - -import ( - "math" - - "cogentcore.org/core/math32" -) - -/////////////////////////////////////////// -// CrossEntropy - -// CrossEntropy32 computes cross-entropy between the two vectors. -// Skips NaN's and panics if lengths are not equal. -func CrossEntropy32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - bv = math32.Max(bv, 0.000001) - bv = math32.Min(bv, 0.999999) - if av >= 1.0 { - ss += -math32.Log(bv) - } else if av <= 0.0 { - ss += -math32.Log(1.0 - bv) - } else { - ss += av*math32.Log(av/bv) + (1-av)*math32.Log((1-av)/(1-bv)) - } - } - return ss -} - -// CrossEntropy64 computes the cross-entropy between the two vectors. -// Skips NaN's and panics if lengths are not equal. -func CrossEntropy64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - bv = math.Max(bv, 0.000001) - bv = math.Min(bv, 0.999999) - if av >= 1.0 { - ss += -math.Log(bv) - } else if av <= 0.0 { - ss += -math.Log(1.0 - bv) - } else { - ss += av*math.Log(av/bv) + (1-av)*math.Log((1-av)/(1-bv)) - } - } - return ss -} diff --git a/tensor/stats/metric/vecfuncs.go b/tensor/stats/metric/vecfuncs.go index 26c2bd7df5..1d9dc1ea20 100644 --- a/tensor/stats/metric/vecfuncs.go +++ b/tensor/stats/metric/vecfuncs.go @@ -11,53 +11,6 @@ import ( "cogentcore.org/core/tensor" ) -// VectorizeOut64 is a version of the [tensor.Vectorize] function -// for metrics, which makes a Float64 output tensor for aggregating -// and computing values, and then copies the results back to the -// original output. This allows metrics functions to operate directly -// on integer valued inputs and produce sensible results. -// It automatically calls NFunc for the nfun function, -// and returns the Float64 output tensor for further processing as needed. -// It uses the _last_ tensor as the output, allowing for multiple inputs. -func VectorizeOut64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) *tensor.Indexed { - n := NFunc(tsr...) - if n <= 0 { - return nil - } - nt := len(tsr) - out := tsr[nt-1] - o64 := tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - etsr := slices.Clone(tsr) - etsr[nt-1] = o64 - for idx := range n { - fun(idx, etsr...) - } - nsub := out.Tensor.Len() - for i := range nsub { - out.Tensor.SetFloat1D(i, o64.Tensor.Float1D(i)) - } - return o64 -} - -// Vectorize2Out64 is a version of the [tensor.Vectorize] function -// for metrics, which makes two Float64 output tensors for aggregating -// and computing values, returning them for final computation. -// It automatically calls NFunc for the nfun function. -func Vectorize2Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2 *tensor.Indexed) { - n := NFunc(tsr...) - if n <= 0 { - return nil, nil - } - nt := len(tsr) - out := tsr[nt-1] - out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - for idx := range n { - fun(idx, tsr[0], tsr[1], out1, out2) - } - return out1, out2 -} - // Vectorize3Out64 is a version of the [tensor.Vectorize] function // for metrics, which makes three Float64 output tensors for aggregating // and computing values, returning them for final computation. @@ -72,7 +25,7 @@ func Vectorize3Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.I out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) out3 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - tsrs := tsr[:nt-1] + tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2, out3) for idx := range n { fun(idx, tsrs...) @@ -105,10 +58,10 @@ func NFunc(tsr ...*tensor.Indexed) int { return min(na, nb) } -// MetricVecFunc is a helper function for metrics functions, dealing with iterating over +// VecFunc is a helper function for metrics functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // It also skips over NaN missing values. -func MetricVecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg float64) float64) { +func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg float64) float64) { nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { @@ -126,11 +79,11 @@ func MetricVecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, } } -// MetricVecSSFunc is a helper function for metric functions, dealing with iterating over +// VecSSFunc is a helper function for metric functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version does sum-of-squares integration over 2 output vectors, // It also skips over NaN missing values. -func MetricVecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(a, b float64) float64) { +func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(a, b float64) float64) { nsub := out2.Tensor.Len() for i := range nsub { if idx == 0 { @@ -162,11 +115,11 @@ func MetricVecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float } } -// MetricVec2inFunc is a helper function for stats functions, dealing with iterating over +// Vec2inFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input vectors, the second input being the output of another stat // e.g., the mean. It also skips over NaN missing values. -func MetricVec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun func(a, b, a2, b2, agg float64) float64) { +func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun func(a, b, a2, b2, agg float64) float64) { nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { @@ -186,11 +139,11 @@ func MetricVec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, f } } -// MetricVec2in3outFunc is a helper function for stats functions, dealing with iterating over +// Vec2in3outFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input, 3 output vectors. The second input being the output of another stat // e.g., the mean. It also skips over NaN missing values. -func MetricVec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, a2, b2, out1, out2, out3 float64) (float64, float64, float64)) { +func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, a2, b2, out1, out2, out3 float64) (float64, float64, float64)) { nsub := out1.Tensor.Len() for i := range nsub { if idx == 0 { @@ -218,10 +171,10 @@ func MetricVec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexe } } -// MetricVec3outFunc is a helper function for stats functions, dealing with iterating over +// Vec3outFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 3 output vectors. It also skips over NaN missing values. -func MetricVec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, out1, out2, out3 float64) (float64, float64, float64)) { +func Vec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, out1, out2, out3 float64) (float64, float64, float64)) { nsub := out1.Tensor.Len() for i := range nsub { if idx == 0 { diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 0ee7792f83..f7a87440d6 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -22,119 +22,105 @@ import ( // if that is needed. type StatsFunc func(in, out *tensor.Indexed) -// CountVecFunc is a Vectorize function for computing the count. -func CountVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + 1 - }) +// CountFuncOut64 computes the count of non-NaN tensor values, +// and returns the Float64 output values for subsequent use. +func CountFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { + return VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + 1 + }) + }, in, out) } // CountFunc computes the count of non-NaN tensor values. // See [StatsFunc] for general information. func CountFunc(in, out *tensor.Indexed) { - VectorizeOut64(CountVecFunc, in, out) + CountFuncOut64(in, out) } -// SumVecFunc is a Vectorize function for computing the sum. -func SumVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + val - }) +// SumFuncOut64 computes the sum of tensor values, +// and returns the Float64 output values for subsequent use. +func SumFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { + return VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + val + }) + }, in, out) } // SumFunc computes the sum of tensor values. // See [StatsFunc] for general information. func SumFunc(in, out *tensor.Indexed) { - VectorizeOut64(SumVecFunc, in, out) -} - -// SumAbsVecFunc is a Vectorize function for computing the sum of abs values. -// This is also known as the L1 norm. -func SumAbsVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { - return agg + math.Abs(val) - }) + SumFuncOut64(in, out) } // SumAbsFunc computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. func SumAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(SumAbsVecFunc, in, out) -} - -// ProdVecFunc is a Vectorize function for computing the product. -func ProdVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { - return agg * val - }) + VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { + return agg + math.Abs(val) + }) + }, in, out) } // ProdFunc computes the product of tensor values. // See [StatsFunc] for general information. func ProdFunc(in, out *tensor.Indexed) { - VectorizeOut64(ProdVecFunc, in, out) -} - -// MinVecFunc is a Vectorize function for computing the min. -func MinVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { - return math.Min(agg, val) - }) + VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { + return agg * val + }) + }, in, out) } // MinFunc computes the min of tensor values. // See [StatsFunc] for general information. func MinFunc(in, out *tensor.Indexed) { - VectorizeOut64(MinVecFunc, in, out) -} - -// MaxVecFunc is a Vectorize function for computing the max. -func MaxVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { - return math.Max(agg, val) - }) + VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, val) + }) + }, in, out) } // MaxFunc computes the max of tensor values. // See [StatsFunc] for general information. func MaxFunc(in, out *tensor.Indexed) { - VectorizeOut64(MaxVecFunc, in, out) -} - -// MinAbsVecFunc is a Vectorize function for computing the min of abs. -func MinAbsVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { - return math.Min(agg, math.Abs(val)) - }) + VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, val) + }) + }, in, out) } // MinAbsFunc computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. func MinAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(MinAbsVecFunc, in, out) -} - -// MaxAbsVecFunc is a Vectorize function for computing the max of abs. -func MaxAbsVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { - return math.Max(agg, math.Abs(val)) - }) + VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { + return math.Min(agg, math.Abs(val)) + }) + }, in, out) } // MaxAbsFunc computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. func MaxAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(MaxAbsVecFunc, in, out) + VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { + return math.Max(agg, math.Abs(val)) + }) + }, in, out) } // MeanFuncOut64 computes the mean of tensor values, -// and returns the Float64 count and mean output values for -// use in subsequent computations. +// and returns the Float64 output values for subsequent use. func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { - sum64 := VectorizeOut64(SumVecFunc, in, out) + sum64 := SumFuncOut64(in, out) count := tensor.NewIndexed(out.Tensor.Clone()) - count64 = VectorizeOut64(CountVecFunc, in, count) + count64 = CountFuncOut64(in, count) nsub := out.Tensor.Len() for i := range nsub { c := count64.Tensor.Float1D(i) @@ -153,21 +139,23 @@ func MeanFunc(in, out *tensor.Indexed) { MeanFuncOut64(in, out) } -// VarVecFunc is a Vectorize function for computing the variance, -// using 3 tensors: in, mean, out. -func VarVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { - dv := val1 - val2 - return agg + dv*dv - }) +// SumSqDevFuncOut64 computes the sum of squared mean deviates of tensor values, +// and returns the Float64 output values for subsequent use. +func SumSqDevFuncOut64(in, out *tensor.Indexed) (ssd64, mean64, count64 *tensor.Indexed) { + mean64, count64 = MeanFuncOut64(in, out) + ssd64 = VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { + dv := val1 - val2 + return agg + dv*dv + }) + }, in, mean64, out) + return } -// VarFuncOut64 computes the sample variance of tensor values. -// and returns the Float64 output values for -// use in subsequent computations. +// VarFuncOut64 computes the sample variance of tensor values, +// and returns the Float64 output values for subsequent use. func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Indexed) { - mean64, count64 = MeanFuncOut64(in, out) - var64 = VectorizeOut64(VarVecFunc, in, mean64, out) + var64, mean64, count64 = SumSqDevFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { c := count64.Tensor.Float1D(i) @@ -215,11 +203,9 @@ func SemFunc(in, out *tensor.Indexed) { } // VarPopFuncOut64 computes the population variance of tensor values. -// and returns the Float64 output values for -// use in subsequent computations. +// and returns the Float64 output values for subsequent use. func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Indexed) { - mean64, count64 = MeanFuncOut64(in, out) - var64 = VectorizeOut64(VarVecFunc, in, mean64, out) + var64, mean64, count64 = SumSqDevFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { c := count64.Tensor.Float1D(i) @@ -266,29 +252,32 @@ func SemPopFunc(in, out *tensor.Indexed) { } } -// SumSqVecFunc is a Vectorize function for computing the sum of squares, -// using 2 separate aggregation tensors. -func SumSqVecFunc(idx int, tsr ...*tensor.Indexed) { - StatVec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { - if val == 0 { +// SumSqScaleFuncOut64 is a helper for sum-of-squares, returning scale and ss +// factors aggregated separately for better numerical stability, per BLAS. +// Returns the Float64 output values for subsequent use. +func SumSqScaleFuncOut64(in, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { + scale64, ss64 = Vectorize2Out64(func(idx int, tsr ...*tensor.Indexed) { + Vec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { + if val == 0 { + return scale, ss + } + absxi := math.Abs(val) + if scale < absxi { + ss = 1 + ss*(scale/absxi)*(scale/absxi) + scale = absxi + } else { + ss = ss + (absxi/scale)*(absxi/scale) + } return scale, ss - } - absxi := math.Abs(val) - if scale < absxi { - ss = 1 + ss*(scale/absxi)*(scale/absxi) - scale = absxi - } else { - ss = ss + (absxi/scale)*(absxi/scale) - } - return scale, ss - }) + }) + }, in, out) + return } // SumSqFuncOut64 computes the sum of squares of tensor values, -// and returns the Float64 output values for -// use in subsequent computations. +// and returns the Float64 output values for subsequent use. func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { - scale64, ss64 := Vectorize2Out64(SumSqVecFunc, in, out) + scale64, ss64 := SumSqScaleFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { scale := scale64.Tensor.Float1D(i) @@ -315,7 +304,7 @@ func SumSqFunc(in, out *tensor.Indexed) { // known as the L2 norm, and returns the Float64 output values for // use in subsequent computations. func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { - scale64, ss64 := Vectorize2Out64(SumSqVecFunc, in, out) + scale64, ss64 := SumSqScaleFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { scale := scale64.Tensor.Float1D(i) diff --git a/tensor/stats/stats/vecfuncs.go b/tensor/stats/stats/vecfuncs.go index 5aa2da98d3..396d439f72 100644 --- a/tensor/stats/stats/vecfuncs.go +++ b/tensor/stats/stats/vecfuncs.go @@ -53,8 +53,10 @@ func Vectorize2Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.I out := tsr[nt-1] out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + tsrs := slices.Clone(tsr[:nt-1]) + tsrs = append(tsrs, out1, out2) for idx := range n { - fun(idx, tsr[0], out1, out2) + fun(idx, tsrs...) } return out1, out2 } @@ -84,10 +86,10 @@ func NFunc(tsr ...*tensor.Indexed) int { return n } -// StatVecFunc is a helper function for stats functions, dealing with iterating over +// VecFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // It also skips over NaN missing values. -func StatVecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { +func VecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { @@ -101,15 +103,12 @@ func StatVecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, ag } } -/////////////////////////////////////////////////////\ -// Two input Tensors - -// StatVec2inFunc is a helper function for stats functions, dealing with iterating over +// Vec2inFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input vectors, the second input being the output of another stat // e.g., the mean for Var. // It also skips over NaN missing values. -func StatVec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(val1, val2, agg float64) float64) { +func Vec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(val1, val2, agg float64) float64) { nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { @@ -124,14 +123,11 @@ func StatVec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun fun } } -/////////////////////////////////////////////////////\ -// Two output Tensors - -// StatVec2outFunc is a helper function for stats functions, dealing with iterating over +// Vec2outFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 output vectors, for separate integration of scale sum squared // It also skips over NaN missing values. -func StatVec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(val, agg1, agg2 float64) (float64, float64)) { +func Vec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(val, agg1, agg2 float64) (float64, float64)) { nsub := out2.Tensor.Len() for i := range nsub { if idx == 0 { From 3f116fc7f2a7b5fe69aa09781f14de2003089bf4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 10 Sep 2024 14:57:43 -0700 Subject: [PATCH 013/311] just need to decide about tolerance func in metric, then done with all docs tests etc --- tensor/base.go | 9 +++ tensor/bits.go | 8 +++ tensor/number.go | 4 ++ tensor/stats/metric/README.md | 30 +++++++++- tensor/stats/metric/abs.go | 90 ------------------------------ tensor/stats/metric/doc.go | 9 +-- tensor/stats/metric/enumgen.go | 10 ++-- tensor/stats/metric/funcs.go | 74 +++++++++++++++++------- tensor/stats/metric/metric_test.go | 49 ++++++++-------- tensor/stats/metric/metrics.go | 58 ++++++++++++++++++- tensor/stats/metric/misc.go | 55 ++++++++++++++++++ tensor/stats/metric/tensor.go | 63 --------------------- tensor/stats/metric/tol.go | 48 ---------------- tensor/stats/metric/vecfuncs.go | 5 +- tensor/stats/stats/README.md | 10 ++++ tensor/stats/stats/funcs.go | 25 +++++---- tensor/stats/stats/stats_test.go | 28 +++++----- tensor/stats/stats/vecfuncs.go | 10 ++-- tensor/string.go | 4 ++ tensor/tensor.go | 16 ++++++ 20 files changed, 313 insertions(+), 292 deletions(-) delete mode 100644 tensor/stats/metric/abs.go create mode 100644 tensor/stats/metric/misc.go delete mode 100644 tensor/stats/metric/tensor.go delete mode 100644 tensor/stats/metric/tol.go diff --git a/tensor/base.go b/tensor/base.go index c3573e75af..c5b84cb9c9 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -66,6 +66,15 @@ func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[i] } func (tsr *Base[T]) Set(i []int, val T) { j := tsr.shape.Offset(i); tsr.Values[j] = val } func (tsr *Base[T]) Set1D(i int, val T) { tsr.Values[i] = val } +// view is implementation of View -- needs final casting +func (tsr *Base[T]) view() *Base[T] { + nw := &Base[T]{} + nw.shape.CopyShape(&tsr.shape) + nw.Values = tsr.Values + nw.Meta = tsr.Meta + return nw +} + // SetShape sets the shape params, resizing backing storage appropriately func (tsr *Base[T]) SetShape(sizes []int, names ...string) { tsr.shape.SetShape(sizes, names...) diff --git a/tensor/bits.go b/tensor/bits.go index 8848e686a9..e4f3e36e50 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -235,6 +235,14 @@ func (tsr *Bits) Clone() Tensor { return csr } +func (tsr *Bits) View() Tensor { + nw := &Bits{} + nw.shape.CopyShape(&tsr.shape) + nw.Values = tsr.Values + nw.Meta = tsr.Meta + return nw +} + // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. diff --git a/tensor/number.go b/tensor/number.go index c5405c9cee..27b73c734a 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -274,6 +274,10 @@ func (tsr *Number[T]) Clone() Tensor { return csr } +func (tsr *Number[T]) View() Tensor { + return &Number[T]{*tsr.view()} +} + // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index f100a0460f..d628925b96 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -2,6 +2,34 @@ `metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Indexed` standard data representation. -The signatures of all such metric functions are identical, captured as types: `metric.Func32` and `metric.Func64` so that other functions that use a metric can take a pointer to any such function. +## Metrics +### Value _increases_ with increasing distance (i.e., difference metric) + +* `Euclidean` or `L2Norm`: the square root of the sum of squares differences between tensor values. +* `SumSquares`: the sum of squares differences between tensor values. +* `Abs`or `L2Norm`: the sum of the absolute value of differences between tensor values. +* `Hamming`: the sum of 1s for every element that is different, i.e., "city block" distance. +* `EuclideanBinTol`: the `Euclidean` square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. +* `SumSquaresBinTol`: the `SumSquares` differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. +* `InvCosine`: is 1-`Cosine`, which is useful to convert it to an Increasing metric where more different vectors have larger metric values. +* `InvCorrelation`: is 1-`Correlation`, which is useful to convert it to an Increasing metric where more different vectors have larger metric values. +* `CrossEntropy`: is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b). + +### Value _decreases_ with increasing distance (i.e., similarity metric) + +* `InnerProduct`: the sum of the co-products of the tensor values. +* `Covariance`: the co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. +* `Correlation`: the standardized `Covariance` in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the `Cosine` of mean-normalized vectors. +* `Cosine`: the high-dimensional angle between two vectors, in range (-1..1) as the normalized `InnerProduct`: inner product / sqrt(ssA * ssB). See also `Correlation`. + +Here is general info about these functions: + +`MetricFunc` is the function signature for a metric function, where the output has the same shape as the inputs but with the outer-most row dimension size of 1, and contains the metric value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. + +Critically, the metric is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ metric for that cell across the different rows. To compute a metric on the `tensor.SubSpace` cells themselves, must call on a `tensor.New1DViewOf` the sub space. See [simat](../simat) package. + +All metric functions skip over NaN's, as a missing value, and use the min of the length of the two tensors. + +Metric functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. diff --git a/tensor/stats/metric/abs.go b/tensor/stats/metric/abs.go deleted file mode 100644 index 7b0dbfe18b..0000000000 --- a/tensor/stats/metric/abs.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package metric - -import ( - "math" - - "cogentcore.org/core/math32" -) - -/////////////////////////////////////////// -// Abs - -// Abs32 computes the sum of absolute value of differences (L1 Norm). -// Skips NaN's and panics if lengths are not equal. -func Abs32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - ss += math32.Abs(av - bv) - } - return ss -} - -// Abs64 computes the sum of absolute value of differences (L1 Norm). -// Skips NaN's and panics if lengths are not equal. -func Abs64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - ss += math.Abs(av - bv) - } - return ss -} - -/////////////////////////////////////////// -// Hamming - -// Hamming32 computes the sum of 1's for every element that is different -// (city block). -// Skips NaN's and panics if lengths are not equal. -func Hamming32(a, b []float32) float32 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float32(0) - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - if av != bv { - ss += 1 - } - } - return ss -} - -// Hamming64 computes the sum of absolute value of differences (L1 Norm). -// Skips NaN's and panics if lengths are not equal. -func Hamming64(a, b []float64) float64 { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - ss := float64(0) - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - if av != bv { - ss += 1 - } - } - return ss -} diff --git a/tensor/stats/metric/doc.go b/tensor/stats/metric/doc.go index 724f01572b..7c7496dd4c 100644 --- a/tensor/stats/metric/doc.go +++ b/tensor/stats/metric/doc.go @@ -3,13 +3,6 @@ // license that can be found in the LICENSE file. /* -Package metric provides various similarity / distance metrics for comparing -floating-point vectors. -All functions have 32 and 64 bit variants, and skip NaN's (often used for missing) -and will panic if the lengths of the two slices are unequal (no error return). - -The signatures of all such metric functions are identical, captured as types: -metric.Func32 and metric.Func64 so that other functions that use a metric -can take a pointer to any such function. +Package metric provides various similarity / distance metrics for comparing tensors, operating on the tensor.Indexed standard data representation. */ package metric diff --git a/tensor/stats/metric/enumgen.go b/tensor/stats/metric/enumgen.go index e1743df0cc..1b14626b5a 100644 --- a/tensor/stats/metric/enumgen.go +++ b/tensor/stats/metric/enumgen.go @@ -6,16 +6,16 @@ import ( "cogentcore.org/core/enums" ) -var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} +var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14} // MetricsN is the highest valid value for type Metrics, plus one. -const MetricsN Metrics = 13 +const MetricsN Metrics = 15 -var _MetricsValueMap = map[string]Metrics{`Euclidean`: 0, `SumSquares`: 1, `Abs`: 2, `Hamming`: 3, `EuclideanBinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `InnerProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} +var _MetricsValueMap = map[string]Metrics{`Euclidean`: 0, `L2Norm`: 1, `SumSquares`: 2, `Abs`: 3, `L1Norm`: 4, `Hamming`: 5, `EuclideanBinTol`: 6, `SumSquaresBinTol`: 7, `InvCosine`: 8, `InvCorrelation`: 9, `CrossEntropy`: 10, `InnerProduct`: 11, `Covariance`: 12, `Correlation`: 13, `Cosine`: 14} -var _MetricsDescMap = map[Metrics]string{0: ``, 1: ``, 2: ``, 3: ``, 4: ``, 5: ``, 6: `InvCosine is 1-Cosine -- useful to convert into an Increasing metric`, 7: `InvCorrelation is 1-Correlation -- useful to convert into an Increasing metric`, 8: ``, 9: `Everything below here is !Increasing -- larger = closer, not farther`, 10: ``, 11: ``, 12: ``} +var _MetricsDescMap = map[Metrics]string{0: `Euclidean is the square root of the sum of squares differences between tensor values, aka the [L2Norm].`, 1: `L2Norm is the square root of the sum of squares differences between tensor values, aka [Euclidean] distance.`, 2: `SumSquares is the sum of squares differences between tensor values.`, 3: `Abs is the sum of the absolute value of differences between tensor values, aka the [L1Norm].`, 4: `L1Norm is the sum of the absolute value of differences between tensor values (same as [Abs]).`, 5: `Hamming computes the sum of 1s for every element that is different, i.e., "city block" distance.`, 6: `EuclideanBinTol computes the [Euclidean] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 7: `SumSquaresBinTol computes the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 8: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 9: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 10: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 11: `InnerProduct is the sum of the co-products of the tensor values.`, 12: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 13: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 14: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [InnerProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} -var _MetricsMap = map[Metrics]string{0: `Euclidean`, 1: `SumSquares`, 2: `Abs`, 3: `Hamming`, 4: `EuclideanBinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `InnerProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} +var _MetricsMap = map[Metrics]string{0: `Euclidean`, 1: `L2Norm`, 2: `SumSquares`, 3: `Abs`, 4: `L1Norm`, 5: `Hamming`, 6: `EuclideanBinTol`, 7: `SumSquaresBinTol`, 8: `InvCosine`, 9: `InvCorrelation`, 10: `CrossEntropy`, 11: `InnerProduct`, 12: `Covariance`, 13: `Correlation`, 14: `Cosine`} // String returns the string representation of this Metrics value. func (i Metrics) String() string { return enums.String(i, _MetricsMap) } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index b678679851..9a757769c2 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -17,7 +17,13 @@ import ( // the outer-most row dimension size of 1, and contains // the metric value(s) for the "cells" in higher-dimensional tensors, // and a single scalar value for a 1D input tensor. -// All metric functions skip over NaN's, as a missing value. +// Critically, the metric is always computed over the outer row dimension, +// so each cell in a higher-dimensional output reflects the _row-wise_ +// metric for that cell across the different rows. To compute a metric +// on the [tensor.SubSpace] cells themselves, must call on a +// [tensor.New1DViewOf] the sub space. See [simat] package. +// All metric functions skip over NaN's, as a missing value, +// and use the min of the length of the two tensors. // Metric functions cannot be computed in parallel, // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required @@ -27,7 +33,7 @@ type MetricFunc func(a, b, out *tensor.Indexed) // SumSquaresScaleFuncOut64 computes the sum of squares differences between tensor values, // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. func SumSquaresScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { - scale64, ss64 = stats.Vectorize2Out64(func(idx int, tsr ...*tensor.Indexed) { + scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { return a - b }) @@ -62,7 +68,7 @@ func SumSquaresFunc(a, b, out *tensor.Indexed) { } // EuclideanFunc computes the Euclidean square root of the sum of squares -// differences between tensor values. +// differences between tensor values, aka the L2 Norm. func EuclideanFunc(a, b, out *tensor.Indexed) { scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() @@ -80,11 +86,36 @@ func EuclideanFunc(a, b, out *tensor.Indexed) { } } +// AbsFunc computes the sum of the absolute value of differences between the +// tensor values, aka the L1 Norm. +// See [MetricFunc] for general information. +func AbsFunc(a, b, out *tensor.Indexed) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { + return agg + math.Abs(a-b) + }) + }, a, b, out) +} + +// HammingFunc computes the sum of 1s for every element that is different, +// i.e., "city block" distance. +// See [MetricFunc] for general information. +func HammingFunc(a, b, out *tensor.Indexed) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { + VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { + if a != b { + agg += 1 + } + return agg + }) + }, a, b, out) +} + // SumSquaresBinTolScaleFuncOut64 computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. func SumSquaresBinTolScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { - scale64, ss64 = stats.Vectorize2Out64(func(idx int, tsr ...*tensor.Indexed) { + scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { d := a - b if math.Abs(d) < 0.5 { @@ -135,10 +166,15 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { } } -// CrossEntropyFunc computes the sum of the co-products of the two on-NaN tensor values. +// CrossEntropyFunc is a standard measure of the difference between two +// probabilty distributions, reflecting the additional entropy (uncertainty) associated +// with measuring probabilities under distribution b when in fact they come from +// distribution a. It is also the entropy of a plus the divergence between a from b, +// using Kullback-Leibler (KL) divergence. It is computed as: +// a * log(a/b) + (1-a) * log(1-a/1-b). // See [MetricFunc] for general information. func CrossEntropyFunc(a, b, out *tensor.Indexed) { - stats.VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { b = math32.Clamp64(b, 0.000001, 0.999999) if a >= 1.0 { @@ -156,20 +192,20 @@ func CrossEntropyFunc(a, b, out *tensor.Indexed) { // InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. // See [MetricFunc] for general information. func InnerProductFunc(a, b, out *tensor.Indexed) { - stats.VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + a*b }) }, a, b, out) } -// CovarianceFunc computes the covariance between two vectors, +// CovarianceFunc computes the co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. func CovarianceFunc(a, b, out *tensor.Indexed) { amean, acount := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - cov64 := stats.VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + cov64 := stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, am, bm, agg float64) float64 { return agg + (a-am)*(b-bm) }) @@ -195,7 +231,7 @@ func CovarianceFunc(a, b, out *tensor.Indexed) { func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { amean, _ := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - ss64, avar64, bvar64 := Vectorize3Out64(func(idx int, tsr ...*tensor.Indexed) { + ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { Vec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { ad := a - am bd := b - bm @@ -223,8 +259,8 @@ func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized covariance). -// Equivalent to the cosine of mean-normalized vectors. +// (i.e., the standardized [CovarianceFunc]). +// Equivalent to the [CosineFunc] of mean-normalized vectors. func CorrelationFunc(a, b, out *tensor.Indexed) { CorrelationFuncOut64(a, b, out) } @@ -234,7 +270,7 @@ func CorrelationFunc(a, b, out *tensor.Indexed) { // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized covariance). -// Equivalent to the cosine of mean-normalized vectors. +// Equivalent to the [CosineFunc] of mean-normalized vectors. // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. func InvCorrelationFunc(a, b, out *tensor.Indexed) { @@ -246,11 +282,11 @@ func InvCorrelationFunc(a, b, out *tensor.Indexed) { } } -// CosineFuncOut64 computes the correlation between two vectors, -// in range (-1..1) as the normalized inner product: -// inner product / sqrt(ssA * ssB). +// CosineFuncOut64 computes the high-dimensional angle between two vectors, +// in range (-1..1) as the normalized [InnerProductFunc]: +// inner product / sqrt(ssA * ssB). See also [CorrelationFunc]. func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { - ss64, avar64, bvar64 := Vectorize3Out64(func(idx int, tsr ...*tensor.Indexed) { + ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { Vec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { ss += a * b avar += a * a @@ -271,9 +307,9 @@ func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { return ss64 } -// CosineFunc computes the correlation between two vectors, +// CosineFunc computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized inner product: -// inner product / sqrt(ssA * ssB). +// inner product / sqrt(ssA * ssB). See also [CorrelationFunc] func CosineFunc(a, b, out *tensor.Indexed) { CosineFuncOut64(a, b, out) } diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index c1648d96f3..67a052710b 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -16,7 +16,9 @@ func TestFuncs(t *testing.T) { a64 := []float64{.5, .2, .1, .7, math.NaN(), .5} b64 := []float64{.2, .9, .1, .7, 0, .2} - results := []float64{math.Sqrt(0.67), 0.67, 0.9, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 1.8090248566170337, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} + results := []float64{math.Sqrt(0.67), math.Sqrt(0.67), 0.67, 1.3, 1.3, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 1.8090248566170337, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} + + tol := 1.0e-8 atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64)) btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64)) @@ -24,47 +26,46 @@ func TestFuncs(t *testing.T) { oix := tensor.NewIndexed(out) EuclideanFunc(atsr, btsr, oix) - assert.InDelta(t, results[Euclidean], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Euclidean], out.Values[0], tol) SumSquaresFunc(atsr, btsr, oix) - assert.InDelta(t, results[SumSquares], out.Values[0], 1.0e-8) + assert.InDelta(t, results[SumSquares], out.Values[0], tol) EuclideanBinTolFunc(atsr, btsr, oix) - assert.InDelta(t, results[EuclideanBinTol], out.Values[0], 1.0e-8) + assert.InDelta(t, results[EuclideanBinTol], out.Values[0], tol) + + AbsFunc(atsr, btsr, oix) + assert.InDelta(t, results[Abs], out.Values[0], tol) + + HammingFunc(atsr, btsr, oix) + assert.Equal(t, results[Hamming], out.Values[0]) SumSquaresBinTolFunc(atsr, btsr, oix) - assert.InDelta(t, results[SumSquaresBinTol], out.Values[0], 1.0e-8) + assert.InDelta(t, results[SumSquaresBinTol], out.Values[0], tol) CovarianceFunc(atsr, btsr, oix) - assert.InDelta(t, results[Covariance], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Covariance], out.Values[0], tol) CorrelationFunc(atsr, btsr, oix) - assert.InDelta(t, results[Correlation], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Correlation], out.Values[0], tol) InvCorrelationFunc(atsr, btsr, oix) - assert.InDelta(t, results[InvCorrelation], out.Values[0], 1.0e-8) + assert.InDelta(t, results[InvCorrelation], out.Values[0], tol) CrossEntropyFunc(atsr, btsr, oix) - assert.InDelta(t, results[CrossEntropy], out.Values[0], 1.0e-8) + assert.InDelta(t, results[CrossEntropy], out.Values[0], tol) InnerProductFunc(atsr, btsr, oix) - assert.InDelta(t, results[InnerProduct], out.Values[0], 1.0e-8) + assert.InDelta(t, results[InnerProduct], out.Values[0], tol) CosineFunc(atsr, btsr, oix) - assert.InDelta(t, results[Cosine], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Cosine], out.Values[0], tol) InvCosineFunc(atsr, btsr, oix) - assert.InDelta(t, results[InvCosine], out.Values[0], 1.0e-8) - - /* - ab := Abs64(a64, b64) - if ab != 0.8999999999999999 { - t.Errorf("Abs64: %g\n", ab) - } - - hm := Hamming64(a64, b64) - if hm != 3 { - t.Errorf("Hamming64: %g\n", hm) - } - */ + assert.InDelta(t, results[InvCosine], out.Values[0], tol) + + for met := Euclidean; met < MetricsN; met++ { + Standard(met, atsr, btsr, oix) + assert.InDelta(t, results[met], out.Values[0], tol) + } } diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 4cf4b15296..c1f8a45ecf 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -20,12 +20,17 @@ var Funcs map[string]MetricFunc func init() { Funcs = make(map[string]MetricFunc) Funcs[Euclidean.String()] = EuclideanFunc + Funcs[L2Norm.String()] = EuclideanFunc Funcs[SumSquares.String()] = SumSquaresFunc + Funcs[Abs.String()] = AbsFunc + Funcs[L1Norm.String()] = AbsFunc + Funcs[Hamming.String()] = HammingFunc Funcs[EuclideanBinTol.String()] = EuclideanBinTolFunc Funcs[SumSquaresBinTol.String()] = SumSquaresBinTolFunc Funcs[InvCosine.String()] = InvCosineFunc Funcs[InvCorrelation.String()] = InvCorrelationFunc Funcs[InnerProduct.String()] = InnerProductFunc + Funcs[CrossEntropy.String()] = CrossEntropyFunc Funcs[Covariance.String()] = CovarianceFunc Funcs[Correlation.String()] = CorrelationFunc Funcs[Cosine.String()] = CosineFunc @@ -53,26 +58,75 @@ func Call(name string, a, b, out *tensor.Indexed) error { type Metrics int32 //enums:enum const ( + // Euclidean is the square root of the sum of squares differences + // between tensor values, aka the [L2Norm]. Euclidean Metrics = iota + + // L2Norm is the square root of the sum of squares differences + // between tensor values, aka [Euclidean] distance. + L2Norm + + // SumSquares is the sum of squares differences between tensor values. SumSquares + + // Abs is the sum of the absolute value of differences + // between tensor values, aka the [L1Norm]. Abs + + // L1Norm is the sum of the absolute value of differences + // between tensor values (same as [Abs]). + L1Norm + + // Hamming is the sum of 1s for every element that is different, + // i.e., "city block" distance. Hamming + // EuclideanBinTol is the [Euclidean] square root of the sum of squares + // differences between tensor values, with binary tolerance: + // differences < 0.5 are thresholded to 0. EuclideanBinTol + + // SumSquaresBinTol is the [SumSquares] differences between tensor values, + // with binary tolerance: differences < 0.5 are thresholded to 0. SumSquaresBinTol - // InvCosine is 1-Cosine -- useful to convert into an Increasing metric + // InvCosine is 1-[Cosine], which is useful to convert it + // to an Increasing metric where more different vectors have larger metric values. InvCosine - // InvCorrelation is 1-Correlation -- useful to convert into an Increasing metric + // InvCorrelation is 1-[Correlation], which is useful to convert it + // to an Increasing metric where more different vectors have larger metric values. InvCorrelation + // CrossEntropy is a standard measure of the difference between two + // probabilty distributions, reflecting the additional entropy (uncertainty) associated + // with measuring probabilities under distribution b when in fact they come from + // distribution a. It is also the entropy of a plus the divergence between a from b, + // using Kullback-Leibler (KL) divergence. It is computed as: + // a * log(a/b) + (1-a) * log(1-a/1-b). CrossEntropy + ///////////////////////////////////////////////////////////////////////// // Everything below here is !Increasing -- larger = closer, not farther + + // InnerProduct is the sum of the co-products of the tensor values. InnerProduct + + // Covariance is co-variance between two vectors, + // i.e., the mean of the co-product of each vector element minus + // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. Covariance + + // Correlation is the standardized [Covariance] in the range (-1..1), + // computed as the mean of the co-product of each vector + // element minus the mean of that vector, normalized by the product of their + // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). + // Equivalent to the [Cosine] of mean-normalized vectors. Correlation + + // Cosine is high-dimensional angle between two vectors, + // in range (-1..1) as the normalized [InnerProduct]: + // inner product / sqrt(ssA * ssB). See also [Correlation]. Cosine ) diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go new file mode 100644 index 0000000000..f6fd8c753b --- /dev/null +++ b/tensor/stats/metric/misc.go @@ -0,0 +1,55 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metric + +import ( + "math" + + "cogentcore.org/core/tensor" +) + +// ClosestRow returns the closest fit between probe pattern and patterns in +// a "vocabulary" tensor with outer-most row dimension, using given metric +// function, *which must have the Increasing property*, i.e., larger = further. +// returns the row and metric value for that row. +// note: this does _not_ use any existing Indexes for the probe (but does for the vocab). +func ClosestRow(probe *tensor.Indexed, vocab *tensor.Indexed, mfun MetricFunc) (int, float64) { + rows, _ := vocab.Tensor.RowCellSize() + mi := -1 + out := tensor.NewFloat64([]int{1}) + oi := tensor.NewIndexed(out) + // todo: need a 1d view of both spaces + mind := math.MaxFloat64 + prview := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) + for ri := range rows { + sub := tensor.NewIndexed(tensor.New1DViewOf(vocab.Tensor.SubSpace([]int{vocab.Index(ri)}))) + mfun(prview, sub, oi) + d := out.Values[0] + if d < mind { + mi = ri + mind = d + } + } + return mi, mind +} + +// todo: + +// Tolerance64 sets a = b for any element where |a-b| <= tol. +// This can be called prior to any metric function. +func Tolerance64(a, b []float64, tol float64) { + if len(a) != len(b) { + panic("metric: slice lengths do not match") + } + for i, av := range a { + bv := b[i] + if math.IsNaN(av) || math.IsNaN(bv) { + continue + } + if math.Abs(av-bv) <= tol { + a[i] = bv + } + } +} diff --git a/tensor/stats/metric/tensor.go b/tensor/stats/metric/tensor.go deleted file mode 100644 index 9e650e18b8..0000000000 --- a/tensor/stats/metric/tensor.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package metric - -/* -// ClosestRow32 returns the closest fit between probe pattern and patterns in -// an tensor with float32 data where the outer-most dimension is assumed to be a row -// (e.g., as a column in an table), using the given metric function, -// *which must have the Increasing property* -- i.e., larger = further. -// returns the row and metric value for that row. -// Col cell sizes must match size of probe (panics if not). -func ClosestRow32(probe tensor.Tensor, col tensor.Tensor, mfun Func32) (int, float32) { - pr := probe.(*tensor.Float32) - cl := col.(*tensor.Float32) - rows := col.Shape().DimSize(0) - csz := col.Len() / rows - if csz != probe.Len() { - panic("metric.ClosestRow32: probe size != cell size of tensor column!\n") - } - ci := -1 - minv := float32(math.MaxFloat32) - for ri := 0; ri < rows; ri++ { - st := ri * csz - rvals := cl.Values[st : st+csz] - v := mfun(pr.Values, rvals) - if v < minv { - ci = ri - minv = v - } - } - return ci, minv -} - -// ClosestRow64 returns the closest fit between probe pattern and patterns in -// a tensor with float64 data where the outer-most dimension is assumed to be a row -// (e.g., as a column in an table), using the given metric function, -// *which must have the Increasing property* -- i.e., larger = further. -// returns the row and metric value for that row. -// Col cell sizes must match size of probe (panics if not). -func ClosestRow64(probe tensor.Tensor, col tensor.Tensor, mfun Func64) (int, float64) { - pr := probe.(*tensor.Float64) - cl := col.(*tensor.Float64) - rows := col.DimSize(0) - csz := col.Len() / rows - if csz != probe.Len() { - panic("metric.ClosestRow64: probe size != cell size of tensor column!\n") - } - ci := -1 - minv := math.MaxFloat64 - for ri := 0; ri < rows; ri++ { - st := ri * csz - rvals := cl.Values[st : st+csz] - v := mfun(pr.Values, rvals) - if v < minv { - ci = ri - minv = v - } - } - return ci, minv -} -*/ diff --git a/tensor/stats/metric/tol.go b/tensor/stats/metric/tol.go deleted file mode 100644 index a99058561f..0000000000 --- a/tensor/stats/metric/tol.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package metric - -import ( - "math" - - "cogentcore.org/core/math32" -) - -/////////////////////////////////////////// -// Tolerance - -// Tolerance32 sets a = b for any element where |a-b| <= tol. -// This can be called prior to any metric function. -func Tolerance32(a, b []float32, tol float32) { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - for i, av := range a { - bv := b[i] - if math32.IsNaN(av) || math32.IsNaN(bv) { - continue - } - if math32.Abs(av-bv) <= tol { - a[i] = bv - } - } -} - -// Tolerance64 sets a = b for any element where |a-b| <= tol. -// This can be called prior to any metric function. -func Tolerance64(a, b []float64, tol float64) { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - if math.Abs(av-bv) <= tol { - a[i] = bv - } - } -} diff --git a/tensor/stats/metric/vecfuncs.go b/tensor/stats/metric/vecfuncs.go index 1d9dc1ea20..6046dd54a6 100644 --- a/tensor/stats/metric/vecfuncs.go +++ b/tensor/stats/metric/vecfuncs.go @@ -14,9 +14,8 @@ import ( // Vectorize3Out64 is a version of the [tensor.Vectorize] function // for metrics, which makes three Float64 output tensors for aggregating // and computing values, returning them for final computation. -// It automatically calls NFunc for the nfun function. -func Vectorize3Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2, out3 *tensor.Indexed) { - n := NFunc(tsr...) +func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2, out3 *tensor.Indexed) { + n := nfunc(tsr...) if n <= 0 { return nil, nil, nil } diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index 4e07484bf5..78dbd0f406 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -34,6 +34,16 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Median`: middle value in sorted ordering (only for Indexed) * `Q1`: Q1 first quartile = 25%ile value = .25 quantile value (only for Indexed) * `Q3`: Q3 third quartile = 75%ile value = .75 quantile value (only for Indexed) + +Here is the general info associated with these function calls: + +`StatsFunc` is the function signature for a stats function, where the output has the same shape as the input but with the outer-most row dimension size of 1, and contains the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. + +Critically, the stat is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ stat for that cell across the different rows. To compute a stat on the `tensor.SubSpace` cells themselves, must call on a [tensor.New1DViewOf] the sub space. + +All stats functions skip over NaN's, as a missing value. + +Stats functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. ## Vectorize functions diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index f7a87440d6..2e187333b7 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -15,6 +15,11 @@ import ( // the outer-most row dimension size of 1, and contains // the stat value(s) for the "cells" in higher-dimensional tensors, // and a single scalar value for a 1D input tensor. +// Critically, the stat is always computed over the outer row dimension, +// so each cell in a higher-dimensional output reflects the _row-wise_ +// stat for that cell across the different rows. To compute a stat +// on the [tensor.SubSpace] cells themselves, must call on a +// [tensor.New1DViewOf] the sub space. // All stats functions skip over NaN's, as a missing value. // Stats functions cannot be computed in parallel, // e.g., using VectorizeThreaded or GPU, due to shared writing @@ -25,7 +30,7 @@ type StatsFunc func(in, out *tensor.Indexed) // CountFuncOut64 computes the count of non-NaN tensor values, // and returns the Float64 output values for subsequent use. func CountFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { - return VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + return VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + 1 }) @@ -41,7 +46,7 @@ func CountFunc(in, out *tensor.Indexed) { // SumFuncOut64 computes the sum of tensor values, // and returns the Float64 output values for subsequent use. func SumFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { - return VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + return VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + val }) @@ -58,7 +63,7 @@ func SumFunc(in, out *tensor.Indexed) { // This is also known as the L1 norm. // See [StatsFunc] for general information. func SumAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) }) @@ -68,7 +73,7 @@ func SumAbsFunc(in, out *tensor.Indexed) { // ProdFunc computes the product of tensor values. // See [StatsFunc] for general information. func ProdFunc(in, out *tensor.Indexed) { - VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { return agg * val }) @@ -78,7 +83,7 @@ func ProdFunc(in, out *tensor.Indexed) { // MinFunc computes the min of tensor values. // See [StatsFunc] for general information. func MinFunc(in, out *tensor.Indexed) { - VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, val) }) @@ -88,7 +93,7 @@ func MinFunc(in, out *tensor.Indexed) { // MaxFunc computes the max of tensor values. // See [StatsFunc] for general information. func MaxFunc(in, out *tensor.Indexed) { - VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, val) }) @@ -98,7 +103,7 @@ func MaxFunc(in, out *tensor.Indexed) { // MinAbsFunc computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. func MinAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, math.Abs(val)) }) @@ -108,7 +113,7 @@ func MinAbsFunc(in, out *tensor.Indexed) { // MaxAbsFunc computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. func MaxAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, math.Abs(val)) }) @@ -143,7 +148,7 @@ func MeanFunc(in, out *tensor.Indexed) { // and returns the Float64 output values for subsequent use. func SumSqDevFuncOut64(in, out *tensor.Indexed) (ssd64, mean64, count64 *tensor.Indexed) { mean64, count64 = MeanFuncOut64(in, out) - ssd64 = VectorizeOut64(func(idx int, tsr ...*tensor.Indexed) { + ssd64 = VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { dv := val1 - val2 return agg + dv*dv @@ -256,7 +261,7 @@ func SemPopFunc(in, out *tensor.Indexed) { // factors aggregated separately for better numerical stability, per BLAS. // Returns the Float64 output values for subsequent use. func SumSqScaleFuncOut64(in, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { - scale64, ss64 = Vectorize2Out64(func(idx int, tsr ...*tensor.Indexed) { + scale64, ss64 = Vectorize2Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { Vec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { if val == 0 { return scale, ss diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index 4cca3404e6..f02b3e395b 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -20,6 +20,8 @@ func TestFuncs64(t *testing.T) { results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} + tol := 1.0e-8 + CountFunc(ix, oix) assert.Equal(t, results[Count], out.Values[0]) @@ -48,41 +50,41 @@ func TestFuncs64(t *testing.T) { assert.Equal(t, results[Mean], out.Values[0]) VarFunc(ix, oix) - assert.InDelta(t, results[Var], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Var], out.Values[0], tol) StdFunc(ix, oix) - assert.InDelta(t, results[Std], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Std], out.Values[0], tol) SemFunc(ix, oix) - assert.InDelta(t, results[Sem], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Sem], out.Values[0], tol) VarPopFunc(ix, oix) - assert.InDelta(t, results[VarPop], out.Values[0], 1.0e-8) + assert.InDelta(t, results[VarPop], out.Values[0], tol) StdPopFunc(ix, oix) - assert.InDelta(t, results[StdPop], out.Values[0], 1.0e-8) + assert.InDelta(t, results[StdPop], out.Values[0], tol) SemPopFunc(ix, oix) - assert.InDelta(t, results[SemPop], out.Values[0], 1.0e-8) + assert.InDelta(t, results[SemPop], out.Values[0], tol) SumSqFunc(ix, oix) - assert.InDelta(t, results[SumSq], out.Values[0], 1.0e-8) + assert.InDelta(t, results[SumSq], out.Values[0], tol) L2NormFunc(ix, oix) - assert.InDelta(t, results[L2Norm], out.Values[0], 1.0e-8) + assert.InDelta(t, results[L2Norm], out.Values[0], tol) MedianFunc(ix, oix) - assert.InDelta(t, results[Median], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Median], out.Values[0], tol) Q1Func(ix, oix) - assert.InDelta(t, results[Q1], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Q1], out.Values[0], tol) Q3Func(ix, oix) - assert.InDelta(t, results[Q3], out.Values[0], 1.0e-8) + assert.InDelta(t, results[Q3], out.Values[0], tol) - for stat := Count; stat <= Q3; stat++ { + for stat := Count; stat < StatsN; stat++ { Standard(stat, ix, oix) - assert.InDelta(t, results[stat], out.Values[0], 1.0e-8) + assert.InDelta(t, results[stat], out.Values[0], tol) } } diff --git a/tensor/stats/stats/vecfuncs.go b/tensor/stats/stats/vecfuncs.go index 396d439f72..6743258c51 100644 --- a/tensor/stats/stats/vecfuncs.go +++ b/tensor/stats/stats/vecfuncs.go @@ -16,12 +16,11 @@ import ( // and computing values, and then copies the results back to the // original output. This allows stats functions to operate directly // on integer valued inputs and produce sensible results. -// It automatically calls NFunc for the nfun function, // and returns the Float64 output tensor for further processing as needed. // It uses the _last_ tensor as the output, allowing for multiple inputs, // as in the case of VarVecFun. -func VectorizeOut64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) *tensor.Indexed { - n := NFunc(tsr...) +func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) *tensor.Indexed { + n := nfunc(tsr...) if n <= 0 { return nil } @@ -43,9 +42,8 @@ func VectorizeOut64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.In // Vectorize2Out64 is a version of the [tensor.Vectorize] function // for stats, which makes two Float64 output tensors for aggregating // and computing values, returning them for final computation. -// It automatically calls NFunc for the nfun function. -func Vectorize2Out64(fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2 *tensor.Indexed) { - n := NFunc(tsr...) +func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2 *tensor.Indexed) { + n := nfunc(tsr...) if n <= 0 { return nil, nil } diff --git a/tensor/string.go b/tensor/string.go index 884c810abc..0316ee75da 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -229,6 +229,10 @@ func (tsr *String) Clone() Tensor { return csr } +func (tsr *String) View() Tensor { + return &String{*tsr.view()} +} + // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. diff --git a/tensor/tensor.go b/tensor/tensor.go index fae9440733..2d0205343a 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -141,6 +141,12 @@ type Tensor interface { // that as a Tensor (which can be converted into the known type as needed). Clone() Tensor + // View clones this tensor, *keeping the same underlying Values slice*, + // instead of making a copy like Clone() does. The main point of this + // is to then change the shape of the view to provide a different way + // of accessing the same data. See [New1DViewOf] for example. + View() Tensor + // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. @@ -213,6 +219,16 @@ func NewOfType(typ reflect.Kind, sizes []int, names ...string) Tensor { } } +// New1DViewOf returns a 1D view into the given tensor, using the same +// underlying values, and just changing the shape to a 1D view. +// This can be useful e.g., for stats and metric functions that report +// on the 1D list of values. +func New1DViewOf(tsr Tensor) Tensor { + vw := tsr.View() + vw.SetShape([]int{tsr.Len()}) + return vw +} + // CopyDense copies a gonum mat.Dense matrix into given Tensor // using standard Float64 interface func CopyDense(to Tensor, dm *mat.Dense) { From 5bc8bd47b3b516888e0ad94df55ca606b72867d5 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 10 Sep 2024 21:32:22 -0700 Subject: [PATCH 014/311] tmath start, with major fixes to vectorize threading -- need bench tests for that. --- tensor/indexed.go | 51 +++-- tensor/shape.go | 7 +- tensor/stats/metric/{vecfuncs.go => vec.go} | 0 tensor/stats/norm/README.md | 10 - tensor/stats/norm/abs.go | 53 ----- tensor/stats/norm/doc.go | 13 -- tensor/stats/norm/norm.go | 223 -------------------- tensor/stats/norm/norm_test.go | 81 ------- tensor/stats/norm/tensor.go | 82 ------- tensor/stats/stats/{vecfuncs.go => vec.go} | 0 tensor/stats/tmath/math.go | 102 +++++++++ tensor/stats/tmath/tmath.go | 23 ++ tensor/stats/tmath/tmath_test.go | 105 +++++++++ tensor/tensor.go | 4 +- tensor/vectorize.go | 70 +++++- 15 files changed, 340 insertions(+), 484 deletions(-) rename tensor/stats/metric/{vecfuncs.go => vec.go} (100%) delete mode 100644 tensor/stats/norm/README.md delete mode 100644 tensor/stats/norm/abs.go delete mode 100644 tensor/stats/norm/doc.go delete mode 100644 tensor/stats/norm/norm.go delete mode 100644 tensor/stats/norm/norm_test.go delete mode 100644 tensor/stats/norm/tensor.go rename tensor/stats/stats/{vecfuncs.go => vec.go} (100%) create mode 100644 tensor/stats/tmath/math.go create mode 100644 tensor/stats/tmath/tmath.go create mode 100644 tensor/stats/tmath/tmath_test.go diff --git a/tensor/indexed.go b/tensor/indexed.go index 7f50b71f20..1e4f33ec0e 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -30,7 +30,6 @@ import ( // tensor data do not indirect through the indexes and must be called directly // on the [Tensor]. type Indexed struct { //types:add - // Tensor that we are an indexed view onto. Tensor Tensor @@ -61,7 +60,19 @@ func (ix *Indexed) SetTensor(tsr Tensor) { ix.Sequential() } -// Index returns the actual index into underlying tensor based on given +// SetShapeFrom sets our shape from given source, calling +// [Tensor.SetShape] with the shape params from source, +// and copying the indexes if present. +func (ix *Indexed) SetShapeFrom(src *Indexed) { + ix.Tensor.SetShapeFrom(src.Tensor) + if src.Indexes == nil { + ix.Indexes = nil + } else { + ix.Indexes = slices.Clone(src.Indexes) + } +} + +// Index returns the actual index into underlying tensor row based on given // index value. If Indexes == nil, index is passed through. func (ix *Indexed) Index(idx int) int { if ix.Indexes == nil { @@ -70,6 +81,16 @@ func (ix *Indexed) Index(idx int) int { return ix.Indexes[idx] } +// RowCellIndex returns the direct Values index into underlying tensor +// based on given overall row * cell index. +func (ix *Indexed) RowCellIndex(idx int) (i1d, ri, ci int) { + _, cells := ix.Tensor.RowCellSize() + ri = idx / cells + ci = idx % cells + i1d = ix.Index(ri)*cells + ci + return +} + // Len returns the length of the index list or number of outer rows dimension. func (ix *Indexed) Len() int { if ix.Indexes == nil { @@ -307,18 +328,26 @@ func (ix *Indexed) NewTensor() Tensor { return nt } -// Clone returns a copy of the current Indexed view into the same -// underlying Tensor as the source, with its own copy of Indexes -// if they were present in source. +// Clone returns a copy of the current Indexed view with a cloned copy of +// the underlying Tensor and copy of the indexes. func (ix *Indexed) Clone() *Indexed { nix := &Indexed{} - nix.CopyFrom(ix) + nix.Tensor = ix.Tensor.Clone() + nix.CopyIndexes(ix) return nix } -// CopyFrom copies from given other Indexed (we have our own unique copy of indexes). -func (ix *Indexed) CopyFrom(oix *Indexed) { - ix.Tensor = oix.Tensor +// CloneIndexes returns a copy of the current Indexed view with new indexes, +// with a pointer to the same underlying Tensor as the source. +func (ix *Indexed) CloneIndexes() *Indexed { + nix := &Indexed{} + nix.Tensor = ix.Tensor + nix.CopyIndexes(ix) + return nix +} + +// CopyIndexes copies indexes from other Indexed view. +func (ix *Indexed) CopyIndexes(oix *Indexed) { if oix.Indexes == nil { ix.Indexes = nil } else { @@ -353,9 +382,7 @@ func (ix *Indexed) InsertRows(at, n int) { // DeleteRows deletes n rows of indexes starting at given index in the list of indexes func (ix *Indexed) DeleteRows(at, n int) { - if ix.Indexes == nil { - return - } + ix.IndexesNeeded() ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) } diff --git a/tensor/shape.go b/tensor/shape.go index e31f7fe553..097b3b6ca7 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -15,14 +15,13 @@ import ( // Per C / Go / Python conventions, indexes are Row-Major, ordered from // outer to inner left-to-right, so the inner-most is right-most. type Shape struct { - - // size per dimension + // size per dimension. Sizes []int - // offsets for each dimension + // offsets for each dimension. Strides []int `display:"-"` - // names of each dimension + // names of each dimension. Names []string `display:"-"` } diff --git a/tensor/stats/metric/vecfuncs.go b/tensor/stats/metric/vec.go similarity index 100% rename from tensor/stats/metric/vecfuncs.go rename to tensor/stats/metric/vec.go diff --git a/tensor/stats/norm/README.md b/tensor/stats/norm/README.md deleted file mode 100644 index ca41bdf01f..0000000000 --- a/tensor/stats/norm/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# norm - -`norm` provides normalization of vector and tensor values. The basic functions operate on either `[]float32` or `[]float64` data, with Tensor versions using those, only for Float32 and Float64 tensors. - -* DivNorm does divisive normalization of elements -* SubNorm does subtractive normalization of elements -* ZScore subtracts the mean and divides by the standard deviation -* Abs performs absolute-value on all elements (e.g., use prior to [stats](../stats) to produce Mean of Abs vals etc). - - diff --git a/tensor/stats/norm/abs.go b/tensor/stats/norm/abs.go deleted file mode 100644 index 699152382d..0000000000 --- a/tensor/stats/norm/abs.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package norm - -import ( - "fmt" - "log/slog" - "math" - - "cogentcore.org/core/math32" - "cogentcore.org/core/tensor" -) - -// Abs32 applies the Abs function to each element in given slice -func Abs32(a []float32) { - for i, av := range a { - if math32.IsNaN(av) { - continue - } - a[i] = math32.Abs(av) - } -} - -// Abs64 applies the Abs function to each element in given slice -func Abs64(a []float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - a[i] = math.Abs(av) - } -} - -func FloatOnlyError() error { - err := fmt.Errorf("Only float32 or float64 data types supported") - slog.Error(err.Error()) - return err -} - -// AbsTensor applies the Abs function to each element in given tensor, -// for float32 and float64 data types. -func AbsTensor(a tensor.Tensor) { - switch tsr := a.(type) { - case *tensor.Float32: - Abs32(tsr.Values) - case *tensor.Float64: - Abs64(tsr.Values) - default: - FloatOnlyError() - } -} diff --git a/tensor/stats/norm/doc.go b/tensor/stats/norm/doc.go deleted file mode 100644 index c3b769433e..0000000000 --- a/tensor/stats/norm/doc.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package norm provides normalization and norm metric computations -e.g., L2 = sqrt of sum of squares of a vector. - -DivNorm does divisive normalization of elements -SubNorm does subtractive normalization of elements -ZScore subtracts the mean and divides by the standard deviation -*/ -package norm diff --git a/tensor/stats/norm/norm.go b/tensor/stats/norm/norm.go deleted file mode 100644 index 1090e2fa53..0000000000 --- a/tensor/stats/norm/norm.go +++ /dev/null @@ -1,223 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package norm - -//go:generate core generate - -import ( - "math" - - "cogentcore.org/core/math32" - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/stats" -) - -// FloatFunc applies given functions to float tensor data, which is either Float32 or Float64 -func FloatFunc(tsr tensor.Tensor, nfunc32 Func32, nfunc64 Func64, stIdx, nIdx int, ffunc32 func(a []float32, fun Func32), ffunc64 func(a []float64, fun Func64)) { - switch tt := tsr.(type) { - case *tensor.Float32: - vals := tt.Values - if nIdx > 0 { - vals = vals[stIdx : stIdx+nIdx] - } - ffunc32(vals, nfunc32) - case *tensor.Float64: - vals := tt.Values - if nIdx > 0 { - vals = vals[stIdx : stIdx+nIdx] - } - ffunc64(vals, nfunc64) - default: - FloatOnlyError() - } -} - -/////////////////////////////////////////// -// DivNorm - -// DivNorm32 does divisive normalization by given norm function -// i.e., it divides each element by the norm value computed from nfunc. -func DivNorm32(a []float32, nfunc Func32) { - nv := nfunc(a) - if nv != 0 { - MultVector32(a, 1/nv) - } -} - -// DivNorm64 does divisive normalization by given norm function -// i.e., it divides each element by the norm value computed from nfunc. -func DivNorm64(a []float64, nfunc Func64) { - nv := nfunc(a) - if nv != 0 { - MultVec64(a, 1/nv) - } -} - -/////////////////////////////////////////// -// SubNorm - -// SubNorm32 does subtractive normalization by given norm function -// i.e., it subtracts norm computed by given function from each element. -func SubNorm32(a []float32, nfunc Func32) { - nv := nfunc(a) - AddVector32(a, -nv) -} - -// SubNorm64 does subtractive normalization by given norm function -// i.e., it subtracts norm computed by given function from each element. -func SubNorm64(a []float64, nfunc Func64) { - nv := nfunc(a) - AddVec64(a, -nv) -} - -/////////////////////////////////////////// -// ZScore - -// ZScore32 subtracts the mean and divides by the standard deviation -func ZScore32(a []float32) { - SubNorm32(a, stats.Mean32) - DivNorm32(a, stats.Std32) -} - -// ZScore64 subtracts the mean and divides by the standard deviation -func ZScore64(a []float64) { - SubNorm64(a, stats.Mean64) - DivNorm64(a, stats.Std64) -} - -/////////////////////////////////////////// -// Unit - -// Unit32 subtracts the min and divides by the max, so that values are in 0-1 unit range -func Unit32(a []float32) { - SubNorm32(a, stats.Min32) - DivNorm32(a, stats.Max32) -} - -// Unit64 subtracts the min and divides by the max, so that values are in 0-1 unit range -func Unit64(a []float64) { - SubNorm64(a, stats.Min64) - DivNorm64(a, stats.Max64) -} - -/////////////////////////////////////////// -// MultVec - -// MultVector32 multiplies vector elements by scalar -func MultVector32(a []float32, val float32) { - for i, av := range a { - if math32.IsNaN(av) { - continue - } - a[i] *= val - } -} - -// MultVec64 multiplies vector elements by scalar -func MultVec64(a []float64, val float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - a[i] *= val - } -} - -/////////////////////////////////////////// -// AddVec - -// AddVector32 adds scalar to vector -func AddVector32(a []float32, val float32) { - for i, av := range a { - if math32.IsNaN(av) { - continue - } - a[i] += val - } -} - -// AddVec64 adds scalar to vector -func AddVec64(a []float64, val float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - a[i] += val - } -} - -/////////////////////////////////////////// -// Thresh - -// Thresh32 thresholds the values of the vector -- anything above the high threshold is set -// to the high value, and everything below the low threshold is set to the low value. -func Thresh32(a []float32, hi bool, hiThr float32, lo bool, loThr float32) { - for i, av := range a { - if math32.IsNaN(av) { - continue - } - if hi && av > hiThr { - a[i] = hiThr - } - if lo && av < loThr { - a[i] = loThr - } - } -} - -// Thresh64 thresholds the values of the vector -- anything above the high threshold is set -// to the high value, and everything below the low threshold is set to the low value. -func Thresh64(a []float64, hi bool, hiThr float64, lo bool, loThr float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - if hi && av > hiThr { - a[i] = hiThr - } - if lo && av < loThr { - a[i] = loThr - } - } -} - -/////////////////////////////////////////// -// Binarize - -// Binarize32 turns vector into binary-valued, by setting anything >= the threshold -// to the high value, and everything below to the low value. -func Binarize32(a []float32, thr, hiVal, loVal float32) { - for i, av := range a { - if math32.IsNaN(av) { - continue - } - if av >= thr { - a[i] = hiVal - } else { - a[i] = loVal - } - } -} - -// Binarize64 turns vector into binary-valued, by setting anything >= the threshold -// to the high value, and everything below to the low value. -func Binarize64(a []float64, thr, hiVal, loVal float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - if av >= thr { - a[i] = hiVal - } else { - a[i] = loVal - } - } -} - -// Func32 is a norm function operating on slice of float32 numbers -type Func32 func(a []float32) float32 - -// Func64 is a norm function operating on slices of float64 numbers -type Func64 func(a []float64) float64 diff --git a/tensor/stats/norm/norm_test.go b/tensor/stats/norm/norm_test.go deleted file mode 100644 index b9b420e8de..0000000000 --- a/tensor/stats/norm/norm_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package norm - -import ( - "testing" - - "cogentcore.org/core/base/tolassert" - "cogentcore.org/core/tensor" - "github.com/stretchr/testify/assert" -) - -func TestNorm32(t *testing.T) { - vals := []float32{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - - zn := []float32{-1.5075567, -1.2060454, -0.90453404, -0.60302263, -0.30151132, 0, 0.3015114, 0.60302263, 0.90453404, 1.2060453, 1.5075567} - nvals := make([]float32, len(vals)) - copy(nvals, vals) - ZScore32(nvals) - assert.Equal(t, zn, nvals) - - copy(nvals, vals) - Unit32(nvals) - assert.Equal(t, vals, nvals) - - tn := []float32{0.2, 0.2, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.8, 0.8} - copy(nvals, vals) - Thresh32(nvals, true, 0.8, true, 0.2) - assert.Equal(t, tn, nvals) - - bn := []float32{0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} - copy(nvals, vals) - Binarize32(nvals, 0.5, 1.0, 0.0) - assert.Equal(t, bn, nvals) - - tsr := tensor.New[float32]([]int{11}).(*tensor.Float32) - copy(tsr.Values, vals) - TensorZScore(tsr, 0) - tolassert.EqualTolSlice(t, zn, tsr.Values, 1.0e-6) - - copy(tsr.Values, vals) - TensorUnit(tsr, 0) - tolassert.EqualTolSlice(t, vals, tsr.Values, 1.0e-6) - -} - -func TestNorm64(t *testing.T) { - vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - - zn := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818} - nvals := make([]float64, len(vals)) - copy(nvals, vals) - ZScore64(nvals) - assert.Equal(t, zn, nvals) - - copy(nvals, vals) - Unit64(nvals) - assert.Equal(t, vals, nvals) - - tn := []float64{0.2, 0.2, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.8, 0.8} - copy(nvals, vals) - Thresh64(nvals, true, 0.8, true, 0.2) - assert.Equal(t, tn, nvals) - - bn := []float64{0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} - copy(nvals, vals) - Binarize64(nvals, 0.5, 1.0, 0.0) - assert.Equal(t, bn, nvals) - - tsr := tensor.New[float64]([]int{11}).(*tensor.Float64) - copy(tsr.Values, vals) - TensorZScore(tsr, 0) - tolassert.EqualTolSlice(t, zn, tsr.Values, 1.0e-6) - - copy(tsr.Values, vals) - TensorUnit(tsr, 0) - tolassert.EqualTolSlice(t, vals, tsr.Values, 1.0e-6) - -} diff --git a/tensor/stats/norm/tensor.go b/tensor/stats/norm/tensor.go deleted file mode 100644 index ffc00db8f7..0000000000 --- a/tensor/stats/norm/tensor.go +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package norm - -import ( - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/stats" -) - -/////////////////////////////////////////// -// DivNorm - -// TensorDivNorm does divisive normalization by given norm function -// computed on the first ndim dims of the tensor, where 0 = all values, -// 1 = norm each of the sub-dimensions under the first outer-most dimension etc. -// ndim must be < NumDims() if not 0. -func TensorDivNorm(tsr tensor.Tensor, ndim int, nfunc32 Func32, nfunc64 Func64) { - if ndim == 0 { - FloatFunc(tsr, nfunc32, nfunc64, 0, 0, DivNorm32, DivNorm64) - } - if ndim >= tsr.NumDims() { - panic("norm.TensorSubNorm32: number of dims must be < NumDims()") - } - sln := 1 - ln := tsr.Len() - for i := 0; i < ndim; i++ { - sln *= tsr.Shape().DimSize(i) - } - dln := ln / sln - for sl := 0; sl < sln; sl++ { - st := sl * dln - FloatFunc(tsr, nfunc32, nfunc64, st, dln, DivNorm32, DivNorm64) - } -} - -/////////////////////////////////////////// -// SubNorm - -// TensorSubNorm does subtractive normalization by given norm function -// computed on the first ndim dims of the tensor, where 0 = all values, -// 1 = norm each of the sub-dimensions under the first outer-most dimension etc. -// ndim must be < NumDims() if not 0 (panics). -func TensorSubNorm(tsr tensor.Tensor, ndim int, nfunc32 Func32, nfunc64 Func64) { - if ndim == 0 { - FloatFunc(tsr, nfunc32, nfunc64, 0, 0, SubNorm32, SubNorm64) - } - if ndim >= tsr.NumDims() { - panic("norm.TensorSubNorm32: number of dims must be < NumDims()") - } - sln := 1 - ln := tsr.Len() - for i := 0; i < ndim; i++ { - sln *= tsr.Shape().DimSize(i) - } - dln := ln / sln - for sl := 0; sl < sln; sl++ { - st := sl * dln - FloatFunc(tsr, nfunc32, nfunc64, st, dln, SubNorm32, SubNorm64) - } -} - -// TensorZScore subtracts the mean and divides by the standard deviation -// computed on the first ndim dims of the tensor, where 0 = all values, -// 1 = norm each of the sub-dimensions under the first outer-most dimension etc. -// ndim must be < NumDims() if not 0 (panics). -// must be a float32 or float64 tensor -func TensorZScore(tsr tensor.Tensor, ndim int) { - TensorSubNorm(tsr, ndim, stats.Mean32, stats.Mean64) - TensorDivNorm(tsr, ndim, stats.Std32, stats.Std64) -} - -// TensorUnit subtracts the min and divides by the max, so that values are in 0-1 unit range -// computed on the first ndim dims of the tensor, where 0 = all values, -// 1 = norm each of the sub-dimensions under the first outer-most dimension etc. -// ndim must be < NumDims() if not 0 (panics). -// must be a float32 or float64 tensor -func TensorUnit(tsr tensor.Tensor, ndim int) { - TensorSubNorm(tsr, ndim, stats.Min32, stats.Min64) - TensorDivNorm(tsr, ndim, stats.Max32, stats.Max64) -} diff --git a/tensor/stats/stats/vecfuncs.go b/tensor/stats/stats/vec.go similarity index 100% rename from tensor/stats/stats/vecfuncs.go rename to tensor/stats/stats/vec.go diff --git a/tensor/stats/tmath/math.go b/tensor/stats/tmath/math.go new file mode 100644 index 0000000000..8b4844d51b --- /dev/null +++ b/tensor/stats/tmath/math.go @@ -0,0 +1,102 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "math" + + "cogentcore.org/core/tensor" +) + +func Abs(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := in.RowCellIndex(idx) + out.Tensor.SetFloat1D(i1d, math.Abs(in.Tensor.Float1D(i1d))) + }, in, out) +} + +func Acos(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := in.RowCellIndex(idx) + out.Tensor.SetFloat1D(i1d, math.Acos(in.Tensor.Float1D(i1d))) + }, in, out) +} + +func Acosh(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := in.RowCellIndex(idx) + out.Tensor.SetFloat1D(i1d, math.Acosh(in.Tensor.Float1D(i1d))) + }, in, out) +} + +/* +func Asin(in, out *tensor.Indexed) +func Asinh(in, out *tensor.Indexed) +func Atan(in, out *tensor.Indexed) +func Atan2(y, in, out *tensor.Indexed) +func Atanh(in, out *tensor.Indexed) +func Cbrt(in, out *tensor.Indexed) +func Ceil(in, out *tensor.Indexed) +func Copysign(f, sign float64) float64 +func Cos(in, out *tensor.Indexed) +func Cosh(in, out *tensor.Indexed) +func Dim(x, y float64) float64 +func Erf(in, out *tensor.Indexed) +func Erfc(in, out *tensor.Indexed) +func Erfcinv(in, out *tensor.Indexed) +func Erfinv(in, out *tensor.Indexed) +func Exp(in, out *tensor.Indexed) +func Exp2(in, out *tensor.Indexed) +func Expm1(in, out *tensor.Indexed) +func FMA(x, y, z float64) float64 +func Float32bits(f float32) uint32 +func Float32frombits(b uint32) float32 +func Float64bits(f float64) uint64 +func Float64frombits(b uint64) float64 +func Floor(in, out *tensor.Indexed) +func Frexp(f float64) (frac float64, exp int) +func Gamma(in, out *tensor.Indexed) +func Hypot(p, q float64) float64 +func Ilogb(x float64) int +func Inf(sign int) float64 +func IsInf(f float64, sign int) bool +func IsNaN(f float64) (is bool) +func J0(in, out *tensor.Indexed) +func J1(in, out *tensor.Indexed) +func Jn(n int, in, out *tensor.Indexed) +func Ldexp(frac float64, exp int) float64 +func Lgamma(x float64) (lgamma float64, sign int) +func Log(in, out *tensor.Indexed) +func Log10(in, out *tensor.Indexed) +func Log1p(in, out *tensor.Indexed) +func Log2(in, out *tensor.Indexed) +func Logb(in, out *tensor.Indexed) +func Max(x, y float64) float64 +func Min(x, y float64) float64 +func Mod(x, y float64) float64 +func Modf(f float64) (int float64, frac float64) +func NaN() float64 +func Nextafter(x, y float64) (r float64) +func Nextafter32(x, y float32) (r float32) +func Pow(x, y float64) float64 +func Pow10(n int) float64 +func Remainder(x, y float64) float64 +func Round(in, out *tensor.Indexed) +func RoundToEven(in, out *tensor.Indexed) +func Signbit(x float64) bool +func Sin(in, out *tensor.Indexed) +func Sincos(x float64) (sin, cos float64) +func Sinh(in, out *tensor.Indexed) +func Sqrt(in, out *tensor.Indexed) +func Tan(in, out *tensor.Indexed) +func Tanh(in, out *tensor.Indexed) +func Trunc(in, out *tensor.Indexed) +func Y0(in, out *tensor.Indexed) +func Y1(in, out *tensor.Indexed) +func Yn(n int, in, out *tensor.Indexed) +*/ diff --git a/tensor/stats/tmath/tmath.go b/tensor/stats/tmath/tmath.go new file mode 100644 index 0000000000..91ff236faa --- /dev/null +++ b/tensor/stats/tmath/tmath.go @@ -0,0 +1,23 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import "cogentcore.org/core/tensor" + +//go:generate core generate + +// Func1in1out is a function that has 1 tensor input and 1 output. +type Func1in1out func(in, out *tensor.Indexed) + +// Func2in1out is a function that has 2 tensor input and 1 output. +type Func2in1out func(in, out *tensor.Indexed) + +// Funcs1in1out is a registry of named math functions that +// take one input and one output tensor. +var Funcs1in1out map[string]Func1in1out + +// Funcs2in1out is a registry of named math functions that +// take two inputs and one output tensor. +var Funcs2in1out map[string]Func2in1out diff --git a/tensor/stats/tmath/tmath_test.go b/tensor/stats/tmath/tmath_test.go new file mode 100644 index 0000000000..8da74b7417 --- /dev/null +++ b/tensor/stats/tmath/tmath_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "math" + "testing" + + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" +) + +type onef func(x float64) float64 + +// Equal does equal testing taking into account NaN +func Equal(t *testing.T, trg, val float64) { + if math.IsNaN(trg) { + if !math.IsNaN(val) { + t.Error("target is NaN but actual is not") + } + return + } + assert.InDelta(t, trg, val, 1.0e-6) +} + +func TestMath(t *testing.T) { + scalar := tensor.NewIndexed(tensor.NewFloat64([]int{1})) + scalar.Tensor.SetFloat1D(0, -5.5) + scout := scalar.Clone() + + vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} + + oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) + oneout := oned.Clone() + + cell2d := tensor.NewIndexed(tensor.NewFloat32([]int{5, 2, 6})) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, ci := cell2d.RowCellIndex(idx) + cell2d.Tensor.SetFloat1D(i1d, oned.Tensor.Float1D(ci)) + }, cell2d) + // cell2d.DeleteRows(3, 1) + cellout := cell2d.Clone() + + mfuncs := []onef{math.Abs, math.Acos} + tfuncs := []Func1in1out{Abs, Acos} + + for i, fun := range mfuncs { + tf := tfuncs[i] + tf(scalar, scout) + tf(oned, oneout) + tf(cell2d, cellout) + + Equal(t, fun(scalar.Tensor.Float1D(0)), scout.Tensor.Float1D(0)) + for i, v := range vals { + Equal(t, fun(v), oneout.Tensor.Float1D(i)) + } + lv := len(vals) + for r := range 5 { + // fmt.Println(r) + si := lv * r + for c, v := range vals { + ov := cellout.Tensor.(*tensor.Float32).Values[si+c] + Equal(t, fun(v), float64(ov)) + } + } + } +} + +/* +func TestNorm64(t *testing.T) { + vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} + + zn := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818} + nvals := make([]float64, len(vals)) + copy(nvals, vals) + ZScore64(nvals) + assert.Equal(t, zn, nvals) + + copy(nvals, vals) + Unit64(nvals) + assert.Equal(t, vals, nvals) + + tn := []float64{0.2, 0.2, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.8, 0.8} + copy(nvals, vals) + Thresh64(nvals, true, 0.8, true, 0.2) + assert.Equal(t, tn, nvals) + + bn := []float64{0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} + copy(nvals, vals) + Binarize64(nvals, 0.5, 1.0, 0.0) + assert.Equal(t, bn, nvals) + + tsr := tensor.New[float64]([]int{11}).(*tensor.Float64) + copy(tsr.Values, vals) + TensorZScore(tsr, 0) + tolassert.EqualTolSlice(t, zn, tsr.Values, 1.0e-6) + + copy(tsr.Values, vals) + TensorUnit(tsr, 0) + tolassert.EqualTolSlice(t, vals, tsr.Values, 1.0e-6) + +} +*/ diff --git a/tensor/tensor.go b/tensor/tensor.go index 2d0205343a..cbb2032f45 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -152,8 +152,8 @@ type Tensor interface { // otherwise it goes through appropriate standard type. CopyFrom(from Tensor) - // SetShapeFrom copies just the shape from given source tensor - // calling SetShape with the shape params from source (see for more docs). + // SetShapeFrom sets our shape from given source tensor, calling + // [Tensor.SetShape] with the shape params from source. SetShapeFrom(from Tensor) // CopyCellsFrom copies given range of values from other tensor into this tensor, diff --git a/tensor/vectorize.go b/tensor/vectorize.go index f4dbabb7f9..3845a573ad 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -6,9 +6,22 @@ package tensor import ( "math" + "runtime" "sync" ) +var ( + // ThreadingThreshod is the threshold in number of tensor elements + // to engage actual parallel processing. + // Heuristically, numbers below this threshold do not result in + // an overall speedup, due to overhead costs. + ThreadingThreshold = 100_000 + + // NumThreads is the number of threads to use for parallel threading. + // The default of 0 causes the [runtime.GOMAXPROCS] to be used. + NumThreads = 0 +) + // Vectorize applies given function 'fun' to tensor elements indexed // by given index, with the 'nfun' providing the number of indexes // to vectorize over, and initializing any output vectors. @@ -32,7 +45,45 @@ func Vectorize(nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed } } -func VectorizeThreaded(threads int, nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { +// VectorizeThreaded is a version of [Vectorize] that will automatically +// distribute the computation in parallel across multiple "threads" (goroutines) +// if the number of elements to be computed times the given flops +// (floating point operations) for the function exceeds the [ThreadingThreshold]. +// Heuristically, numbers below this threshold do not result +// in an overall speedup, due to overhead costs. +func VectorizeThreaded(flops int, nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { + n := nfun(tsr...) + if n <= 0 { + return + } + if flops < 0 { + flops = 1 + } + if n*flops < ThreadingThreshold { + Vectorize(nfun, fun, tsr...) + return + } + VectorizeOnThreads(0, nfun, fun, tsr...) +} + +// DefaultNumThreads returns the default number of threads to use: +// NumThreads if non-zero, otherwise [runtime.GOMAXPROCS]. +func DefaultNumThreads() int { + if NumThreads > 0 { + return NumThreads + } + return runtime.GOMAXPROCS(0) +} + +// VectorizeOnThreads runs given [Vectorize] function on given number +// of threads. Use [VectorizeThreaded] to only use parallel threads when +// it is likely to be beneficial, in terms of the ThreadingThreshold. +// If threads is 0, then the [DefaultNumThreads] will be used: +// GOMAXPROCS subject to NumThreads constraint if non-zero. +func VectorizeOnThreads(threads int, nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { + if threads == 0 { + threads = DefaultNumThreads() + } n := nfun(tsr...) if n <= 0 { return @@ -55,15 +106,26 @@ func VectorizeThreaded(threads int, nfun func(tsr ...*Indexed) int, fun func(idx wait.Wait() } -// NFirst is an N function for Vectorize that returns the number of -// indexes of the first tensor. -func NFirst(tsr ...*Indexed) int { +// NFirstRows is an N function for Vectorize that returns the number of +// outer-dimension rows (or Indexes) of the first tensor. +func NFirstRows(tsr ...*Indexed) int { if len(tsr) == 0 { return 0 } return tsr[0].Len() } +// NFirstLen is an N function for Vectorize that returns the number of +// elements in the tensor, including the Indexes view. +func NFirstLen(tsr ...*Indexed) int { + if len(tsr) == 0 { + return 0 + } + ft := tsr[0] + _, cells := ft.Tensor.RowCellSize() + return cells * ft.Len() +} + // NMinNotLast is an N function for Vectorize that returns the min number of // indexes of all but the last tensor. This is used when the last tensor is // the output of the function, operating on the prior vector(s). From b3e9a2603723873ded05b87e5db68112ec0a4b4a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 11 Sep 2024 01:22:07 -0700 Subject: [PATCH 015/311] addition operator working --- tensor/indexed.go | 19 +- tensor/stats/metric/vec.go | 4 +- tensor/stats/stats/vec.go | 4 +- tensor/stats/tmath/README.md | 16 ++ tensor/stats/tmath/doc.go | 8 + tensor/stats/tmath/math.go | 359 ++++++++++++++++++++++++++----- tensor/stats/tmath/ops.go | 66 ++++++ tensor/stats/tmath/tmath_test.go | 6 +- tensor/vectorize.go | 22 +- 9 files changed, 424 insertions(+), 80 deletions(-) create mode 100644 tensor/stats/tmath/README.md create mode 100644 tensor/stats/tmath/doc.go create mode 100644 tensor/stats/tmath/ops.go diff --git a/tensor/indexed.go b/tensor/indexed.go index 1e4f33ec0e..9c5a80037f 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -91,14 +91,25 @@ func (ix *Indexed) RowCellIndex(idx int) (i1d, ri, ci int) { return } -// Len returns the length of the index list or number of outer rows dimension. -func (ix *Indexed) Len() int { +// Rows returns the effective number of rows in this Indexed view, +// which is the length of the index list or number of outer +// rows dimension of tensor if no indexes. +func (ix *Indexed) Rows() int { if ix.Indexes == nil { return ix.Tensor.DimSize(0) } return len(ix.Indexes) } +// Len returns the total number of elements in the tensor, +// taking into account the Indexes via [Rows], +// as Rows() * cell size. +func (ix *Indexed) Len() int { + rows := ix.Rows() + _, cells := ix.Tensor.RowCellSize() + return cells * rows +} + // DeleteInvalid deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. func (ix *Indexed) DeleteInvalid() { @@ -106,7 +117,7 @@ func (ix *Indexed) DeleteInvalid() { ix.Indexes = nil return } - ni := ix.Len() + ni := ix.Rows() for i := ni - 1; i >= 0; i-- { if ix.Indexes[i] >= ix.Tensor.DimSize(0) { ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) @@ -155,7 +166,7 @@ func (ix *Indexed) ExcludeMissing1D() { //types:add return } ix.IndexesNeeded() - ni := ix.Len() + ni := ix.Rows() for i := ni - 1; i >= 0; i-- { if math.IsNaN(ix.Tensor.Float1D(ix.Indexes[i])) { ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 6046dd54a6..287d495285 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -41,7 +41,7 @@ func OutShape(ish *tensor.Shape) *tensor.Shape { return osh } -// NFunc is the nfun for metrics functions, returning the min length across the +// NFunc is the nfun for metrics functions, returning the min number of rows across the // two input tensors, and initializing the _last_ one to hold the output // with the first, row dimension set to 1. func NFunc(tsr ...*tensor.Indexed) int { @@ -50,7 +50,7 @@ func NFunc(tsr ...*tensor.Indexed) int { return 0 } a, b, out := tsr[0], tsr[1], tsr[nt-1] - na, nb := a.Len(), b.Len() + na, nb := a.Rows(), b.Rows() osh := OutShape(a.Tensor.Shape()) out.Tensor.SetShape(osh.Sizes, osh.Names...) out.Indexes = []int{0} diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 6743258c51..db54e80ebe 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -68,7 +68,7 @@ func OutShape(ish *tensor.Shape) *tensor.Shape { return osh } -// NFunc is the nfun for stats functions, returning the length of the +// NFunc is the nfun for stats functions, returning number of rows of the // first tensor, and initializing the _last_ one to hold the output // with the first, row dimension set to 1. func NFunc(tsr ...*tensor.Indexed) int { @@ -77,7 +77,7 @@ func NFunc(tsr ...*tensor.Indexed) int { return 0 } in, out := tsr[0], tsr[nt-1] - n := in.Len() + n := in.Rows() osh := OutShape(in.Tensor.Shape()) out.Tensor.SetShape(osh.Sizes, osh.Names...) out.Indexes = []int{0} diff --git a/tensor/stats/tmath/README.md b/tensor/stats/tmath/README.md new file mode 100644 index 0000000000..fe9d65154b --- /dev/null +++ b/tensor/stats/tmath/README.md @@ -0,0 +1,16 @@ +# tmath is the Tensor math library + +# math functions + +All the standard library [math](https://pkg.go.dev/math) functions are implemented on `*tensor.Indexed`. + +To properly handle the row-wise indexes, all processing is done using row, cell indexes, with the row indirected through the indexes. + +# norm functions + +* DivNorm does divisive normalization of elements +* SubNorm does subtractive normalization of elements +* ZScore subtracts the mean and divides by the standard deviation +* Abs performs absolute-value on all elements (e.g., use prior to [stats](../stats) to produce Mean of Abs vals etc). + + diff --git a/tensor/stats/tmath/doc.go b/tensor/stats/tmath/doc.go new file mode 100644 index 0000000000..b5c427f75f --- /dev/null +++ b/tensor/stats/tmath/doc.go @@ -0,0 +1,8 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package tmath provides basic math operations and functions that operate on tensor.Indexed. +*/ +package tmath diff --git a/tensor/stats/tmath/math.go b/tensor/stats/tmath/math.go index 8b4844d51b..1fba5c3d4d 100644 --- a/tensor/stats/tmath/math.go +++ b/tensor/stats/tmath/math.go @@ -13,90 +13,335 @@ import ( func Abs(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := in.RowCellIndex(idx) - out.Tensor.SetFloat1D(i1d, math.Abs(in.Tensor.Float1D(i1d))) + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Abs(tsr[0].Tensor.Float1D(i1d))) }, in, out) } func Acos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := in.RowCellIndex(idx) - out.Tensor.SetFloat1D(i1d, math.Acos(in.Tensor.Float1D(i1d))) + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Acos(tsr[0].Tensor.Float1D(i1d))) }, in, out) } func Acosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := in.RowCellIndex(idx) - out.Tensor.SetFloat1D(i1d, math.Acosh(in.Tensor.Float1D(i1d))) + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Acosh(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Asin(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Asin(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Asinh(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Asinh(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Atan(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Atan(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Atanh(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Atanh(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Cbrt(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Cbrt(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Ceil(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Ceil(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Cos(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Cos(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Cosh(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Cosh(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Erf(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Erf(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Erfc(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Erfc(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Erfcinv(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Erfcinv(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Erfinv(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Erfinv(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Exp(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Exp(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Exp2(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Exp2(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Expm1(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Expm1(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Floor(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Floor(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Gamma(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Gamma(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func J0(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.J0(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func J1(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.J1(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Log(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Log(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Log10(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Log10(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Log1p(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Log1p(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Log2(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Log2(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Logb(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Logb(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Round(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Round(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func RoundToEven(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.RoundToEven(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Sin(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Sin(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Sinh(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Sinh(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Sqrt(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Sqrt(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Tan(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Tan(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Tanh(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Tanh(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Trunc(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Trunc(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Y0(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Y0(tsr[0].Tensor.Float1D(i1d))) + }, in, out) +} + +func Y1(in, out *tensor.Indexed) { + out.SetShapeFrom(in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i1d, math.Y1(tsr[0].Tensor.Float1D(i1d))) }, in, out) } /* -func Asin(in, out *tensor.Indexed) -func Asinh(in, out *tensor.Indexed) -func Atan(in, out *tensor.Indexed) func Atan2(y, in, out *tensor.Indexed) -func Atanh(in, out *tensor.Indexed) -func Cbrt(in, out *tensor.Indexed) -func Ceil(in, out *tensor.Indexed) func Copysign(f, sign float64) float64 -func Cos(in, out *tensor.Indexed) -func Cosh(in, out *tensor.Indexed) func Dim(x, y float64) float64 -func Erf(in, out *tensor.Indexed) -func Erfc(in, out *tensor.Indexed) -func Erfcinv(in, out *tensor.Indexed) -func Erfinv(in, out *tensor.Indexed) -func Exp(in, out *tensor.Indexed) -func Exp2(in, out *tensor.Indexed) -func Expm1(in, out *tensor.Indexed) -func FMA(x, y, z float64) float64 -func Float32bits(f float32) uint32 -func Float32frombits(b uint32) float32 -func Float64bits(f float64) uint64 -func Float64frombits(b uint64) float64 -func Floor(in, out *tensor.Indexed) -func Frexp(f float64) (frac float64, exp int) -func Gamma(in, out *tensor.Indexed) func Hypot(p, q float64) float64 -func Ilogb(x float64) int -func Inf(sign int) float64 -func IsInf(f float64, sign int) bool -func IsNaN(f float64) (is bool) -func J0(in, out *tensor.Indexed) -func J1(in, out *tensor.Indexed) -func Jn(n int, in, out *tensor.Indexed) -func Ldexp(frac float64, exp int) float64 -func Lgamma(x float64) (lgamma float64, sign int) -func Log(in, out *tensor.Indexed) -func Log10(in, out *tensor.Indexed) -func Log1p(in, out *tensor.Indexed) -func Log2(in, out *tensor.Indexed) -func Logb(in, out *tensor.Indexed) func Max(x, y float64) float64 func Min(x, y float64) float64 func Mod(x, y float64) float64 -func Modf(f float64) (int float64, frac float64) -func NaN() float64 func Nextafter(x, y float64) (r float64) func Nextafter32(x, y float32) (r float32) func Pow(x, y float64) float64 -func Pow10(n int) float64 func Remainder(x, y float64) float64 -func Round(in, out *tensor.Indexed) -func RoundToEven(in, out *tensor.Indexed) + +func Inf(sign int) float64 +func IsInf(f float64, sign int) bool +func IsNaN(f float64) (is bool) +func NaN() float64 func Signbit(x float64) bool -func Sin(in, out *tensor.Indexed) -func Sincos(x float64) (sin, cos float64) -func Sinh(in, out *tensor.Indexed) -func Sqrt(in, out *tensor.Indexed) -func Tan(in, out *tensor.Indexed) -func Tanh(in, out *tensor.Indexed) -func Trunc(in, out *tensor.Indexed) -func Y0(in, out *tensor.Indexed) -func Y1(in, out *tensor.Indexed) + +func Float32bits(f float32) uint32 +func Float32frombits(b uint32) float32 +func Float64bits(f float64) uint64 +func Float64frombits(b uint64) float64 + +func FMA(x, y, z float64) float64 + +func Jn(n int, in, out *tensor.Indexed) func Yn(n int, in, out *tensor.Indexed) + +func Ldexp(frac float64, exp int) float64 + +func Ilogb(x float64) int +func Pow10(n int) float64 + +func Frexp(f float64) (frac float64, exp int) +func Modf(f float64) (int float64, frac float64) +func Lgamma(x float64) (lgamma float64, sign int) +func Sincos(x float64) (sin, cos float64) */ diff --git a/tensor/stats/tmath/ops.go b/tensor/stats/tmath/ops.go new file mode 100644 index 0000000000..d7cb668625 --- /dev/null +++ b/tensor/stats/tmath/ops.go @@ -0,0 +1,66 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "cogentcore.org/core/tensor" +) + +// Add adds two tensors into output. +// If one is a scalar, it is added to all elements. +// If one is the same size as the Cell SubSpace of the other +// then the SubSpace is added to each row of the other. +// Otherwise an element-wise addition is performed +// for overlapping cells. +func Add(a, b, out *tensor.Indexed) { + if b.Len() == 1 { + AddScalar(b.FloatRowCell(0, 0), a, out) + return + } + if a.Len() == 1 { + AddScalar(a.FloatRowCell(0, 0), b, out) + return + } + arows, acells := a.Tensor.RowCellSize() + brows, bcells := b.Tensor.RowCellSize() + if brows*bcells == acells { + AddSubSpace(a, b, out) + return + } + if arows*acells == bcells { + AddSubSpace(b, a, out) + return + } + // just do element-wise + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + return tensor.NMinLen(2, tsr...) + }, + func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := tsr[0].RowCellIndex(idx) + ib, _, _ := tsr[1].RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)+tsr[1].Tensor.Float1D(ib)) + }, a, b, out) +} + +// AddScalar adds a scalar to given tensor into output. +func AddScalar(scalar float64, a, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := a.RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, a.Tensor.Float1D(ia)+scalar) + }, a, out) +} + +// AddSubSpace adds the subspace tensor to each row in the given tensor, +// into the output tensor. +func AddSubSpace(a, sub, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ai, _, ci := a.RowCellIndex(idx) + si, _, _ := sub.RowCellIndex(ci) + out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)+sub.Tensor.Float1D(si)) + }, a, sub, out) +} diff --git a/tensor/stats/tmath/tmath_test.go b/tensor/stats/tmath/tmath_test.go index 8da74b7417..5500045140 100644 --- a/tensor/stats/tmath/tmath_test.go +++ b/tensor/stats/tmath/tmath_test.go @@ -22,7 +22,7 @@ func Equal(t *testing.T, trg, val float64) { } return } - assert.InDelta(t, trg, val, 1.0e-6) + assert.InDelta(t, trg, val, 1.0e-4) } func TestMath(t *testing.T) { @@ -43,8 +43,8 @@ func TestMath(t *testing.T) { // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() - mfuncs := []onef{math.Abs, math.Acos} - tfuncs := []Func1in1out{Abs, Acos} + mfuncs := []onef{math.Abs, math.Acos, math.Acosh, math.Asin, math.Asinh, math.Atan, math.Atanh, math.Cbrt, math.Ceil, math.Cos, math.Cosh, math.Erf, math.Erfc, math.Erfcinv, math.Erfcinv, math.Erfinv, math.Exp, math.Exp2, math.Expm1, math.Floor, math.Gamma, math.J0, math.J1, math.Log, math.Log10, math.Log1p, math.Log2, math.Logb, math.Round, math.RoundToEven, math.Sin, math.Sinh, math.Sqrt, math.Tan, math.Tanh, math.Trunc, math.Y0, math.Y1} + tfuncs := []Func1in1out{Abs, Acos, Acosh, Asin, Asinh, Atan, Atanh, Cbrt, Ceil, Cos, Cosh, Erf, Erfc, Erfcinv, Erfcinv, Erfinv, Exp, Exp2, Expm1, Floor, Gamma, J0, J1, Log, Log10, Log1p, Log2, Logb, Round, RoundToEven, Sin, Sinh, Sqrt, Tan, Tanh, Trunc, Y0, Y1} for i, fun := range mfuncs { tf := tfuncs[i] diff --git a/tensor/vectorize.go b/tensor/vectorize.go index 3845a573ad..cb036cb2af 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -112,30 +112,28 @@ func NFirstRows(tsr ...*Indexed) int { if len(tsr) == 0 { return 0 } - return tsr[0].Len() + return tsr[0].Rows() } // NFirstLen is an N function for Vectorize that returns the number of -// elements in the tensor, including the Indexes view. +// elements in the tensor, taking into account the Indexes view. func NFirstLen(tsr ...*Indexed) int { if len(tsr) == 0 { return 0 } - ft := tsr[0] - _, cells := ft.Tensor.RowCellSize() - return cells * ft.Len() + return tsr[0].Len() } -// NMinNotLast is an N function for Vectorize that returns the min number of -// indexes of all but the last tensor. This is used when the last tensor is -// the output of the function, operating on the prior vector(s). -func NMinNotLast(tsr ...*Indexed) int { - nt := len(tsr) - if nt < 2 { +// NMinLen is an N function for Vectorize that returns the min number of +// elements across given number of tensors in the list. Use a closure +// to call this with the nt. +func NMinLen(nt int, tsr ...*Indexed) int { + nt = min(len(tsr), nt) + if nt == 0 { return 0 } n := tsr[0].Len() - for i := 1; i < nt-1; i++ { + for i := 1; i < nt; i++ { n = min(n, tsr[0].Len()) } return n From 00eec4b4840c6a63b5f24d4c8eb14bda61fb6e86 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 11 Sep 2024 02:28:27 -0700 Subject: [PATCH 016/311] all element-wise ops in place --- tensor/stats/tmath/README.md | 2 + tensor/stats/tmath/ops.go | 201 +++++++++++++++++++++++++++++++++ tensor/stats/tmath/ops_test.go | 129 +++++++++++++++++++++ 3 files changed, 332 insertions(+) create mode 100644 tensor/stats/tmath/ops_test.go diff --git a/tensor/stats/tmath/README.md b/tensor/stats/tmath/README.md index fe9d65154b..303bcf8e31 100644 --- a/tensor/stats/tmath/README.md +++ b/tensor/stats/tmath/README.md @@ -6,6 +6,8 @@ All the standard library [math](https://pkg.go.dev/math) functions are implement To properly handle the row-wise indexes, all processing is done using row, cell indexes, with the row indirected through the indexes. +The standard Add, Sub, Mul, Div mathematical operators all operate element-wise, with a separate MatMul for matrix multiplication, which operates through gonum routines, for 2D Float64 tensor shapes with no indexes, so that the raw float64 values can be passed directly to gonum. + # norm functions * DivNorm does divisive normalization of elements diff --git a/tensor/stats/tmath/ops.go b/tensor/stats/tmath/ops.go index d7cb668625..1e095e88e7 100644 --- a/tensor/stats/tmath/ops.go +++ b/tensor/stats/tmath/ops.go @@ -64,3 +64,204 @@ func AddSubSpace(a, sub, out *tensor.Indexed) { out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)+sub.Tensor.Float1D(si)) }, a, sub, out) } + +//////////////////////////////////////////////////////////// +// Sub + +// Sub subtracts two tensors into output. +// If one is a scalar, it is subtracted from all elements. +// If one is the same size as the Cell SubSpace of the other +// then the SubSpace is subtracted from each row of the other. +// Otherwise an element-wise subtractition is performed +// for overlapping cells. +func Sub(a, b, out *tensor.Indexed) { + if b.Len() == 1 { + SubScalar(1, b.FloatRowCell(0, 0), a, out) + return + } + if a.Len() == 1 { + SubScalar(-1, a.FloatRowCell(0, 0), b, out) + return + } + arows, acells := a.Tensor.RowCellSize() + brows, bcells := b.Tensor.RowCellSize() + if brows*bcells == acells { + SubSubSpace(1, a, b, out) + return + } + if arows*acells == bcells { + SubSubSpace(-1, b, a, out) + return + } + // just do element-wise + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + return tensor.NMinLen(2, tsr...) + }, + func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := tsr[0].RowCellIndex(idx) + ib, _, _ := tsr[1].RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)-tsr[1].Tensor.Float1D(ib)) + }, a, b, out) +} + +// SubScalar subtracts a scalar from given tensor into output. +// sign determines which way the subtraction goes: 1 = a-scalar, -1 = scalar-a +func SubScalar(sign float64, scalar float64, a, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := a.RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, sign*(a.Tensor.Float1D(ia)-scalar)) + }, a, out) +} + +// SubSubSpace subtracts the subspace tensor to each row in the given tensor, +// into the output tensor. +func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ai, _, ci := a.RowCellIndex(idx) + si, _, _ := sub.RowCellIndex(ci) + out.Tensor.SetFloat1D(ai, sign*(a.Tensor.Float1D(ai)-sub.Tensor.Float1D(si))) + }, a, sub, out) +} + +//////////////////////////////////////////////////////////// +// Mul + +// Mul does element-wise multiplication between two tensors into output. +// If one is a scalar, it multiplies all elements. +// If one is the same size as the Cell SubSpace of the other +// then the SubSpace multiplies each row of the other. +// Otherwise an element-wise multiplication is performed +// for overlapping cells. +func Mul(a, b, out *tensor.Indexed) { + if b.Len() == 1 { + MulScalar(b.FloatRowCell(0, 0), a, out) + return + } + if a.Len() == 1 { + MulScalar(a.FloatRowCell(0, 0), b, out) + return + } + arows, acells := a.Tensor.RowCellSize() + brows, bcells := b.Tensor.RowCellSize() + if brows*bcells == acells { + MulSubSpace(a, b, out) + return + } + if arows*acells == bcells { + MulSubSpace(b, a, out) + return + } + // just do element-wise + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + return tensor.NMinLen(2, tsr...) + }, + func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := tsr[0].RowCellIndex(idx) + ib, _, _ := tsr[1].RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)*tsr[1].Tensor.Float1D(ib)) + }, a, b, out) +} + +// MulScalar multiplies a scalar to given tensor into output. +func MulScalar(scalar float64, a, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := a.RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, a.Tensor.Float1D(ia)*scalar) + }, a, out) +} + +// MulSubSpace multiplies the subspace tensor to each row in the given tensor, +// into the output tensor. +func MulSubSpace(a, sub, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ai, _, ci := a.RowCellIndex(idx) + si, _, _ := sub.RowCellIndex(ci) + out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)*sub.Tensor.Float1D(si)) + }, a, sub, out) +} + +//////////////////////////////////////////////////////////// +// Div + +// Div does element-wise division between two tensors into output. +// If one is a scalar, it divides all elements. +// If one is the same size as the Cell SubSpace of the other +// then the SubSpace divides each row of the other. +// Otherwise an element-wise division is performed +// for overlapping cells. +func Div(a, b, out *tensor.Indexed) { + if b.Len() == 1 { + DivScalar(b.FloatRowCell(0, 0), a, out) + return + } + if a.Len() == 1 { + DivScalarInv(a.FloatRowCell(0, 0), b, out) + return + } + arows, acells := a.Tensor.RowCellSize() + brows, bcells := b.Tensor.RowCellSize() + if brows*bcells == acells { + DivSubSpace(a, b, out) + return + } + if arows*acells == bcells { + DivSubSpaceInv(b, a, out) + return + } + // just do element-wise + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + return tensor.NMinLen(2, tsr...) + }, + func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := tsr[0].RowCellIndex(idx) + ib, _, _ := tsr[1].RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)/tsr[1].Tensor.Float1D(ib)) + }, a, b, out) +} + +// DivScalar divides given tensor elements by scalar into output. +func DivScalar(scalar float64, a, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := a.RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, a.Tensor.Float1D(ia)/scalar) + }, a, out) +} + +// DivScalarInv divides scalar by given tensor elements into output. +func DivScalarInv(scalar float64, a, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ia, _, _ := a.RowCellIndex(idx) + out.Tensor.SetFloat1D(ia, scalar/a.Tensor.Float1D(ia)) + }, a, out) +} + +// DivSubSpace divides each row of the given tensor by the subspace tensor elements, +// into the output tensor. +func DivSubSpace(a, sub, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ai, _, ci := a.RowCellIndex(idx) + si, _, _ := sub.RowCellIndex(ci) + out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)/sub.Tensor.Float1D(si)) + }, a, sub, out) +} + +// DivSubSpaceInv divides the subspace tensor by each row of the given tensor, +// into the output tensor. +func DivSubSpaceInv(a, sub, out *tensor.Indexed) { + out.SetShapeFrom(a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + ai, _, ci := a.RowCellIndex(idx) + si, _, _ := sub.RowCellIndex(ci) + out.Tensor.SetFloat1D(ai, sub.Tensor.Float1D(si)/a.Tensor.Float1D(ai)) + }, a, sub, out) +} diff --git a/tensor/stats/tmath/ops_test.go b/tensor/stats/tmath/ops_test.go new file mode 100644 index 0000000000..c1f2608370 --- /dev/null +++ b/tensor/stats/tmath/ops_test.go @@ -0,0 +1,129 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "testing" + + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" +) + +func TestAdd(t *testing.T) { + scalar := tensor.NewIndexed(tensor.NewFloat64([]int{1})) + scalar.Tensor.SetFloat1D(0, -5.5) + scb := scalar.Clone() + scb.Tensor.SetFloat1D(0, -4.0) + scout := scalar.Clone() + + vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} + + oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) + oneout := oned.Clone() + + cell2d := tensor.NewIndexed(tensor.NewFloat32([]int{5, 2, 6})) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i1d, _, ci := cell2d.RowCellIndex(idx) + cell2d.Tensor.SetFloat1D(i1d, oned.Tensor.Float1D(ci)) + }, cell2d) + // cell2d.DeleteRows(3, 1) + cellout := cell2d.Clone() + + Add(scalar, scb, scout) + assert.Equal(t, -5.5+-4, scout.Tensor.Float1D(0)) + + Add(scalar, oned, oneout) + for i, v := range vals { + assert.Equal(t, v+-5.5, oneout.Tensor.Float1D(i)) + } + + Add(oned, oned, oneout) + for i, v := range vals { + assert.Equal(t, v+v, oneout.Tensor.Float1D(i)) + } + + Add(cell2d, oned, cellout) + for ri := range 5 { + for i, v := range vals { + assert.InDelta(t, v+v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + } + } + + Sub(scalar, scb, scout) + assert.Equal(t, -5.5 - -4, scout.Tensor.Float1D(0)) + + Sub(scb, scalar, scout) + assert.Equal(t, -4 - -5.5, scout.Tensor.Float1D(0)) + + Sub(scalar, oned, oneout) + for i, v := range vals { + assert.Equal(t, -5.5-v, oneout.Tensor.Float1D(i)) + } + + Sub(oned, scalar, oneout) + for i, v := range vals { + assert.Equal(t, v - -5.5, oneout.Tensor.Float1D(i)) + } + + Sub(oned, oned, oneout) + for i, v := range vals { + assert.Equal(t, v-v, oneout.Tensor.Float1D(i)) + } + + Sub(cell2d, oned, cellout) + for ri := range 5 { + for i, v := range vals { + assert.InDelta(t, v-v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + } + } + + Mul(scalar, scb, scout) + assert.Equal(t, -5.5*-4, scout.Tensor.Float1D(0)) + + Mul(scalar, oned, oneout) + for i, v := range vals { + assert.Equal(t, v*-5.5, oneout.Tensor.Float1D(i)) + } + + Mul(oned, oned, oneout) + for i, v := range vals { + assert.Equal(t, v*v, oneout.Tensor.Float1D(i)) + } + + Mul(cell2d, oned, cellout) + for ri := range 5 { + for i, v := range vals { + assert.InDelta(t, v*v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + } + } + + Div(scalar, scb, scout) + assert.Equal(t, -5.5/-4, scout.Tensor.Float1D(0)) + + Div(scb, scalar, scout) + assert.Equal(t, -4/-5.5, scout.Tensor.Float1D(0)) + + Div(scalar, oned, oneout) + for i, v := range vals { + assert.Equal(t, -5.5/v, oneout.Tensor.Float1D(i)) + } + + Div(oned, scalar, oneout) + for i, v := range vals { + assert.Equal(t, v/-5.5, oneout.Tensor.Float1D(i)) + } + + Div(oned, oned, oneout) + for i, v := range vals { + assert.Equal(t, v/v, oneout.Tensor.Float1D(i)) + } + + Div(cell2d, oned, cellout) + for ri := range 5 { + for i, v := range vals { + assert.InDelta(t, v/v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + } + } +} From 7e1926368423e5e9d0093ede252d61ae85370649 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 11 Sep 2024 03:11:52 -0700 Subject: [PATCH 017/311] Unit and Z normalization functions in place using new math ops + stats. --- tensor/stats/stats/funcs.go | 19 ++++++--- tensor/stats/tmath/norm.go | 75 ++++++++++++++++++++++++++++++++++ tensor/stats/tmath/ops_test.go | 8 ++++ 3 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 tensor/stats/tmath/norm.go diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 2e187333b7..1ee43e79b7 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -180,15 +180,24 @@ func VarFunc(in, out *tensor.Indexed) { VarFuncOut64(in, out) } +// StdFuncOut64 computes the sample standard deviation of tensor values. +// and returns the Float64 output values for subsequent use. +func StdFuncOut64(in, out *tensor.Indexed) (std64, mean64, count64 *tensor.Indexed) { + std64, mean64, count64 = VarFuncOut64(in, out) + nsub := out.Tensor.Len() + for i := range nsub { + std := math.Sqrt(std64.Tensor.Float1D(i)) + std64.Tensor.SetFloat1D(i, std) + out.Tensor.SetFloat1D(i, std) + } + return +} + // StdFunc computes the sample standard deviation of tensor values. // Sqrt of variance from [VarFunc]. See also [StdPopFunc]. // See [StatsFunc] for general information. func StdFunc(in, out *tensor.Indexed) { - var64, _, _ := VarFuncOut64(in, out) - nsub := out.Tensor.Len() - for i := range nsub { - out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) - } + StdFuncOut64(in, out) } // SemFunc computes the sample standard error of the mean of tensor values. diff --git a/tensor/stats/tmath/norm.go b/tensor/stats/tmath/norm.go new file mode 100644 index 0000000000..9fe9c2ef2a --- /dev/null +++ b/tensor/stats/tmath/norm.go @@ -0,0 +1,75 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" +) + +// ZScore computes Z-normalized values into given output tensor, +// subtracting the Mean and dividing by the standard deviation. +func ZScore(a, out *tensor.Indexed) { + osh := stats.OutShape(a.Tensor.Shape()) + mout := tensor.NewIndexed(tensor.NewFloat64(osh.Sizes)) + std, mean, _ := stats.StdFuncOut64(a, mout) + out.SetShapeFrom(a) + Sub(a, mean, out) + Div(out, std, out) // for use out as in too +} + +// UnitNorm computes unit normalized values into given output tensor, +// subtracting the Min value and dividing by the Max of the remaining numbers. +func UnitNorm(a, out *tensor.Indexed) { + osh := stats.OutShape(a.Tensor.Shape()) + mout := tensor.NewIndexed(tensor.NewFloat64(osh.Sizes)) + stats.MinFunc(a, mout) + out.SetShapeFrom(a) + Sub(a, mout, out) + stats.MaxFunc(out, mout) + Div(out, mout, out) // for use out as in too +} + +/* + +/////////////////////////////////////////// +// Thresh + + +// Thresh64 thresholds the values of the vector -- anything above the high threshold is set +// to the high value, and everything below the low threshold is set to the low value. +func Thresh64(a []float64, hi bool, hiThr float64, lo bool, loThr float64) { + for i, av := range a { + if math.IsNaN(av) { + continue + } + if hi && av > hiThr { + a[i] = hiThr + } + if lo && av < loThr { + a[i] = loThr + } + } +} + +/////////////////////////////////////////// +// Binarize + +// Binarize64 turns vector into binary-valued, by setting anything >= the threshold +// to the high value, and everything below to the low value. +func Binarize64(a []float64, thr, hiVal, loVal float64) { + for i, av := range a { + if math.IsNaN(av) { + continue + } + if av >= thr { + a[i] = hiVal + } else { + a[i] = loVal + } + } +} + +*/ diff --git a/tensor/stats/tmath/ops_test.go b/tensor/stats/tmath/ops_test.go index c1f2608370..7037d19ded 100644 --- a/tensor/stats/tmath/ops_test.go +++ b/tensor/stats/tmath/ops_test.go @@ -5,6 +5,7 @@ package tmath import ( + "fmt" "testing" "cogentcore.org/core/tensor" @@ -126,4 +127,11 @@ func TestAdd(t *testing.T) { assert.InDelta(t, v/v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) } } + + ZScore(oned, oneout) + fmt.Println(oneout.Tensor) + + UnitNorm(oned, oneout) + fmt.Println(oneout.Tensor) + } From 1d28f6245fd5d129672fcdabfb309b0a75a084ca Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 11 Sep 2024 14:27:42 -0700 Subject: [PATCH 018/311] tensor global Funcs registry with arg count info and CallOut func that makes output tensors for you -- handy. this is now fully "cosl ready" --- tensor/README.md | 16 ++- tensor/base.go | 3 + tensor/funcs.go | 188 +++++++++++++++++++++++++ tensor/funcs_test.go | 48 +++++++ tensor/indexed.go | 6 + tensor/stats/metric/enumgen.go | 10 +- tensor/stats/metric/metric_test.go | 2 +- tensor/stats/metric/metrics.go | 65 +++------ tensor/stats/metric/misc.go | 19 --- tensor/stats/stats/stats.go | 67 ++++----- tensor/stats/tmath/README.md | 18 --- tensor/stats/tmath/norm.go | 75 ---------- tensor/stats/tmath/tmath.go | 23 --- tensor/tmath/README.md | 18 +++ tensor/{stats => }/tmath/doc.go | 0 tensor/{stats => }/tmath/math.go | 188 +++++++++++++++---------- tensor/tmath/norm.go | 61 ++++++++ tensor/{stats => }/tmath/ops.go | 6 +- tensor/{stats => }/tmath/ops_test.go | 31 +++- tensor/tmath/tmath.go | 7 + tensor/{stats => }/tmath/tmath_test.go | 4 +- 21 files changed, 542 insertions(+), 313 deletions(-) create mode 100644 tensor/funcs.go create mode 100644 tensor/funcs_test.go delete mode 100644 tensor/stats/tmath/README.md delete mode 100644 tensor/stats/tmath/norm.go delete mode 100644 tensor/stats/tmath/tmath.go create mode 100644 tensor/tmath/README.md rename tensor/{stats => }/tmath/doc.go (100%) rename tensor/{stats => }/tmath/math.go (58%) create mode 100644 tensor/tmath/norm.go rename tensor/{stats => }/tmath/ops.go (98%) rename tensor/{stats => }/tmath/ops_test.go (76%) create mode 100644 tensor/tmath/tmath.go rename tensor/{stats => }/tmath/tmath_test.go (85%) diff --git a/tensor/README.md b/tensor/README.md index fea4086d38..f35e14b262 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -2,23 +2,27 @@ Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used `numpy` and `pandas` libraries in python, and the commercial MATLAB framework. -The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. +The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. The indexes provide a specific view onto the underlying [Tensor] data, applying to the outer-most _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. -The [cosl](../cosl) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. +The `float64` ("Float") and `string` types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. -The indexes provide a specific view onto the underlying [Tensor] data, applying to the outer-most _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. +The [cosl](../cosl) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. Use the `[Set]FloatRowCell` methods wherever possible, for the most efficient and natural indirection through the indexes. The 1D methods on underlying tensor data do not indirect through the indexes and must be called directly on the [Tensor]. -* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `Indexed` that provides an indexed view into the rows of the table for highly efficient filtering and sorting of data. +The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. It is completely up to the compute function how to interpret the index. There is a Threaded version of this for parallelizable functions, and a GPU version. - Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including the plotcore. +All tensor package functions are registered using a single name to function map (`Funcs`). + +* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `Indexed` that provides a shared index for the column tensors. + + Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including [plot/plotcore](../plot/plotcore). * [datafs](datafs) provides a virtual filesystem (FS) for organizing arbitrary collections of data, supporting interactive, ad-hoc (notebook style) as well as systematic data processing. Interactive [cosl](../cosl) shell commands (`cd`, `ls`, `mkdir` etc) can be used to navigate the data space, with numerical expressions immediately available to operate on the data and save results back to the filesystem. Furthermore, the data can be directly copied to / from the OS filesystem to persist it, and `cosl` can transparently access data on remote systems through ssh. Furthermore, the [databrowser](databrowser) provides a fully interactive GUI for inspecting and plotting data. * [tensorcore](tensorcore) provides core widgets for graphically displaying the `Tensor` and `Table` data, which are used in `datafs`. -* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data. +* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data, including the standard `+, -, *, /` operators. `cosl` then calls these functions. * [stats](stats) implements a number of different ways of analyzing tensor and table data diff --git a/tensor/base.go b/tensor/base.go index c5b84cb9c9..ae5142c4a9 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -77,6 +77,9 @@ func (tsr *Base[T]) view() *Base[T] { // SetShape sets the shape params, resizing backing storage appropriately func (tsr *Base[T]) SetShape(sizes []int, names ...string) { + if len(sizes) == 0 { + sizes = []int{0} + } tsr.shape.SetShape(sizes, names...) nln := tsr.Len() tsr.Values = slicesx.SetLength(tsr.Values, nln) diff --git a/tensor/funcs.go b/tensor/funcs.go new file mode 100644 index 0000000000..9e77aaa56b --- /dev/null +++ b/tensor/funcs.go @@ -0,0 +1,188 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "fmt" + "reflect" + "strings" + + "cogentcore.org/core/base/errors" +) + +// Func represents a registered tensor function, which has +// In number of input *tensor.Indexed arguments, and Out +// number of output arguments. This quantification of the +// argument count is important for allowing the cosl script +// language to properly parse expressions involving these functions. +type Func struct { + // Name is the original CamelCase Go name for function + Name string + + // Fun is the function, which must only take some number of *tensor.Indexed args + Fun any + + // In is number of input args + In int + + // Out is number of output args + Out int +} + +// NewFunc creates a new Func desciption of the given +// function, with specified number of output arguments. +// The remaining arguments in the function (automatically +// determined) are classified as input arguments. +func NewFunc(name string, fun any, out int) (*Func, error) { + fn := &Func{Name: name, Fun: fun} + nargs := fn.ArgCount() + if out > nargs { + return nil, fmt.Errorf("tensor.NewFunc: too many output args for function %q, which takes %d (-1 means function signature is not recognized)", name, nargs) + } + fn.Out = out + fn.In = 1 - out + return fn, nil +} + +// Funcs is the global tensor named function registry. +// All functions must be of the form: func(a, b *tensor.Indexed) +// i.e., taking some specific number of Indexed arguments, +// with the number of output vs. input arguments registered. +var Funcs map[string]*Func + +// AddFunc adds given named function to the global tensor named function +// registry, which is used in cosl to call functions by name, and +// in specific packages to call functions by enum String() names. +// Use the standard Go CamelCase name -- will be auto-lowercased. +// The number of output arguments must be provided here; +// the number of input arguments is automatically set from that. +func AddFunc(name string, fun any, out int) error { + if Funcs == nil { + Funcs = make(map[string]*Func) + } + nm := strings.ToLower(name) + _, ok := Funcs[nm] + if ok { + return errors.Log(fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name)) + } + fn, err := NewFunc(name, fun, out) + if errors.Log(err) != nil { + return err + } + Funcs[nm] = fn + // note: can record orig camel name if needed for docs etc later. + return nil +} + +// Call calls function of given name, with given set of arguments +// (input and output) appropriate for the given function. +// An error is returned if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match. +func Call(name string, tsr ...*Indexed) error { + nm := strings.ToLower(name) + fn, ok := Funcs[nm] + if !ok { + return errors.Log(fmt.Errorf("tensor.Call: function of name %q not registered", name)) + } + return fn.Call(tsr...) +} + +// CallOut calls function of given name, with given set of _input_ +// arguments appropriate for the given function, returning newly created +// output tensors. +// An error is returned if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match. +func CallOut(name string, tsr ...*Indexed) ([]*Indexed, error) { + nm := strings.ToLower(name) + fn, ok := Funcs[nm] + if !ok { + return nil, errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) + } + return fn.CallOut(tsr...) +} + +// ArgCount returns the number of arguments the function takes, +// using a type switch. +func (fn *Func) ArgCount() int { + nargs := -1 + switch fn.Fun.(type) { + case func(a *Indexed): + nargs = 1 + case func(a, b *Indexed): + nargs = 2 + case func(a, b, c *Indexed): + nargs = 3 + case func(a, b, c, d *Indexed): + nargs = 4 + case func(a, b, c, d, e *Indexed): + nargs = 5 + } + return nargs +} + +// ArgCheck returns an error if the number of args in list does not +// match the number required as specified. +func (fn *Func) ArgCheck(n int, tsr ...*Indexed) error { + if len(tsr) != n { + return fmt.Errorf("tensor.Call: args passed to %q: %d does not match required: %d", fn.Name, len(tsr), n) + } + return nil +} + +// Call calls function with given set of input & output arguments +// appropriate for the given function (error if not). +func (fn *Func) Call(tsr ...*Indexed) error { + switch f := fn.Fun.(type) { + case func(a *Indexed): + if err := fn.ArgCheck(1, tsr...); err != nil { + return err + } + f(tsr[0]) + case func(a, b *Indexed): + if err := fn.ArgCheck(2, tsr...); err != nil { + return err + } + f(tsr[0], tsr[1]) + case func(a, b, c *Indexed): + if err := fn.ArgCheck(3, tsr...); err != nil { + return err + } + f(tsr[0], tsr[1], tsr[2]) + case func(a, b, c, d *Indexed): + if err := fn.ArgCheck(4, tsr...); err != nil { + return err + } + f(tsr[0], tsr[1], tsr[2], tsr[3]) + case func(a, b, c, d, e *Indexed): + if err := fn.ArgCheck(5, tsr...); err != nil { + return err + } + f(tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) + } + return nil +} + +// CallOut calls function with given set of _input_ arguments +// appropriate for the given function (error if not). +// Newly-created output values are returned. +func (fn *Func) CallOut(tsr ...*Indexed) ([]*Indexed, error) { + if fn.Out == 0 { + err := fn.Call(tsr...) + return nil, err + } + typ := reflect.Float64 + if fn.In > 0 { + typ = tsr[0].Tensor.DataType() + } + outs := make([]*Indexed, fn.Out) + for i := range outs { + outs[i] = NewIndexed(NewOfType(typ, nil)) + } + tsr = append(tsr, outs...) + err := fn.Call(tsr...) + return outs, err +} diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go new file mode 100644 index 0000000000..6bd8681726 --- /dev/null +++ b/tensor/funcs_test.go @@ -0,0 +1,48 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "math" + "testing" + + "github.com/stretchr/testify/assert" +) + +func abs(in, out *Indexed) { + out.SetShapeFrom(in) + VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...*Indexed) { + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Abs(tsr[0].Tensor.Float1D(i))) + }, in, out) +} + +func TestFuncs(t *testing.T) { + err := AddFunc("Abs", abs, 1) + assert.NoError(t, err) + + err = AddFunc("Abs", abs, 1) + assert.Error(t, err) + + err = AddFunc("Abs3", abs, 3) + assert.Error(t, err) + + vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} + + oned := NewIndexed(NewNumberFromSlice(vals)) + oneout := oned.Clone() + + err = Call("Abs", oned, oneout) + assert.NoError(t, err) + + assert.Equal(t, 1.507556722888818, oneout.Tensor.Float1D(0)) + + err = Call("Abs", oned) + assert.Error(t, err) + + out, err := CallOut("Abs", oned) + assert.NoError(t, err) + assert.Equal(t, oneout.Tensor.(*Float64).Values, out[0].Tensor.(*Float64).Values) +} diff --git a/tensor/indexed.go b/tensor/indexed.go index 9c5a80037f..626b1c7652 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -54,6 +54,12 @@ func NewIndexed(tsr Tensor, idxs ...[]int) *Indexed { return ix } +// NewFloatScalar is a convenience method to quickly get an Indexed +// representation of a single float64 scalar value, for use in math routines etc. +func NewFloatScalar(val float64) *Indexed { + return &Indexed{Tensor: NewNumberFromSlice([]float64{val})} +} + // SetTensor sets as indexes into given tensor with sequential initial indexes func (ix *Indexed) SetTensor(tsr Tensor) { ix.Tensor = tsr diff --git a/tensor/stats/metric/enumgen.go b/tensor/stats/metric/enumgen.go index 1b14626b5a..409eb8e9fa 100644 --- a/tensor/stats/metric/enumgen.go +++ b/tensor/stats/metric/enumgen.go @@ -6,16 +6,16 @@ import ( "cogentcore.org/core/enums" ) -var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14} +var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} // MetricsN is the highest valid value for type Metrics, plus one. -const MetricsN Metrics = 15 +const MetricsN Metrics = 13 -var _MetricsValueMap = map[string]Metrics{`Euclidean`: 0, `L2Norm`: 1, `SumSquares`: 2, `Abs`: 3, `L1Norm`: 4, `Hamming`: 5, `EuclideanBinTol`: 6, `SumSquaresBinTol`: 7, `InvCosine`: 8, `InvCorrelation`: 9, `CrossEntropy`: 10, `InnerProduct`: 11, `Covariance`: 12, `Correlation`: 13, `Cosine`: 14} +var _MetricsValueMap = map[string]Metrics{`Euclidean`: 0, `SumSquares`: 1, `Abs`: 2, `Hamming`: 3, `EuclideanBinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `InnerProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} -var _MetricsDescMap = map[Metrics]string{0: `Euclidean is the square root of the sum of squares differences between tensor values, aka the [L2Norm].`, 1: `L2Norm is the square root of the sum of squares differences between tensor values, aka [Euclidean] distance.`, 2: `SumSquares is the sum of squares differences between tensor values.`, 3: `Abs is the sum of the absolute value of differences between tensor values, aka the [L1Norm].`, 4: `L1Norm is the sum of the absolute value of differences between tensor values (same as [Abs]).`, 5: `Hamming computes the sum of 1s for every element that is different, i.e., "city block" distance.`, 6: `EuclideanBinTol computes the [Euclidean] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 7: `SumSquaresBinTol computes the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 8: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 9: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 10: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 11: `InnerProduct is the sum of the co-products of the tensor values.`, 12: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 13: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 14: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [InnerProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} +var _MetricsDescMap = map[Metrics]string{0: `Euclidean is the square root of the sum of squares differences between tensor values, aka the L2Norm.`, 1: `SumSquares is the sum of squares differences between tensor values.`, 2: `Abs is the sum of the absolute value of differences between tensor values, aka the L1Norm.`, 3: `Hamming is the sum of 1s for every element that is different, i.e., "city block" distance.`, 4: `EuclideanBinTol is the [Euclidean] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 5: `SumSquaresBinTol is the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 6: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 7: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 8: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 9: `InnerProduct is the sum of the co-products of the tensor values.`, 10: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 11: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 12: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [InnerProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} -var _MetricsMap = map[Metrics]string{0: `Euclidean`, 1: `L2Norm`, 2: `SumSquares`, 3: `Abs`, 4: `L1Norm`, 5: `Hamming`, 6: `EuclideanBinTol`, 7: `SumSquaresBinTol`, 8: `InvCosine`, 9: `InvCorrelation`, 10: `CrossEntropy`, 11: `InnerProduct`, 12: `Covariance`, 13: `Correlation`, 14: `Cosine`} +var _MetricsMap = map[Metrics]string{0: `Euclidean`, 1: `SumSquares`, 2: `Abs`, 3: `Hamming`, 4: `EuclideanBinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `InnerProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} // String returns the string representation of this Metrics value. func (i Metrics) String() string { return enums.String(i, _MetricsMap) } diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 67a052710b..2ef32732cf 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -16,7 +16,7 @@ func TestFuncs(t *testing.T) { a64 := []float64{.5, .2, .1, .7, math.NaN(), .5} b64 := []float64{.2, .9, .1, .7, 0, .2} - results := []float64{math.Sqrt(0.67), math.Sqrt(0.67), 0.67, 1.3, 1.3, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 1.8090248566170337, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} + results := []float64{math.Sqrt(0.67), 0.67, 1.3, 3, 0.7, 0.49, 1 - 0.7319115529256469, 1 - 0.11189084777289171, 1.8090248566170337, 0.88, 0.008, 0.11189084777289171, 0.7319115529256469} tol := 1.0e-8 diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index c1f8a45ecf..09b94d9fdd 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -7,51 +7,36 @@ package metric import ( - "fmt" - + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) -// Funcs is a registry of named metric functions, -// which can then be called by standard enum or -// string name for custom functions. -var Funcs map[string]MetricFunc - func init() { - Funcs = make(map[string]MetricFunc) - Funcs[Euclidean.String()] = EuclideanFunc - Funcs[L2Norm.String()] = EuclideanFunc - Funcs[SumSquares.String()] = SumSquaresFunc - Funcs[Abs.String()] = AbsFunc - Funcs[L1Norm.String()] = AbsFunc - Funcs[Hamming.String()] = HammingFunc - Funcs[EuclideanBinTol.String()] = EuclideanBinTolFunc - Funcs[SumSquaresBinTol.String()] = SumSquaresBinTolFunc - Funcs[InvCosine.String()] = InvCosineFunc - Funcs[InvCorrelation.String()] = InvCorrelationFunc - Funcs[InnerProduct.String()] = InnerProductFunc - Funcs[CrossEntropy.String()] = CrossEntropyFunc - Funcs[Covariance.String()] = CovarianceFunc - Funcs[Correlation.String()] = CorrelationFunc - Funcs[Cosine.String()] = CosineFunc + tensor.AddFunc(Euclidean.String(), EuclideanFunc, 1) + tensor.AddFunc(SumSquares.String(), SumSquaresFunc, 1) + tensor.AddFunc(Abs.String(), AbsFunc, 1) + tensor.AddFunc(Hamming.String(), HammingFunc, 1) + tensor.AddFunc(EuclideanBinTol.String(), EuclideanBinTolFunc, 1) + tensor.AddFunc(SumSquaresBinTol.String(), SumSquaresBinTolFunc, 1) + tensor.AddFunc(InvCosine.String(), InvCosineFunc, 1) + tensor.AddFunc(InvCorrelation.String(), InvCorrelationFunc, 1) + tensor.AddFunc(InnerProduct.String(), InnerProductFunc, 1) + tensor.AddFunc(CrossEntropy.String(), CrossEntropyFunc, 1) + tensor.AddFunc(Covariance.String(), CovarianceFunc, 1) + tensor.AddFunc(Correlation.String(), CorrelationFunc, 1) + tensor.AddFunc(Cosine.String(), CosineFunc, 1) } // Standard calls a standard Metrics enum function on given tensors. // Output results are in the out tensor. func Standard(metric Metrics, a, b, out *tensor.Indexed) { - Funcs[metric.String()](a, b, out) + tensor.Call(metric.String(), a, b, out) } -// Call calls a registered stats function on given tensors. -// Output results are in the out tensor. Returns an -// error if name not found. -func Call(name string, a, b, out *tensor.Indexed) error { - f, ok := Funcs[name] - if !ok { - return fmt.Errorf("metric.Call: function %q not registered", name) - } - f(a, b, out) - return nil +// StandardOut calls a standard Metrics enum function on given tensors, +// returning output as a newly created tensor. +func StandardOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { + return errors.Log1(tensor.CallOut(metric.String(), a, b))[0] // note: error should never happen } // Metrics are standard metric functions @@ -59,24 +44,16 @@ type Metrics int32 //enums:enum const ( // Euclidean is the square root of the sum of squares differences - // between tensor values, aka the [L2Norm]. + // between tensor values, aka the L2Norm. Euclidean Metrics = iota - // L2Norm is the square root of the sum of squares differences - // between tensor values, aka [Euclidean] distance. - L2Norm - // SumSquares is the sum of squares differences between tensor values. SumSquares // Abs is the sum of the absolute value of differences - // between tensor values, aka the [L1Norm]. + // between tensor values, aka the L1Norm. Abs - // L1Norm is the sum of the absolute value of differences - // between tensor values (same as [Abs]). - L1Norm - // Hamming is the sum of 1s for every element that is different, // i.e., "city block" distance. Hamming diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index f6fd8c753b..d0d65a88ab 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -34,22 +34,3 @@ func ClosestRow(probe *tensor.Indexed, vocab *tensor.Indexed, mfun MetricFunc) ( } return mi, mind } - -// todo: - -// Tolerance64 sets a = b for any element where |a-b| <= tol. -// This can be called prior to any metric function. -func Tolerance64(a, b []float64, tol float64) { - if len(a) != len(b) { - panic("metric: slice lengths do not match") - } - for i, av := range a { - bv := b[i] - if math.IsNaN(av) || math.IsNaN(bv) { - continue - } - if math.Abs(av-bv) <= tol { - a[i] = bv - } - } -} diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index ca50e94294..2a86133190 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -5,59 +5,46 @@ package stats import ( - "fmt" - + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) //go:generate core generate -// Funcs is a registry of named stats functions, -// which can then be called by standard enum or -// string name for custom functions. -var Funcs map[string]StatsFunc - func init() { - Funcs = make(map[string]StatsFunc) - Funcs[Count.String()] = CountFunc - Funcs[Sum.String()] = SumFunc - Funcs[SumAbs.String()] = SumAbsFunc - Funcs[L1Norm.String()] = SumAbsFunc - Funcs[Prod.String()] = ProdFunc - Funcs[Min.String()] = MinFunc - Funcs[Max.String()] = MaxFunc - Funcs[MinAbs.String()] = MinAbsFunc - Funcs[MaxAbs.String()] = MaxAbsFunc - Funcs[Mean.String()] = MeanFunc - Funcs[Var.String()] = VarFunc - Funcs[Std.String()] = StdFunc - Funcs[Sem.String()] = SemFunc - Funcs[SumSq.String()] = SumSqFunc - Funcs[L2Norm.String()] = L2NormFunc - Funcs[VarPop.String()] = VarPopFunc - Funcs[StdPop.String()] = StdPopFunc - Funcs[SemPop.String()] = SemPopFunc - Funcs[Median.String()] = MedianFunc - Funcs[Q1.String()] = Q1Func - Funcs[Q3.String()] = Q3Func + tensor.AddFunc(Count.String(), CountFunc, 1) + tensor.AddFunc(Sum.String(), SumFunc, 1) + tensor.AddFunc(SumAbs.String(), SumAbsFunc, 1) + tensor.AddFunc(L1Norm.String(), SumAbsFunc, 1) + tensor.AddFunc(Prod.String(), ProdFunc, 1) + tensor.AddFunc(Min.String(), MinFunc, 1) + tensor.AddFunc(Max.String(), MaxFunc, 1) + tensor.AddFunc(MinAbs.String(), MinAbsFunc, 1) + tensor.AddFunc(MaxAbs.String(), MaxAbsFunc, 1) + tensor.AddFunc(Mean.String(), MeanFunc, 1) + tensor.AddFunc(Var.String(), VarFunc, 1) + tensor.AddFunc(Std.String(), StdFunc, 1) + tensor.AddFunc(Sem.String(), SemFunc, 1) + tensor.AddFunc(SumSq.String(), SumSqFunc, 1) + tensor.AddFunc(L2Norm.String(), L2NormFunc, 1) + tensor.AddFunc(VarPop.String(), VarPopFunc, 1) + tensor.AddFunc(StdPop.String(), StdPopFunc, 1) + tensor.AddFunc(SemPop.String(), SemPopFunc, 1) + tensor.AddFunc(Median.String(), MedianFunc, 1) + tensor.AddFunc(Q1.String(), Q1Func, 1) + tensor.AddFunc(Q3.String(), Q3Func, 1) } // Standard calls a standard Stats enum function on given tensors. // Output results are in the out tensor. func Standard(stat Stats, in, out *tensor.Indexed) { - Funcs[stat.String()](in, out) + tensor.Call(stat.String(), in, out) } -// Call calls a registered stats function on given tensors. -// Output results are in the out tensor. Returns an -// error if name not found. -func Call(name string, in, out *tensor.Indexed) error { - f, ok := Funcs[name] - if !ok { - return fmt.Errorf("stats.Call: function %q not registered", name) - } - f(in, out) - return nil +// StandardOut calls a standard Stats enum function on given tensor, +// returning output as a newly created tensor. +func StandardOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { + return errors.Log1(tensor.CallOut(stat.String(), in))[0] // note: error should never happen } // Stats is a list of different standard aggregation functions, which can be used diff --git a/tensor/stats/tmath/README.md b/tensor/stats/tmath/README.md deleted file mode 100644 index 303bcf8e31..0000000000 --- a/tensor/stats/tmath/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# tmath is the Tensor math library - -# math functions - -All the standard library [math](https://pkg.go.dev/math) functions are implemented on `*tensor.Indexed`. - -To properly handle the row-wise indexes, all processing is done using row, cell indexes, with the row indirected through the indexes. - -The standard Add, Sub, Mul, Div mathematical operators all operate element-wise, with a separate MatMul for matrix multiplication, which operates through gonum routines, for 2D Float64 tensor shapes with no indexes, so that the raw float64 values can be passed directly to gonum. - -# norm functions - -* DivNorm does divisive normalization of elements -* SubNorm does subtractive normalization of elements -* ZScore subtracts the mean and divides by the standard deviation -* Abs performs absolute-value on all elements (e.g., use prior to [stats](../stats) to produce Mean of Abs vals etc). - - diff --git a/tensor/stats/tmath/norm.go b/tensor/stats/tmath/norm.go deleted file mode 100644 index 9fe9c2ef2a..0000000000 --- a/tensor/stats/tmath/norm.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tmath - -import ( - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/stats" -) - -// ZScore computes Z-normalized values into given output tensor, -// subtracting the Mean and dividing by the standard deviation. -func ZScore(a, out *tensor.Indexed) { - osh := stats.OutShape(a.Tensor.Shape()) - mout := tensor.NewIndexed(tensor.NewFloat64(osh.Sizes)) - std, mean, _ := stats.StdFuncOut64(a, mout) - out.SetShapeFrom(a) - Sub(a, mean, out) - Div(out, std, out) // for use out as in too -} - -// UnitNorm computes unit normalized values into given output tensor, -// subtracting the Min value and dividing by the Max of the remaining numbers. -func UnitNorm(a, out *tensor.Indexed) { - osh := stats.OutShape(a.Tensor.Shape()) - mout := tensor.NewIndexed(tensor.NewFloat64(osh.Sizes)) - stats.MinFunc(a, mout) - out.SetShapeFrom(a) - Sub(a, mout, out) - stats.MaxFunc(out, mout) - Div(out, mout, out) // for use out as in too -} - -/* - -/////////////////////////////////////////// -// Thresh - - -// Thresh64 thresholds the values of the vector -- anything above the high threshold is set -// to the high value, and everything below the low threshold is set to the low value. -func Thresh64(a []float64, hi bool, hiThr float64, lo bool, loThr float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - if hi && av > hiThr { - a[i] = hiThr - } - if lo && av < loThr { - a[i] = loThr - } - } -} - -/////////////////////////////////////////// -// Binarize - -// Binarize64 turns vector into binary-valued, by setting anything >= the threshold -// to the high value, and everything below to the low value. -func Binarize64(a []float64, thr, hiVal, loVal float64) { - for i, av := range a { - if math.IsNaN(av) { - continue - } - if av >= thr { - a[i] = hiVal - } else { - a[i] = loVal - } - } -} - -*/ diff --git a/tensor/stats/tmath/tmath.go b/tensor/stats/tmath/tmath.go deleted file mode 100644 index 91ff236faa..0000000000 --- a/tensor/stats/tmath/tmath.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tmath - -import "cogentcore.org/core/tensor" - -//go:generate core generate - -// Func1in1out is a function that has 1 tensor input and 1 output. -type Func1in1out func(in, out *tensor.Indexed) - -// Func2in1out is a function that has 2 tensor input and 1 output. -type Func2in1out func(in, out *tensor.Indexed) - -// Funcs1in1out is a registry of named math functions that -// take one input and one output tensor. -var Funcs1in1out map[string]Func1in1out - -// Funcs2in1out is a registry of named math functions that -// take two inputs and one output tensor. -var Funcs2in1out map[string]Func2in1out diff --git a/tensor/tmath/README.md b/tensor/tmath/README.md new file mode 100644 index 0000000000..1133efd10d --- /dev/null +++ b/tensor/tmath/README.md @@ -0,0 +1,18 @@ +# tmath is the Tensor math library + +# math functions + +All the standard library [math](https://pkg.go.dev/math) functions are implemented on `*tensor.Indexed`. + +To properly handle the row-wise indexes, all processing is done using row, cell indexes, with the row indirected through the indexes. + +The output result tensor(s) can be the same as the input for all functions (except where specifically noted), to perform an in-place operation on the same data. + +The standard `Add`, `Sub`, `Mul`, `Div` (`+, -, *, /`) mathematical operators all operate element-wise, with a separate MatMul for matrix multiplication, which operates through gonum routines, for 2D Float64 tensor shapes with no indexes, so that the raw float64 values can be passed directly to gonum. + +# norm functions + +* `UnitNorm` subtracts `min` and divides by resulting `max` to normalize to 0..1 unit range. +* `ZScore` subtracts the mean and divides by the standard deviation. +* `Clamp` enforces min, max range, clamping values to those bounds if they exceed them. +* `Binarize` sets all values below a given threshold to 0, and those above to 1. diff --git a/tensor/stats/tmath/doc.go b/tensor/tmath/doc.go similarity index 100% rename from tensor/stats/tmath/doc.go rename to tensor/tmath/doc.go diff --git a/tensor/stats/tmath/math.go b/tensor/tmath/math.go similarity index 58% rename from tensor/stats/tmath/math.go rename to tensor/tmath/math.go index 1fba5c3d4d..39008ec87b 100644 --- a/tensor/stats/tmath/math.go +++ b/tensor/tmath/math.go @@ -10,299 +10,339 @@ import ( "cogentcore.org/core/tensor" ) +func init() { + tensor.AddFunc("Abs", Abs, 1) + tensor.AddFunc("Acos", Acos, 1) + tensor.AddFunc("Acosh", Acosh, 1) + tensor.AddFunc("Asin", Asin, 1) + tensor.AddFunc("Asinh", Asinh, 1) + tensor.AddFunc("Atan", Atan, 1) + tensor.AddFunc("Atanh", Atanh, 1) + tensor.AddFunc("Cbrt", Cbrt, 1) + tensor.AddFunc("Ceil", Ceil, 1) + tensor.AddFunc("Cos", Cos, 1) + tensor.AddFunc("Cosh", Cosh, 1) + tensor.AddFunc("Erf", Erf, 1) + tensor.AddFunc("Erfc", Erfc, 1) + tensor.AddFunc("Erfcinv", Erfcinv, 1) + tensor.AddFunc("Erfinv", Erfinv, 1) + tensor.AddFunc("Exp", Exp, 1) + tensor.AddFunc("Exp2", Exp2, 1) + tensor.AddFunc("Expm1", Expm1, 1) + tensor.AddFunc("Floor", Floor, 1) + tensor.AddFunc("Gamma", Gamma, 1) + tensor.AddFunc("J0", J0, 1) + tensor.AddFunc("J1", J1, 1) + tensor.AddFunc("Log", Log, 1) + tensor.AddFunc("Log10", Log10, 1) + tensor.AddFunc("Log1p", Log1p, 1) + tensor.AddFunc("Log2", Log2, 1) + tensor.AddFunc("Logb", Logb, 1) + tensor.AddFunc("Round", Round, 1) + tensor.AddFunc("RoundToEven", RoundToEven, 1) + tensor.AddFunc("Sin", Sin, 1) + tensor.AddFunc("Sinh", Sinh, 1) + tensor.AddFunc("Sqrt", Sqrt, 1) + tensor.AddFunc("Tan", Tan, 1) + tensor.AddFunc("Tanh", Tanh, 1) + tensor.AddFunc("Trunc", Trunc, 1) + tensor.AddFunc("Y0", Y0, 1) + tensor.AddFunc("Y1", Y1, 1) +} + func Abs(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Abs(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Abs(tsr[0].Tensor.Float1D(i))) }, in, out) } func Acos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Acos(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Acos(tsr[0].Tensor.Float1D(i))) }, in, out) } func Acosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Acosh(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Acosh(tsr[0].Tensor.Float1D(i))) }, in, out) } func Asin(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Asin(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Asin(tsr[0].Tensor.Float1D(i))) }, in, out) } func Asinh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Asinh(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Asinh(tsr[0].Tensor.Float1D(i))) }, in, out) } func Atan(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Atan(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Atan(tsr[0].Tensor.Float1D(i))) }, in, out) } func Atanh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Atanh(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Atanh(tsr[0].Tensor.Float1D(i))) }, in, out) } func Cbrt(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Cbrt(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Cbrt(tsr[0].Tensor.Float1D(i))) }, in, out) } func Ceil(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Ceil(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Ceil(tsr[0].Tensor.Float1D(i))) }, in, out) } func Cos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Cos(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Cos(tsr[0].Tensor.Float1D(i))) }, in, out) } func Cosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Cosh(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Cosh(tsr[0].Tensor.Float1D(i))) }, in, out) } func Erf(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Erf(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Erf(tsr[0].Tensor.Float1D(i))) }, in, out) } func Erfc(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Erfc(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Erfc(tsr[0].Tensor.Float1D(i))) }, in, out) } func Erfcinv(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Erfcinv(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Erfcinv(tsr[0].Tensor.Float1D(i))) }, in, out) } func Erfinv(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Erfinv(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Erfinv(tsr[0].Tensor.Float1D(i))) }, in, out) } func Exp(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Exp(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Exp(tsr[0].Tensor.Float1D(i))) }, in, out) } func Exp2(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Exp2(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Exp2(tsr[0].Tensor.Float1D(i))) }, in, out) } func Expm1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Expm1(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Expm1(tsr[0].Tensor.Float1D(i))) }, in, out) } func Floor(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Floor(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Floor(tsr[0].Tensor.Float1D(i))) }, in, out) } func Gamma(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Gamma(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Gamma(tsr[0].Tensor.Float1D(i))) }, in, out) } func J0(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.J0(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.J0(tsr[0].Tensor.Float1D(i))) }, in, out) } func J1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.J1(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.J1(tsr[0].Tensor.Float1D(i))) }, in, out) } func Log(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Log(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Log(tsr[0].Tensor.Float1D(i))) }, in, out) } func Log10(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Log10(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Log10(tsr[0].Tensor.Float1D(i))) }, in, out) } func Log1p(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Log1p(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Log1p(tsr[0].Tensor.Float1D(i))) }, in, out) } func Log2(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Log2(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Log2(tsr[0].Tensor.Float1D(i))) }, in, out) } func Logb(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Logb(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Logb(tsr[0].Tensor.Float1D(i))) }, in, out) } func Round(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Round(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Round(tsr[0].Tensor.Float1D(i))) }, in, out) } func RoundToEven(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.RoundToEven(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.RoundToEven(tsr[0].Tensor.Float1D(i))) }, in, out) } func Sin(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Sin(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Sin(tsr[0].Tensor.Float1D(i))) }, in, out) } func Sinh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Sinh(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Sinh(tsr[0].Tensor.Float1D(i))) }, in, out) } func Sqrt(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Sqrt(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Sqrt(tsr[0].Tensor.Float1D(i))) }, in, out) } func Tan(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Tan(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Tan(tsr[0].Tensor.Float1D(i))) }, in, out) } func Tanh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Tanh(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Tanh(tsr[0].Tensor.Float1D(i))) }, in, out) } func Trunc(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Trunc(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Trunc(tsr[0].Tensor.Float1D(i))) }, in, out) } func Y0(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Y0(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Y0(tsr[0].Tensor.Float1D(i))) }, in, out) } func Y1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i1d, math.Y1(tsr[0].Tensor.Float1D(i1d))) + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math.Y1(tsr[0].Tensor.Float1D(i))) }, in, out) } diff --git a/tensor/tmath/norm.go b/tensor/tmath/norm.go new file mode 100644 index 0000000000..9cee1295d2 --- /dev/null +++ b/tensor/tmath/norm.go @@ -0,0 +1,61 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" +) + +// ZScore computes Z-normalized values into given output tensor, +// subtracting the Mean and dividing by the standard deviation. +func ZScore(a, out *tensor.Indexed) { + mout := tensor.NewIndexed(tensor.NewFloat64(nil)) + std, mean, _ := stats.StdFuncOut64(a, mout) + Sub(a, mean, out) + Div(out, std, out) +} + +// UnitNorm computes unit normalized values into given output tensor, +// subtracting the Min value and dividing by the Max of the remaining numbers. +func UnitNorm(a, out *tensor.Indexed) { + mout := tensor.NewIndexed(tensor.NewFloat64(nil)) + stats.MinFunc(a, mout) + Sub(a, mout, out) + stats.MaxFunc(out, mout) + Div(out, mout, out) +} + +// Clamp ensures that all values are within min, max limits, clamping +// values to those bounds if they exceed them. min and max args are +// treated as scalars (first value used). +func Clamp(in, minv, maxv, out *tensor.Indexed) { + out.SetShapeFrom(in) + mn := minv.Tensor.Float1D(0) + mx := maxv.Tensor.Float1D(0) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i, _, _ := tsr[0].RowCellIndex(idx) + tsr[1].Tensor.SetFloat1D(i, math32.Clamp64(tsr[0].Tensor.Float1D(i), mn, mx)) + }, in, out) +} + +// Binarize results in a binary-valued output by setting +// values >= the threshold to 1, else 0. threshold is +// treated as a scalar (first value used). +func Binarize(in, threshold, out *tensor.Indexed) { + out.SetShapeFrom(in) + thr := threshold.Tensor.Float1D(0) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + i, _, _ := tsr[0].RowCellIndex(idx) + v := tsr[0].Tensor.Float1D(i) + if v >= thr { + v = 1 + } else { + v = 0 + } + tsr[1].Tensor.SetFloat1D(i, v) + }, in, out) +} diff --git a/tensor/stats/tmath/ops.go b/tensor/tmath/ops.go similarity index 98% rename from tensor/stats/tmath/ops.go rename to tensor/tmath/ops.go index 1e095e88e7..641173192e 100644 --- a/tensor/stats/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -117,6 +117,7 @@ func SubScalar(sign float64, scalar float64, a, out *tensor.Indexed) { // SubSubSpace subtracts the subspace tensor to each row in the given tensor, // into the output tensor. +// sign determines which way the subtraction goes: 1 = a-sub, -1 = sub-a func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { @@ -235,7 +236,8 @@ func DivScalar(scalar float64, a, out *tensor.Indexed) { }, a, out) } -// DivScalarInv divides scalar by given tensor elements into output. +// DivScalarInv divides scalar by given tensor elements into output +// (inverse of [DivScalar]). func DivScalarInv(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { @@ -256,7 +258,7 @@ func DivSubSpace(a, sub, out *tensor.Indexed) { } // DivSubSpaceInv divides the subspace tensor by each row of the given tensor, -// into the output tensor. +// into the output tensor (inverse of [DivSubSpace]) func DivSubSpaceInv(a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { diff --git a/tensor/stats/tmath/ops_test.go b/tensor/tmath/ops_test.go similarity index 76% rename from tensor/stats/tmath/ops_test.go rename to tensor/tmath/ops_test.go index 7037d19ded..6bb024be98 100644 --- a/tensor/stats/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -5,10 +5,10 @@ package tmath import ( - "fmt" "testing" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" "github.com/stretchr/testify/assert" ) @@ -129,9 +129,32 @@ func TestAdd(t *testing.T) { } ZScore(oned, oneout) - fmt.Println(oneout.Tensor) + mout := tensor.NewIndexed(tensor.NewFloat64(nil)) + std, mean, _ := stats.StdFuncOut64(oneout, mout) + assert.InDelta(t, 1.0, std.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 0.0, mean.Tensor.Float1D(0), 1.0e-6) UnitNorm(oned, oneout) - fmt.Println(oneout.Tensor) - + stats.MinFunc(oneout, mout) + assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) + stats.MaxFunc(oneout, mout) + assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) + // fmt.Println(oneout.Tensor) + + minv := tensor.NewFloatScalar(0) + maxv := tensor.NewFloatScalar(1) + Clamp(oned, minv, maxv, oneout) + stats.MinFunc(oneout, mout) + assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) + stats.MaxFunc(oneout, mout) + assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) + // fmt.Println(oneout.Tensor) + + thr := tensor.NewFloatScalar(0.5) + Binarize(oned, thr, oneout) + stats.MinFunc(oneout, mout) + assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) + stats.MaxFunc(oneout, mout) + assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) + // fmt.Println(oneout.Tensor) } diff --git a/tensor/tmath/tmath.go b/tensor/tmath/tmath.go new file mode 100644 index 0000000000..48896141f1 --- /dev/null +++ b/tensor/tmath/tmath.go @@ -0,0 +1,7 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +//go:generate core generate diff --git a/tensor/stats/tmath/tmath_test.go b/tensor/tmath/tmath_test.go similarity index 85% rename from tensor/stats/tmath/tmath_test.go rename to tensor/tmath/tmath_test.go index 5500045140..691291174b 100644 --- a/tensor/stats/tmath/tmath_test.go +++ b/tensor/tmath/tmath_test.go @@ -43,8 +43,8 @@ func TestMath(t *testing.T) { // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() - mfuncs := []onef{math.Abs, math.Acos, math.Acosh, math.Asin, math.Asinh, math.Atan, math.Atanh, math.Cbrt, math.Ceil, math.Cos, math.Cosh, math.Erf, math.Erfc, math.Erfcinv, math.Erfcinv, math.Erfinv, math.Exp, math.Exp2, math.Expm1, math.Floor, math.Gamma, math.J0, math.J1, math.Log, math.Log10, math.Log1p, math.Log2, math.Logb, math.Round, math.RoundToEven, math.Sin, math.Sinh, math.Sqrt, math.Tan, math.Tanh, math.Trunc, math.Y0, math.Y1} - tfuncs := []Func1in1out{Abs, Acos, Acosh, Asin, Asinh, Atan, Atanh, Cbrt, Ceil, Cos, Cosh, Erf, Erfc, Erfcinv, Erfcinv, Erfinv, Exp, Exp2, Expm1, Floor, Gamma, J0, J1, Log, Log10, Log1p, Log2, Logb, Round, RoundToEven, Sin, Sinh, Sqrt, Tan, Tanh, Trunc, Y0, Y1} + mfuncs := []onef{math.Abs, math.Acos, math.Acosh, math.Asin, math.Asinh, math.Atan, math.Atanh, math.Cbrt, math.Ceil, math.Cos, math.Cosh, math.Erf, math.Erfc, math.Erfcinv, math.Erfinv, math.Exp, math.Exp2, math.Expm1, math.Floor, math.Gamma, math.J0, math.J1, math.Log, math.Log10, math.Log1p, math.Log2, math.Logb, math.Round, math.RoundToEven, math.Sin, math.Sinh, math.Sqrt, math.Tan, math.Tanh, math.Trunc, math.Y0, math.Y1} + tfuncs := []Func1in1out{Abs, Acos, Acosh, Asin, Asinh, Atan, Atanh, Cbrt, Ceil, Cos, Cosh, Erf, Erfc, Erfcinv, Erfinv, Exp, Exp2, Expm1, Floor, Gamma, J0, J1, Log, Log10, Log1p, Log2, Logb, Round, RoundToEven, Sin, Sinh, Sqrt, Tan, Tanh, Trunc, Y0, Y1} for i, fun := range mfuncs { tf := tfuncs[i] From 032cd52031dd4bdc464a22b20c4951d6fe38cea4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 12 Sep 2024 00:17:08 -0700 Subject: [PATCH 019/311] distance / similarity matrix computation goes in metric where it belongs -- simat goes bye bye after doing LabeledMatrix part. --- tensor/indexed.go | 10 +++ tensor/stats/metric/README.md | 2 + tensor/stats/metric/matrix.go | 115 +++++++++++++++++++++++++++++ tensor/stats/metric/metric_test.go | 28 +++++++ tensor/stats/metric/misc.go | 9 +-- tensor/stats/simat/README.md | 4 +- tensor/tensor.go | 6 +- 7 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 tensor/stats/metric/matrix.go diff --git a/tensor/indexed.go b/tensor/indexed.go index 626b1c7652..b4d2d038d2 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -87,6 +87,15 @@ func (ix *Indexed) Index(idx int) int { return ix.Indexes[idx] } +// RowCellSize returns the size of the outer-most Row shape dimension +// (via [Indexed.Rows] method), and the size of all the remaining +// inner dimensions (the "cell" size). +func (ix *Indexed) RowCellSize() (rows, cells int) { + _, cells = ix.Tensor.RowCellSize() + rows = ix.Rows() + return +} + // RowCellIndex returns the direct Values index into underlying tensor // based on given overall row * cell index. func (ix *Indexed) RowCellIndex(idx int) (i1d, ri, ci int) { @@ -430,6 +439,7 @@ func (ix *Indexed) Float(i []int) float64 { func (ix *Indexed) SetFloat(i []int, val float64) { if ix.Indexes == nil { ix.Tensor.SetFloat(i, val) + return } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index d628925b96..3f40867380 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -2,6 +2,8 @@ `metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Indexed` standard data representation. +The `Matrix` function returns a distance / similarity matrix computed from the n-dimensional "cells" of row-organized tensor data, and the `SimMat` type provides labels for displaying such matricies. + ## Metrics ### Value _increases_ with increasing distance (i.e., difference metric) diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go new file mode 100644 index 0000000000..c081b7e493 --- /dev/null +++ b/tensor/stats/metric/matrix.go @@ -0,0 +1,115 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metric + +import ( + "cogentcore.org/core/math32/vecint" + "cogentcore.org/core/tensor" +) + +// note: this answer gives an index into the upper triangular +// https://math.stackexchange.com/questions/2134011/conversion-of-upper-triangle-linear-index-from-index-on-symmetrical-array +// return TriangularN(n) - ((n-c)*(n-c-1))/2 + r - c - 1 <- this works for lower excluding diag +// return (n * (n - 1) / 2) - ((n-r)*(n-r-1))/2 + c <- this works for upper including diag +// but I wasn't able to get an equation for r, c back from index, for this "including diagonal" +// https://stackoverflow.com/questions/27086195/linear-index-upper-triangular-matrix?rq=3 +// python just iterates manually and returns a list +// https://github.com/numpy/numpy/blob/v2.1.0/numpy/lib/_twodim_base_impl.py#L902-L985 + +// TriangularN returns the number of elements in the triangular region +// of a square matrix of given size, where the triangle includes the +// n elements along the diagonal. +func TriangularN(n int) int { + return n + (n*(n-1))/2 +} + +// TriangularLIndicies returns the list of r, c indexes (as X, Y coordinates) +// for the lower triangular portion of a square matrix of size n, +// including the diagonal. +func TriangularLIndicies(n int) []vecint.Vector2i { + trin := TriangularN(n) + coords := make([]vecint.Vector2i, trin) + i := 0 + for r := range n { + for c := range n { + if c <= r { + coords[i] = vecint.Vector2i{X: r, Y: c} + i++ + } + } + } + return coords +} + +// Matrix computes the rows x rows distance / similarity matrix between +// all sub-space cells of the given higher dimensional input tensor, +// which must have at least 2 dimensions: the outer-most rows, +// and within that, 1+dimensional patterns that the given distance metric +// function is applied to, with the results filling the elements of the output matrix. +// The resulting matrix is symmetric, and only the lower triangular part +// is computed, with results copied to the upper triangular region, +// for maximum efficiency. +// See also [LabeledMatrix] struct which can add labels for displaying the matrix. +func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { + rows, cells := in.RowCellSize() + if rows == 0 || cells == 0 { + return + } + msz := []int{rows, rows} + out.Tensor.SetShape(msz) + mout := tensor.NewFloatScalar(0.0) + coords := TriangularLIndicies(rows) + nc := len(coords) + // note: flops estimating 3 per item on average -- different for different metrics. + tensor.VectorizeThreaded(cells*3, func(tsr ...*tensor.Indexed) int { return nc }, + func(idx int, tsr ...*tensor.Indexed) { + c := coords[idx] + sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].Tensor.SubSpace([]int{tsr[0].Index(c.X)}))) + sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].Tensor.SubSpace([]int{tsr[0].Index(c.Y)}))) + mfun(sa, sb, mout) + tsr[1].SetFloat([]int{c.X, c.Y}, mout.Tensor.Float1D(0)) + }, in, out) + for _, c := range coords { // copy to upper + if c.X == c.Y { // exclude diag + continue + } + out.Tensor.SetFloat([]int{c.Y, c.X}, out.Tensor.Float([]int{c.X, c.Y})) + } +} + +// CrossMatrix computes the distance / similarity matrix between +// two different sets of patterns in the two input tensors, where +// the patterns are in the sub-space cells of the tensors, +// which must have at least 2 dimensions: the outer-most rows, +// and within that, 1+dimensional patterns that the given distance metric +// function is applied to, with the results filling in the cells of the output matrix. +// The rows of the output matrix are the rows of the first input tensor, +// and the columns of the output are the rows of the second input tensor. +// See also [LabeledMatrix] struct which can add labels for displaying the matrix. +func CrossMatrix(a, b, out *tensor.Indexed, mfun MetricFunc) { + arows, acells := a.RowCellSize() + if arows == 0 || acells == 0 { + return + } + brows, bcells := b.RowCellSize() + if brows == 0 || bcells == 0 { + return + } + msz := []int{arows, brows} + out.Tensor.SetShape(msz) + mout := tensor.NewFloatScalar(0.0) + // note: flops estimating 3 per item on average -- different for different metrics. + flops := min(acells, bcells) * 3 + nc := arows * brows + tensor.VectorizeThreaded(flops, func(tsr ...*tensor.Indexed) int { return nc }, + func(idx int, tsr ...*tensor.Indexed) { + ar := idx / brows + br := idx % brows + sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].Tensor.SubSpace([]int{tsr[0].Index(ar)}))) + sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[1].Tensor.SubSpace([]int{tsr[1].Index(br)}))) + mfun(sa, sb, mout) + tsr[2].SetFloat([]int{ar, br}, mout.Tensor.Float1D(0)) + }, a, b, out) +} diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 2ef32732cf..8b5fcde656 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -8,7 +8,9 @@ import ( "math" "testing" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" "github.com/stretchr/testify/assert" ) @@ -69,3 +71,29 @@ func TestFuncs(t *testing.T) { assert.InDelta(t, results[met], out.Values[0], tol) } } + +func TestMatrix(t *testing.T) { + var simres = `Tensor: [12, 12] +[0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 +[1]: 3.4641016151377544 0 9.38083151964686 8.717797887081348 9.273618495495704 8.831760866327848 5.830951894845301 4.69041575982343 8.717797887081348 7.937253933193772 6.324555320336759 5.291502622129181 +[2]: 8.831760866327848 9.38083151964686 0 3.4641016151377544 4.242640687119285 5.0990195135927845 9.38083151964686 9.899494936611665 4.47213595499958 5.744562646538029 9.38083151964686 9.899494936611665 +[3]: 9.273618495495704 8.717797887081348 3.4641016151377544 0 5.477225575051661 3.7416573867739413 9.797958971132712 9.273618495495704 5.656854249492381 4.58257569495584 9.797958971132712 9.273618495495704 +[4]: 8.717797887081348 9.273618495495704 4.242640687119285 5.477225575051661 0 4 8.831760866327848 9.38083151964686 4.242640687119285 5.5677643628300215 8.831760866327848 9.38083151964686 +[5]: 9.38083151964686 8.831760866327848 5.0990195135927845 3.7416573867739413 4 0 9.486832980505138 8.94427190999916 5.830951894845301 4.795831523312719 9.486832980505138 8.94427190999916 +[6]: 4.69041575982343 5.830951894845301 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 0 3.4641016151377544 9.16515138991168 9.539392014169456 4.242640687119285 5.477225575051661 +[7]: 5.830951894845301 4.69041575982343 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 3.4641016151377544 0 9.695359714832659 9 5.477225575051661 4.242640687119285 +[8]: 8.12403840463596 8.717797887081348 4.47213595499958 5.656854249492381 4.242640687119285 5.830951894845301 9.16515138991168 9.695359714832659 0 3.605551275463989 9.16515138991168 9.695359714832659 +[9]: 8.54400374531753 7.937253933193772 5.744562646538029 4.58257569495584 5.5677643628300215 4.795831523312719 9.539392014169456 9 3.605551275463989 0 9.539392014169456 9 +[10]: 5.291502622129181 6.324555320336759 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 4.242640687119285 5.477225575051661 9.16515138991168 9.539392014169456 0 3.4641016151377544 +[11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 +` + dt := &table.Table{} + err := dt.OpenCSV("../clust/testdata/faces.dat", table.Tab) + assert.NoError(t, err) + // smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) + in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) + out := tensor.NewIndexed(tensor.NewFloat64(nil)) + Matrix(in, out, EuclideanFunc) + // fmt.Println(out.Tensor) + assert.Equal(t, simres, out.Tensor.String()) +} diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index d0d65a88ab..5c631613d4 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -15,18 +15,17 @@ import ( // function, *which must have the Increasing property*, i.e., larger = further. // returns the row and metric value for that row. // note: this does _not_ use any existing Indexes for the probe (but does for the vocab). -func ClosestRow(probe *tensor.Indexed, vocab *tensor.Indexed, mfun MetricFunc) (int, float64) { +func ClosestRow(probe, vocab *tensor.Indexed, mfun MetricFunc) (int, float64) { rows, _ := vocab.Tensor.RowCellSize() mi := -1 - out := tensor.NewFloat64([]int{1}) - oi := tensor.NewIndexed(out) + out := tensor.NewFloatScalar(0.0) // todo: need a 1d view of both spaces mind := math.MaxFloat64 prview := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) for ri := range rows { sub := tensor.NewIndexed(tensor.New1DViewOf(vocab.Tensor.SubSpace([]int{vocab.Index(ri)}))) - mfun(prview, sub, oi) - d := out.Values[0] + mfun(prview, sub, out) + d := out.Tensor.Float1D(0) if d < mind { mi = ri mind = d diff --git a/tensor/stats/simat/README.md b/tensor/stats/simat/README.md index 6d2f2ee7f7..71b32f2f1e 100644 --- a/tensor/stats/simat/README.md +++ b/tensor/stats/simat/README.md @@ -1,8 +1,8 @@ # simat -`simat` provides similarity / distance matrix functions that create a `SimMat` matrix from Tensor or Table data. Any metric function defined in metric package (or user-created) can be used. +`simat` provides similarity / distance matrix functions that create a `SimMat` matrix from Tensor or Table data. Any metric function defined in the [metric](../metric) package (or user-created) can be used. The SimMat contains the Tensor of the similarity matrix values, and labels for the Rows and Columns. -The `etview` package provides a `SimMatGrid` widget that displays the SimMat with the labels. +The [tensorcore](../tensorcore) package provides a `SimMatGrid` widget that displays the SimMat with the labels. diff --git a/tensor/tensor.go b/tensor/tensor.go index cbb2032f45..7950be5fd9 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -39,7 +39,8 @@ type Tensor interface { // Existing names will be preserved if not presented. SetShape(sizes []int, names ...string) - // Len returns the number of elements in the tensor (product of shape dimensions). + // Len returns the number of elements in the tensor, + // which is the product of all shape dimensions. Len() int // NumDims returns the total number of dimensions. @@ -50,7 +51,8 @@ type Tensor interface { // RowCellSize returns the size of the outer-most Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). - // Used for Tensors that are columns in a data table. + // Commonly used to organize multiple instances (rows) of higher-dimensional + // patterns (cells), and the [Indexed] type operates on the outer row dimension. RowCellSize() (rows, cells int) // DataType returns the type of the data elements in the tensor. From cdf17eba9401f4a5a9cc44ab39318fe7c9915851 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 12 Sep 2024 03:02:59 -0700 Subject: [PATCH 020/311] in process major update to use varargs for all index functions; names are purely optional to enable this, and value goes first instead of last in Set methods. --- tensor/base.go | 44 +++++++++++------- tensor/bits.go | 62 ++++++++++++++----------- tensor/funcs.go | 2 +- tensor/funcs_test.go | 2 +- tensor/indexed.go | 36 +++++++-------- tensor/io.go | 2 +- tensor/number.go | 77 +++++++++++++------------------ tensor/projection2d.go | 14 +++--- tensor/shape.go | 57 ++++++++++++++++------- tensor/stats/stats/funcs.go | 34 +++++++------- tensor/stats/stats/quantiles.go | 2 +- tensor/stats/stats/stats_test.go | 10 ++-- tensor/stats/stats/table.go | 4 +- tensor/stats/stats/vec.go | 29 ++++++------ tensor/string.go | 61 ++++++++++--------------- tensor/table/io.go | 12 ++--- tensor/table/splits.go | 12 ++--- tensor/table/table.go | 59 ++++++++++++------------ tensor/table/util.go | 2 +- tensor/tensor.go | 78 ++++++++++++++++---------------- tensor/tensor_test.go | 76 +++++++++++++++---------------- 21 files changed, 350 insertions(+), 325 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index ae5142c4a9..b4b1185294 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -61,10 +61,17 @@ func (tsr *Base[T]) Bytes() []byte { return slicesx.ToBytes(tsr.Values) } -func (tsr *Base[T]) Value(i []int) T { j := tsr.shape.Offset(i); return tsr.Values[j] } -func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[i] } -func (tsr *Base[T]) Set(i []int, val T) { j := tsr.shape.Offset(i); tsr.Values[j] = val } -func (tsr *Base[T]) Set1D(i int, val T) { tsr.Values[i] = val } +func (tsr *Base[T]) Value(i ...int) T { + return tsr.Values[tsr.shape.Offset(i...)] +} + +func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[i] } + +func (tsr *Base[T]) Set(val T, i ...int) { + tsr.Values[tsr.shape.Offset(i...)] = val +} + +func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[i] = val } // view is implementation of View -- needs final casting func (tsr *Base[T]) view() *Base[T] { @@ -75,16 +82,18 @@ func (tsr *Base[T]) view() *Base[T] { return nw } -// SetShape sets the shape params, resizing backing storage appropriately -func (tsr *Base[T]) SetShape(sizes []int, names ...string) { - if len(sizes) == 0 { - sizes = []int{0} - } - tsr.shape.SetShape(sizes, names...) +// SetShape sets the shape params, resizing backing storage appropriately. +func (tsr *Base[T]) SetShape(sizes ...int) { + tsr.shape.SetShape(sizes...) nln := tsr.Len() tsr.Values = slicesx.SetLength(tsr.Values, nln) } +// SetNames sets the dimension names of the tensor shape. +func (tsr *Base[T]) SetNames(names ...string) { + tsr.shape.SetNames(names...) +} + // SetNumRows sets the number of rows (outer-most dimension) in a RowMajor organized tensor. func (tsr *Base[T]) SetNumRows(rows int) { rows = max(1, rows) // must be > 0 @@ -100,26 +109,29 @@ func (tsr *Base[T]) SetNumRows(rows int) { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. -func (tsr *Base[T]) subSpaceImpl(offs []int) *Base[T] { +func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { nd := tsr.NumDims() od := len(offs) if od >= nd { return nil } stsr := &Base[T]{} - stsr.SetShape(tsr.shape.Sizes[od:], tsr.shape.Names[od:]...) + stsr.SetShape(tsr.shape.Sizes[od:]...) + if tsr.shape.Names != nil { + stsr.shape.SetNames(tsr.shape.Names...) + } sti := make([]int, nd) copy(sti, offs) - stoff := tsr.shape.Offset(sti) + stoff := tsr.shape.Offset(sti...) sln := stsr.Len() stsr.Values = tsr.Values[stoff : stoff+sln] return stsr } -func (tsr *Base[T]) StringValue(i []int) string { - j := tsr.shape.Offset(i) - return reflectx.ToString(tsr.Values[j]) +func (tsr *Base[T]) StringValue(i ...int) string { + return reflectx.ToString(tsr.Values[tsr.shape.Offset(i...)]) } + func (tsr *Base[T]) String1D(off int) string { return reflectx.ToString(tsr.Values[off]) } func (tsr *Base[T]) StringRowCell(row, cell int) string { diff --git a/tensor/bits.go b/tensor/bits.go index e4f3e36e50..6590be71b0 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -27,9 +27,9 @@ type Bits struct { // NewBits returns a new n-dimensional tensor of bit values // with the given sizes per dimension (shape), and optional dimension names. -func NewBits(sizes []int, names ...string) *Bits { +func NewBits(sizes ...int) *Bits { tsr := &Bits{} - tsr.SetShape(sizes, names...) + tsr.SetShape(sizes...) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr } @@ -101,21 +101,31 @@ func (tsr *Bits) RowCellSize() (rows, cells int) { } // Value returns value at given tensor index -func (tsr *Bits) Value(i []int) bool { j := int(tsr.shape.Offset(i)); return tsr.Values.Index(j) } +func (tsr *Bits) Value(i ...int) bool { + return tsr.Values.Index(tsr.shape.Offset(i...)) +} // Value1D returns value at given tensor 1D (flat) index func (tsr *Bits) Value1D(i int) bool { return tsr.Values.Index(i) } -func (tsr *Bits) Set(i []int, val bool) { j := int(tsr.shape.Offset(i)); tsr.Values.Set(j, val) } -func (tsr *Bits) Set1D(i int, val bool) { tsr.Values.Set(i, val) } +func (tsr *Bits) Set(val bool, i ...int) { + tsr.Values.Set(tsr.shape.Offset(i...), val) +} + +func (tsr *Bits) Set1D(val bool, i int) { tsr.Values.Set(i, val) } // SetShape sets the shape params, resizing backing storage appropriately -func (tsr *Bits) SetShape(sizes []int, names ...string) { - tsr.shape.SetShape(sizes, names...) +func (tsr *Bits) SetShape(sizes ...int) { + tsr.shape.SetShape(sizes...) nln := tsr.Len() tsr.Values.SetLen(nln) } +// SetNames sets the dimension names of the tensor shape. +func (tsr *Bits) SetNames(names ...string) { + tsr.shape.SetNames(names...) +} + // SetNumRows sets the number of rows (outer-most dimension) in a RowMajor organized tensor. func (tsr *Bits) SetNumRows(rows int) { rows = max(1, rows) // must be > 0 @@ -126,36 +136,33 @@ func (tsr *Bits) SetNumRows(rows int) { } // SubSpace is not possible with Bits -func (tsr *Bits) SubSpace(offs []int) Tensor { +func (tsr *Bits) SubSpace(offs ...int) Tensor { return nil } -func (tsr *Bits) Float(i []int) float64 { - j := tsr.shape.Offset(i) - return BoolToFloat64(tsr.Values.Index(j)) +func (tsr *Bits) Float(i ...int) float64 { + return BoolToFloat64(tsr.Values.Index(tsr.shape.Offset(i...))) } -func (tsr *Bits) SetFloat(i []int, val float64) { - j := tsr.shape.Offset(i) - tsr.Values.Set(j, Float64ToBool(val)) +func (tsr *Bits) SetFloat(val float64, i ...int) { + tsr.Values.Set(tsr.shape.Offset(i...), Float64ToBool(val)) } -func (tsr *Bits) StringValue(i []int) string { - j := tsr.shape.Offset(i) - return reflectx.ToString(tsr.Values.Index(j)) +func (tsr *Bits) StringValue(i ...int) string { + return reflectx.ToString(tsr.Values.Index(tsr.shape.Offset(i...))) } -func (tsr *Bits) SetString(i []int, val string) { +func (tsr *Bits) SetString(val string, i ...int) { if bv, err := reflectx.ToBool(val); err == nil { - j := tsr.shape.Offset(i) - tsr.Values.Set(j, bv) + tsr.Values.Set(tsr.shape.Offset(i...), bv) } } func (tsr *Bits) Float1D(off int) float64 { return BoolToFloat64(tsr.Values.Index(off)) } -func (tsr *Bits) SetFloat1D(off int, val float64) { + +func (tsr *Bits) SetFloat1D(val float64, off int) { tsr.Values.Set(off, Float64ToBool(val)) } @@ -163,7 +170,8 @@ func (tsr *Bits) FloatRowCell(row, cell int) float64 { _, sz := tsr.RowCellSize() return BoolToFloat64(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bits) SetFloatRowCell(row, cell int, val float64) { + +func (tsr *Bits) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.RowCellSize() tsr.Values.Set(row*sz+cell, Float64ToBool(val)) } @@ -177,7 +185,7 @@ func (tsr *Bits) Floats(flt *[]float64) { } // SetFloats sets tensor values from a []float64 slice (copies values). -func (tsr *Bits) SetFloats(vals []float64) { +func (tsr *Bits) SetFloats(vals ...float64) { sz := min(tsr.Len(), len(vals)) for j := 0; j < sz; j++ { tsr.Values.Set(j, Float64ToBool(vals[j])) @@ -188,7 +196,7 @@ func (tsr *Bits) String1D(off int) string { return reflectx.ToString(tsr.Values.Index(off)) } -func (tsr *Bits) SetString1D(off int, val string) { +func (tsr *Bits) SetString1D(val string, off int) { if bv, err := reflectx.ToBool(val); err == nil { tsr.Values.Set(off, bv) } @@ -199,7 +207,7 @@ func (tsr *Bits) StringRowCell(row, cell int) string { return reflectx.ToString(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bits) SetStringRowCell(row, cell int, val string) { +func (tsr *Bits) SetStringRowCell(val string, row, cell int) { if bv, err := reflectx.ToBool(val); err == nil { _, sz := tsr.RowCellSize() tsr.Values.Set(row*sz+cell, bv) @@ -260,7 +268,9 @@ func (tsr *Bits) CopyFrom(frm Tensor) { // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). func (tsr *Bits) SetShapeFrom(frm Tensor) { - tsr.SetShape(frm.Shape().Sizes, frm.Shape().Names...) + sh := frm.Shape() + tsr.SetShape(sh.Sizes...) + tsr.SetNames(sh.Names...) } // CopyCellsFrom copies given range of values from other tensor into this tensor, diff --git a/tensor/funcs.go b/tensor/funcs.go index 9e77aaa56b..77652e00b7 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -180,7 +180,7 @@ func (fn *Func) CallOut(tsr ...*Indexed) ([]*Indexed, error) { } outs := make([]*Indexed, fn.Out) for i := range outs { - outs[i] = NewIndexed(NewOfType(typ, nil)) + outs[i] = NewIndexed(NewOfType(typ)) } tsr = append(tsr, outs...) err := fn.Call(tsr...) diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 6bd8681726..decdf5732c 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -15,7 +15,7 @@ func abs(in, out *Indexed) { out.SetShapeFrom(in) VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...*Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Abs(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Abs(tsr[0].Tensor.Float1D(i)), i) }, in, out) } diff --git a/tensor/indexed.go b/tensor/indexed.go index b4d2d038d2..403e69a6b2 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -345,9 +345,9 @@ func (ix *Indexed) NewTensor() Tensor { for r := range rows { for c := range cells { if str { - nt.SetStringRowCell(r, c, ix.StringRowCell(r, c)) + nt.SetStringRowCell(ix.StringRowCell(r, c), r, c) } else { - nt.SetFloatRowCell(r, c, ix.FloatRowCell(r, c)) + nt.SetFloatRowCell(ix.FloatRowCell(r, c), r, c) } } } @@ -425,25 +425,25 @@ func (ix *Indexed) Swap(i, j int) { // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. -func (ix *Indexed) Float(i []int) float64 { +func (ix *Indexed) Float(i ...int) float64 { if ix.Indexes == nil { - return ix.Tensor.Float(i) + return ix.Tensor.Float(i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.Float(ic) + return ix.Tensor.Float(ic...) } // SetFloat sets the value of given index as a float64 // The first index value is indirected through the [Indexes]. -func (ix *Indexed) SetFloat(i []int, val float64) { +func (ix *Indexed) SetFloat(val float64, i ...int) { if ix.Indexes == nil { - ix.Tensor.SetFloat(i, val) + ix.Tensor.SetFloat(val, i...) return } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetFloat(ic, val) + ix.Tensor.SetFloat(val, ic...) } // FloatRowCell returns the value at given row and cell, @@ -458,30 +458,30 @@ func (ix *Indexed) FloatRowCell(row, cell int) float64 { // where row is outer-most dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetFloatRowCell(row, cell int, val float64) { - ix.Tensor.SetFloatRowCell(ix.Index(row), cell, val) +func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { + ix.Tensor.SetFloatRowCell(val, ix.Index(row), cell) } // StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. -func (ix *Indexed) StringValue(i []int) string { +func (ix *Indexed) StringValue(i ...int) string { if ix.Indexes == nil { - return ix.Tensor.StringValue(i) + return ix.Tensor.StringValue(i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.StringValue(ic) + return ix.Tensor.StringValue(ic...) } // SetString sets the value of given index as a string // The first index value is indirected through the [Indexes]. -func (ix *Indexed) SetString(i []int, val string) { +func (ix *Indexed) SetString(val string, i ...int) { if ix.Indexes == nil { - ix.Tensor.SetString(i, val) + ix.Tensor.SetString(val, i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetString(ic, val) + ix.Tensor.SetString(val, ic...) } // StringRowCell returns the value at given row and cell, @@ -496,6 +496,6 @@ func (ix *Indexed) StringRowCell(row, cell int) string { // where row is outer-most dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetStringRowCell(row, cell int, val string) { - ix.Tensor.SetStringRowCell(ix.Index(row), cell, val) +func (ix *Indexed) SetStringRowCell(val string, row, cell int) { + ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) } diff --git a/tensor/io.go b/tensor/io.go index 78bfa69f62..f10a31010e 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -105,7 +105,7 @@ func ReadCSV(tsr Tensor, r io.Reader, delim rune) error { for ri := 0; ri < rows; ri++ { for ci := 0; ci < cols; ci++ { str := rec[ri][ci] - tsr.SetString1D(idx, str) + tsr.SetString1D(str, idx) idx++ if idx >= sz { goto done diff --git a/tensor/number.go b/tensor/number.go index 27b73c734a..f398c2e014 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -38,39 +38,39 @@ type Byte = Number[byte] // NewFloat32 returns a new Float32 tensor // with the given sizes per dimension (shape), and optional dimension names. -func NewFloat32(sizes []int, names ...string) *Float32 { - return New[float32](sizes, names...).(*Float32) +func NewFloat32(sizes ...int) *Float32 { + return New[float32](sizes...).(*Float32) } // NewFloat64 returns a new Float64 tensor // with the given sizes per dimension (shape), and optional dimension names. -func NewFloat64(sizes []int, names ...string) *Float64 { - return New[float64](sizes, names...).(*Float64) +func NewFloat64(sizes ...int) *Float64 { + return New[float64](sizes...).(*Float64) } // NewInt returns a new Int tensor // with the given sizes per dimension (shape), and optional dimension names. -func NewInt(sizes []int, names ...string) *Int { - return New[int](sizes, names...).(*Int) +func NewInt(sizes ...int) *Int { + return New[int](sizes...).(*Int) } // NewInt32 returns a new Int32 tensor // with the given sizes per dimension (shape), and optional dimension names. -func NewInt32(sizes []int, names ...string) *Int32 { - return New[int32](sizes, names...).(*Int32) +func NewInt32(sizes ...int) *Int32 { + return New[int32](sizes...).(*Int32) } // NewByte returns a new Byte tensor // with the given sizes per dimension (shape), and optional dimension names. -func NewByte(sizes []int, names ...string) *Byte { - return New[uint8](sizes, names...).(*Byte) +func NewByte(sizes ...int) *Byte { + return New[uint8](sizes...).(*Byte) } // NewNumber returns a new n-dimensional tensor of numerical values // with the given sizes per dimension (shape), and optional dimension names. -func NewNumber[T num.Number](sizes []int, names ...string) *Number[T] { +func NewNumber[T num.Number](sizes ...int) *Number[T] { tsr := &Number[T]{} - tsr.SetShape(sizes, names...) + tsr.SetShape(sizes...) tsr.Values = make([]T, tsr.Len()) return tsr } @@ -89,10 +89,9 @@ func NewNumberShape[T num.Number](shape *Shape) *Number[T] { // The resulting Tensor thus "wraps" the given values. func NewNumberFromSlice[T num.Number](vals []T) Tensor { n := len(vals) - sizes := []int{n} tsr := &Number[T]{} tsr.Values = vals - tsr.SetShape(sizes) + tsr.SetShape(n) return tsr } @@ -100,31 +99,19 @@ func (tsr *Number[T]) IsString() bool { return false } -func (tsr *Number[T]) AddScalar(i []int, val float64) float64 { - j := tsr.shape.Offset(i) - tsr.Values[j] += T(val) - return float64(tsr.Values[j]) -} - -func (tsr *Number[T]) MulScalar(i []int, val float64) float64 { - j := tsr.shape.Offset(i) - tsr.Values[j] *= T(val) - return float64(tsr.Values[j]) -} - -func (tsr *Number[T]) SetString(i []int, val string) { +func (tsr *Number[T]) SetString(val string, i ...int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { - j := tsr.shape.Offset(i) - tsr.Values[j] = T(fv) + tsr.Values[tsr.shape.Offset(i...)] = T(fv) } } -func (tsr Number[T]) SetString1D(off int, val string) { +func (tsr Number[T]) SetString1D(val string, off int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { tsr.Values[off] = T(fv) } } -func (tsr *Number[T]) SetStringRowCell(row, cell int, val string) { + +func (tsr *Number[T]) SetStringRowCell(val string, row, cell int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = T(fv) @@ -155,21 +142,19 @@ func (tsr *Number[T]) String() string { return b.String() } -func (tsr *Number[T]) Float(i []int) float64 { - j := tsr.shape.Offset(i) - return float64(tsr.Values[j]) +func (tsr *Number[T]) Float(i ...int) float64 { + return float64(tsr.Values[tsr.shape.Offset(i...)]) } -func (tsr *Number[T]) SetFloat(i []int, val float64) { - j := tsr.shape.Offset(i) - tsr.Values[j] = T(val) +func (tsr *Number[T]) SetFloat(val float64, i ...int) { + tsr.Values[tsr.shape.Offset(i...)] = T(val) } func (tsr *Number[T]) Float1D(i int) float64 { return float64(tsr.Values[i]) } -func (tsr *Number[T]) SetFloat1D(i int, val float64) { +func (tsr *Number[T]) SetFloat1D(val float64, i int) { tsr.Values[i] = T(val) } @@ -179,7 +164,7 @@ func (tsr *Number[T]) FloatRowCell(row, cell int) float64 { return float64(tsr.Values[i]) } -func (tsr *Number[T]) SetFloatRowCell(row, cell int, val float64) { +func (tsr *Number[T]) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = T(val) } @@ -201,7 +186,7 @@ func (tsr *Number[T]) Floats(flt *[]float64) { } // SetFloats sets tensor values from a []float64 slice (copies values). -func (tsr *Number[T]) SetFloats(flt []float64) { +func (tsr *Number[T]) SetFloats(flt ...float64) { switch vals := any(tsr.Values).(type) { case []float64: copy(vals, flt) @@ -220,12 +205,12 @@ func (tsr *Number[T]) At(i, j int) float64 { log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") return 0 } else if nd == 2 { - return tsr.Float([]int{i, j}) + return tsr.Float(i, j) } else { ix := make([]int, nd) ix[nd-2] = i ix[nd-1] = j - return tsr.Float(ix) + return tsr.Float(ix...) } } @@ -295,7 +280,9 @@ func (tsr *Number[T]) CopyFrom(frm Tensor) { // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). func (tsr *Number[T]) SetShapeFrom(frm Tensor) { - tsr.SetShape(frm.Shape().Sizes, frm.Shape().Names...) + sh := frm.Shape() + tsr.SetShape(sh.Sizes...) + tsr.SetNames(sh.Names...) } // CopyCellsFrom copies given range of values from other tensor into this tensor, @@ -321,8 +308,8 @@ func (tsr *Number[T]) CopyCellsFrom(frm Tensor, to, start, n int) { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. -func (tsr *Number[T]) SubSpace(offs []int) Tensor { - b := tsr.subSpaceImpl(offs) +func (tsr *Number[T]) SubSpace(offs ...int) Tensor { + b := tsr.subSpaceImpl(offs...) rt := &Number[T]{Base: *b} return rt } diff --git a/tensor/projection2d.go b/tensor/projection2d.go index 592cecc4e1..62a8e3be3d 100644 --- a/tensor/projection2d.go +++ b/tensor/projection2d.go @@ -68,18 +68,18 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { return col } case 2: - return shp.Offset([]int{row, col}) + return shp.Offset(row, col) case 3: if oddRow { ny := shp.DimSize(1) yy := row / ny y := row % ny - return shp.Offset([]int{yy, y, col}) + return shp.Offset(yy, y, col) } else { nx := shp.DimSize(2) xx := col / nx x := col % nx - return shp.Offset([]int{xx, row, x}) + return shp.Offset(xx, row, x) } case 4: ny := shp.DimSize(2) @@ -88,7 +88,7 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { nx := shp.DimSize(3) xx := col / nx x := col % nx - return shp.Offset([]int{yy, xx, y, x}) + return shp.Offset(yy, xx, y, x) case 5: // todo: oddRows version! nyy := shp.DimSize(1) @@ -100,7 +100,7 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { nx := shp.DimSize(4) xx := col / nx x := col % nx - return shp.Offset([]int{yyy, yy, xx, y, x}) + return shp.Offset(yyy, yy, xx, y, x) } return 0 } @@ -166,7 +166,7 @@ func Projection2DString(tsr Tensor, oddRow bool, row, col int) string { // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DSet(tsr Tensor, oddRow bool, row, col int, val float64) { idx := Projection2DIndex(tsr.Shape(), oddRow, row, col) - tsr.SetFloat1D(idx, val) + tsr.SetFloat1D(val, idx) } // Projection2DSetString sets a string value at given row, col coords for a 2D projection @@ -176,5 +176,5 @@ func Projection2DSet(tsr Tensor, oddRow bool, row, col int, val float64) { // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DSetString(tsr Tensor, oddRow bool, row, col int, val string) { idx := Projection2DIndex(tsr.Shape(), oddRow, row, col) - tsr.SetString1D(idx, val) + tsr.SetString1D(val, idx) } diff --git a/tensor/shape.go b/tensor/shape.go index 097b3b6ca7..e878d9b6b6 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -7,6 +7,8 @@ package tensor import ( "fmt" "slices" + + "cogentcore.org/core/base/slicesx" ) // Shape manages a tensor's shape information, including strides and dimension names @@ -21,26 +23,31 @@ type Shape struct { // offsets for each dimension. Strides []int `display:"-"` - // names of each dimension. + // optional names of each dimension. Names []string `display:"-"` } -// NewShape returns a new shape with given sizes and optional dimension names. +// NewShape returns a new shape with given sizes. // RowMajor ordering is used by default. -func NewShape(sizes []int, names ...string) *Shape { +func NewShape(sizes ...int) *Shape { sh := &Shape{} - sh.SetShape(sizes, names...) + sh.SetShape(sizes...) return sh } // SetShape sets the shape size and optional names // RowMajor ordering is used by default. -func (sh *Shape) SetShape(sizes []int, names ...string) { +func (sh *Shape) SetShape(sizes ...int) { sh.Sizes = slices.Clone(sizes) sh.Strides = RowMajorStrides(sizes) - sh.Names = make([]string, len(sh.Sizes)) - if len(names) == len(sizes) { - copy(sh.Names, names) +} + +// SetNames sets the shape dimension names. +func (sh *Shape) SetNames(names ...string) { + if len(names) == 0 { + sh.Names = nil + } else { + sh.Names = slicesx.SetLength(names, len(sh.Sizes)) } } @@ -49,7 +56,11 @@ func (sh *Shape) SetShape(sizes []int, names ...string) { func (sh *Shape) CopyShape(cp *Shape) { sh.Sizes = slices.Clone(cp.Sizes) sh.Strides = slices.Clone(cp.Strides) - sh.Names = slices.Clone(cp.Names) + if cp.Names == nil { + sh.Names = nil + } else { + sh.Names = slices.Clone(cp.Names) + } } // Len returns the total length of elements in the tensor @@ -69,10 +80,20 @@ func (sh *Shape) Len() int { func (sh *Shape) NumDims() int { return len(sh.Sizes) } // DimSize returns the size of given dimension. -func (sh *Shape) DimSize(i int) int { return sh.Sizes[i] } +func (sh *Shape) DimSize(i int) int { + if sh.Sizes == nil { + return 0 + } + return sh.Sizes[i] +} // DimName returns the name of given dimension. -func (sh *Shape) DimName(i int) string { return sh.Names[i] } +func (sh *Shape) DimName(i int) string { + if sh.Names == nil { + return "" + } + return sh.Names[i] +} // DimByName returns the index of the given dimension name. // returns -1 if not found. @@ -130,7 +151,7 @@ func (sh *Shape) RowCellSize() (rows, cells int) { // Offset returns the "flat" 1D array index into an element at the given n-dimensional index. // No checking is done on the length or size of the index values relative to the shape of the tensor. -func (sh *Shape) Offset(index []int) int { +func (sh *Shape) Offset(index ...int) int { var offset int for i, v := range index { offset += v * sh.Strides[i] @@ -236,8 +257,12 @@ func AddShapes(shape1, shape2 *Shape) *Shape { nsh := make([]int, len(sh1)+len(sh2)) copy(nsh, sh1) copy(nsh[len(sh1):], sh2) - nms := make([]string, len(sh1)+len(sh2)) - copy(nms, shape1.Names) - copy(nms[len(sh1):], shape2.Names) - return NewShape(nsh, nms...) + sh := NewShape(nsh...) + if shape1.Names != nil || shape2.Names != nil { + nms := make([]string, len(sh1)+len(sh2)) + copy(nms, shape1.Names) + copy(nms[len(sh1):], shape2.Names) + sh.SetNames(nms...) + } + return sh } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 1ee43e79b7..00d979ef76 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -132,8 +132,8 @@ func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { if c == 0 { continue } - sum64.Tensor.SetFloat1D(i, sum64.Tensor.Float1D(i)/c) - out.Tensor.SetFloat1D(i, sum64.Tensor.Float1D(i)) + sum64.Tensor.SetFloat1D(sum64.Tensor.Float1D(i)/c, i) + out.Tensor.SetFloat1D(sum64.Tensor.Float1D(i), i) } return sum64, count64 } @@ -167,8 +167,8 @@ func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Index if c < 2 { continue } - var64.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)/(c-1)) - out.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)) + var64.Tensor.SetFloat1D(var64.Tensor.Float1D(i)/(c-1), i) + out.Tensor.SetFloat1D(var64.Tensor.Float1D(i), i) } return } @@ -187,8 +187,8 @@ func StdFuncOut64(in, out *tensor.Indexed) (std64, mean64, count64 *tensor.Index nsub := out.Tensor.Len() for i := range nsub { std := math.Sqrt(std64.Tensor.Float1D(i)) - std64.Tensor.SetFloat1D(i, std) - out.Tensor.SetFloat1D(i, std) + std64.Tensor.SetFloat1D(std, i) + out.Tensor.SetFloat1D(std, i) } return } @@ -209,9 +209,9 @@ func SemFunc(in, out *tensor.Indexed) { for i := range nsub { c := count64.Tensor.Float1D(i) if c < 2 { - out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i)), i) } else { - out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c)) + out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c), i) } } } @@ -226,8 +226,8 @@ func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.In if c == 0 { continue } - var64.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)/c) - out.Tensor.SetFloat1D(i, var64.Tensor.Float1D(i)) + var64.Tensor.SetFloat1D(var64.Tensor.Float1D(i)/c, i) + out.Tensor.SetFloat1D(var64.Tensor.Float1D(i), i) } return } @@ -246,7 +246,7 @@ func StdPopFunc(in, out *tensor.Indexed) { var64, _, _ := VarPopFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i)), i) } } @@ -259,9 +259,9 @@ func SemPopFunc(in, out *tensor.Indexed) { for i := range nsub { c := count64.Tensor.Float1D(i) if c < 2 { - out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i)), i) } else { - out.Tensor.SetFloat1D(i, math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c)) + out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c), i) } } } @@ -302,8 +302,8 @@ func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { } else { v = scale * scale * ss } - scale64.Tensor.SetFloat1D(i, v) - out.Tensor.SetFloat1D(i, v) + scale64.Tensor.SetFloat1D(v, i) + out.Tensor.SetFloat1D(v, i) } return scale64 } @@ -329,8 +329,8 @@ func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { } else { v = scale * math.Sqrt(ss) } - scale64.Tensor.SetFloat1D(i, v) - out.Tensor.SetFloat1D(i, v) + scale64.Tensor.SetFloat1D(v, i) + out.Tensor.SetFloat1D(v, i) } return scale64 } diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index df4e87467b..70f0f67dd1 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -49,7 +49,7 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { hiv := sin.FloatRowCell(lwii+1, 0) val = (1-phi)*lwv + phi*hiv } - out.Tensor.SetFloat1D(i, val) + out.Tensor.SetFloat1D(val, i) } return nil } diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index f02b3e395b..a4170ee739 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -15,7 +15,7 @@ import ( func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) - out := tensor.NewFloat64([]int{1}) + out := tensor.NewFloat64(1) oix := tensor.NewIndexed(out) results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} @@ -92,7 +92,7 @@ func TestFuncsInt(t *testing.T) { vals := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} tsr := tensor.NewNumberFromSlice(vals) ix := tensor.NewIndexed(tsr) - out := tensor.NewInt([]int{1}) + out := tensor.NewInt(1) oix := tensor.NewIndexed(out) results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} @@ -156,16 +156,16 @@ func TestFuncsInt(t *testing.T) { func TestFuncsCell(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9} - tsr := tensor.NewFloat32([]int{20, 10}) + tsr := tensor.NewFloat32(20, 10) for i := range 20 { for j := range 10 { - tsr.SetFloatRowCell(i, j, vals[j]) + tsr.SetFloatRowCell(vals[j], i, j) } } ix := tensor.NewIndexed(tsr) - out := tensor.NewFloat32([]int{20, 10}) + out := tensor.NewFloat32(20, 10) oix := tensor.NewIndexed(out) CountFunc(ix, oix) diff --git a/tensor/stats/stats/table.go b/tensor/stats/stats/table.go index 018fb75a9a..e7ee6fea57 100644 --- a/tensor/stats/stats/table.go +++ b/tensor/stats/stats/table.go @@ -58,7 +58,7 @@ func MeanTables(dts []*table.Table) *table.Table { ci := si + j cv := cl.Float1D(ci) cv += dc.Float1D(ci) - cl.SetFloat1D(ci, cv) + cl.SetFloat1D(cv, ci) } } } @@ -69,7 +69,7 @@ func MeanTables(dts []*table.Table) *table.Table { cv := cl.Float1D(ci) if rns[ri] > 0 { cv /= float64(rns[ri]) - cl.SetFloat1D(ci, cv) + cl.SetFloat1D(cv, ci) } } } diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index db54e80ebe..9fc932f072 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -26,7 +26,7 @@ func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, ts } nt := len(tsr) out := tsr[nt-1] - o64 := tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + o64 := tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes...)) etsr := slices.Clone(tsr) etsr[nt-1] = o64 for idx := range n { @@ -34,7 +34,7 @@ func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, ts } nsub := out.Tensor.Len() for i := range nsub { - out.Tensor.SetFloat1D(i, o64.Tensor.Float1D(i)) + out.Tensor.SetFloat1D(o64.Tensor.Float1D(i), i) } return o64 } @@ -49,8 +49,8 @@ func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } nt := len(tsr) out := tsr[nt-1] - out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes...)) + out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes...)) tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2) for idx := range n { @@ -79,7 +79,8 @@ func NFunc(tsr ...*tensor.Indexed) int { in, out := tsr[0], tsr[nt-1] n := in.Rows() osh := OutShape(in.Tensor.Shape()) - out.Tensor.SetShape(osh.Sizes, osh.Names...) + out.Tensor.SetShape(osh.Sizes...) + out.Tensor.SetNames(osh.Names...) out.Indexes = []int{0} return n } @@ -91,13 +92,13 @@ func VecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg fl nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(i, ini) + out.Tensor.SetFloat1D(ini, i) } val := in.FloatRowCell(idx, i) if math.IsNaN(val) { continue } - out.Tensor.SetFloat1D(i, fun(val, out.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(fun(val, out.Tensor.Float1D(i)), i) } } @@ -110,14 +111,14 @@ func Vec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(va nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(i, ini) + out.Tensor.SetFloat1D(ini, i) } - val1 := in1.FloatRowCell(idx, i) + val1 := in1.FloatRowCell(i, idx) if math.IsNaN(val1) { continue } val2 := in2.Tensor.Float1D(i) - out.Tensor.SetFloat1D(i, fun(val1, val2, out.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(fun(val1, val2, out.Tensor.Float1D(i)), i) } } @@ -129,8 +130,8 @@ func Vec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu nsub := out2.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(i, ini1) - out2.Tensor.SetFloat1D(i, ini2) + out1.Tensor.SetFloat1D(ini1, i) + out2.Tensor.SetFloat1D(ini2, i) } val := in.FloatRowCell(idx, i) if math.IsNaN(val) { @@ -138,7 +139,7 @@ func Vec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu } ag1, ag2 := out1.Tensor.Float1D(i), out2.Tensor.Float1D(i) ag1, ag2 = fun(val, ag1, ag2) - out1.Tensor.SetFloat1D(i, ag1) - out2.Tensor.SetFloat1D(i, ag2) + out1.Tensor.SetFloat1D(ag1, i) + out2.Tensor.SetFloat1D(ag2, i) } } diff --git a/tensor/string.go b/tensor/string.go index 0316ee75da..0f631992be 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -21,10 +21,10 @@ type String struct { } // NewString returns a new n-dimensional tensor of string values -// with the given sizes per dimension (shape), and optional dimension names. -func NewString(sizes []int, names ...string) *String { +// with the given sizes per dimension (shape). +func NewString(sizes ...int) *String { tsr := &String{} - tsr.SetShape(sizes, names...) + tsr.SetShape(sizes...) tsr.Values = make([]string, tsr.Len()) return tsr } @@ -41,12 +41,11 @@ func NewStringShape(shape *Shape) *String { // NewStringFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewStringFromSlice(vals []string) Tensor { +func NewStringFromSlice(vals ...string) Tensor { n := len(vals) - sizes := []int{n} tsr := &String{} tsr.Values = vals - tsr.SetShape(sizes) + tsr.SetShape(n) return tsr } @@ -68,30 +67,16 @@ func (tsr *String) IsString() bool { return true } -func (tsr *String) AddScalar(i []int, val float64) float64 { - j := tsr.shape.Offset(i) - fv := StringToFloat64(tsr.Values[j]) + val - tsr.Values[j] = Float64ToString(fv) - return fv -} - -func (tsr *String) MulScalar(i []int, val float64) float64 { - j := tsr.shape.Offset(i) - fv := StringToFloat64(tsr.Values[j]) * val - tsr.Values[j] = Float64ToString(fv) - return fv -} - -func (tsr *String) SetString(i []int, val string) { - j := tsr.shape.Offset(i) +func (tsr *String) SetString(val string, i ...int) { + j := tsr.shape.Offset(i...) tsr.Values[j] = val } -func (tsr String) SetString1D(off int, val string) { +func (tsr String) SetString1D(val string, off int) { tsr.Values[off] = val } -func (tsr *String) SetStringRowCell(row, cell int, val string) { +func (tsr *String) SetStringRowCell(val string, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = val } @@ -121,21 +106,19 @@ func (tsr *String) String() string { return b.String() } -func (tsr *String) Float(i []int) float64 { - j := tsr.shape.Offset(i) - return StringToFloat64(tsr.Values[j]) +func (tsr *String) Float(i ...int) float64 { + return StringToFloat64(tsr.Values[tsr.shape.Offset(i...)]) } -func (tsr *String) SetFloat(i []int, val float64) { - j := tsr.shape.Offset(i) - tsr.Values[j] = Float64ToString(val) +func (tsr *String) SetFloat(val float64, i ...int) { + tsr.Values[tsr.shape.Offset(i...)] = Float64ToString(val) } func (tsr *String) Float1D(off int) float64 { return StringToFloat64(tsr.Values[off]) } -func (tsr *String) SetFloat1D(off int, val float64) { +func (tsr *String) SetFloat1D(val float64, off int) { tsr.Values[off] = Float64ToString(val) } @@ -144,7 +127,7 @@ func (tsr *String) FloatRowCell(row, cell int) float64 { return StringToFloat64(tsr.Values[row*sz+cell]) } -func (tsr *String) SetFloatRowCell(row, cell int, val float64) { +func (tsr *String) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[row*sz+cell] = Float64ToString(val) } @@ -161,7 +144,7 @@ func (tsr *String) Floats(flt *[]float64) { } // SetFloats sets tensor values from a []float64 slice (copies values). -func (tsr *String) SetFloats(flt []float64) { +func (tsr *String) SetFloats(flt ...float64) { for i, v := range flt { tsr.Values[i] = Float64ToString(v) } @@ -175,12 +158,12 @@ func (tsr *String) At(i, j int) float64 { log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") return 0 } else if nd == 2 { - return tsr.Float([]int{i, j}) + return tsr.Float(i, j) } else { ix := make([]int, nd) ix[nd-2] = i ix[nd-1] = j - return tsr.Float(ix) + return tsr.Float(ix...) } } @@ -250,7 +233,9 @@ func (tsr *String) CopyFrom(frm Tensor) { // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). func (tsr *String) SetShapeFrom(frm Tensor) { - tsr.SetShape(frm.Shape().Sizes, frm.Shape().Names...) + sh := frm.Shape() + tsr.SetShape(sh.Sizes...) + tsr.SetNames(sh.Names...) } // CopyCellsFrom copies given range of values from other tensor into this tensor, @@ -276,8 +261,8 @@ func (tsr *String) CopyCellsFrom(frm Tensor, to, start, n int) { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. -func (tsr *String) SubSpace(offs []int) Tensor { - b := tsr.subSpaceImpl(offs) +func (tsr *String) SubSpace(offs ...int) Tensor { + b := tsr.subSpaceImpl(offs...) rt := &String{Base: *b} return rt } diff --git a/tensor/table/io.go b/tensor/table/io.go index d8470882f4..197745bdee 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -200,12 +200,12 @@ func (dt *Table) ReadCSVRow(rec []string, row int) { str := rec[ci] if !tsr.IsString() { if str == "" || str == "NaN" || str == "-NaN" || str == "Inf" || str == "-Inf" { - tsr.SetFloat1D(stoff+cc, nan) + tsr.SetFloat1D(nan, stoff+cc) } else { - tsr.SetString1D(stoff+cc, strings.TrimSpace(str)) + tsr.SetString1D(strings.TrimSpace(str), stoff+cc) } } else { - tsr.SetString1D(stoff+cc, strings.TrimSpace(str)) + tsr.SetString1D(strings.TrimSpace(str), stoff+cc) } ci++ if ci >= len(rec) { @@ -256,7 +256,7 @@ func ConfigFromTableHeaders(dt *Table, hdrs []string) error { hd = hd[:lbst] csh := ShapeFromString(dims) // new tensor starting - dt.AddTensorColumnOfType(typ, hd, csh, "Row") + dt.AddTensorColumnOfType(typ, hd, csh...) continue } dimst = strings.Index(hd, "[") @@ -502,7 +502,7 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { } rc++ } else { - csh := tensor.NewShape(tsr.Shape().Sizes[1:]) // cell shape + csh := tensor.NewShape(tsr.Shape().Sizes[1:]...) // cell shape tc := csh.Len() for ti := 0; ti < tc; ti++ { vl := "" @@ -535,7 +535,7 @@ func (dt *Table) TableHeaders() []string { if tsr.NumDims() == 1 { hdrs = append(hdrs, nm) } else { - csh := tensor.NewShape(tsr.Shape().Sizes[1:]) // cell shape + csh := tensor.NewShape(tsr.Shape().Sizes[1:]...) // cell shape tc := csh.Len() nd := csh.NumDims() fnm := nm + fmt.Sprintf("[%v:", nd) diff --git a/tensor/table/splits.go b/tensor/table/splits.go index 9f81c65941..6bc5f8d406 100644 --- a/tensor/table/splits.go +++ b/tensor/table/splits.go @@ -404,13 +404,13 @@ func (spl *Splits) AggsToTable(colName bool) *Table { if colName == AddAggName { an += ":" + ag.Name } - st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:], col.Shape().Names[1:]...) + st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:]...) } for si := range spl.Splits { cidx := 0 for ci := range spl.Levels { col := st.Columns[cidx] - col.SetString1D(si, spl.Values[si][ci]) + col.SetString1D(spl.Values[si][ci], si) cidx++ } for _, ag := range spl.Aggs { @@ -419,7 +419,7 @@ func (spl *Splits) AggsToTable(colName bool) *Table { sti := si * csz av := ag.Aggs[si] for j, a := range av { - col.SetFloat1D(sti+j, a) + col.SetFloat1D(a, sti+j) } cidx++ } @@ -453,7 +453,7 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { if colName == AddAggName { an += ":" + ag.Name } - st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:], col.Shape().Names[1:]...) + st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:]...) } var cpcol []string for _, cn := range dt.ColumnNames { @@ -467,7 +467,7 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { cidx := 0 for ci := range spl.Levels { col := st.Columns[cidx] - col.SetString1D(si, spl.Values[si][ci]) + col.SetString1D(spl.Values[si][ci], si) cidx++ } for _, ag := range spl.Aggs { @@ -476,7 +476,7 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { sti := si * csz av := ag.Aggs[si] for j, a := range av { - col.SetFloat1D(sti+j, a) + col.SetFloat1D(a, sti+j) } cidx++ } diff --git a/tensor/table/table.go b/tensor/table/table.go index 42c507b0ae..eb4e1c7cf3 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -133,7 +133,8 @@ func (dt *Table) UpdateColumnNameMap() error { // see AddColumnTensor for n-dimensional cells. func AddColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string) tensor.Tensor { rows := max(1, dt.Rows) - tsr := tensor.New[T]([]int{rows}, "Row") + tsr := tensor.New[T](rows) + tsr.SetNames("Row") dt.AddColumn(tsr, name) return tsr } @@ -143,7 +144,8 @@ func AddColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Tab // The cells of this column hold a single scalar value. func InsertColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string, idx int) tensor.Tensor { rows := max(1, dt.Rows) - tsr := tensor.New[T]([]int{rows}, "Row") + tsr := tensor.New[T](rows) + tsr.SetNames("Row") dt.InsertColumn(tsr, name, idx) return tsr } @@ -152,11 +154,11 @@ func InsertColumn[T string | bool | float32 | float64 | int | int32 | byte](dt * // (which must be unique), and dimensionality of each _cell_. // An outer-most Row dimension will be added to this dimensionality to create // the tensor column. -func AddTensorColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string, cellSizes []int, dimNames ...string) tensor.Tensor { +func AddTensorColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string, cellSizes ...int) tensor.Tensor { rows := max(1, dt.Rows) sz := append([]int{rows}, cellSizes...) - nms := append([]string{"Row"}, dimNames...) - tsr := tensor.New[T](sz, nms...) + tsr := tensor.New[T](sz...) + tsr.SetNames("Row") dt.AddColumn(tsr, name) return tsr } @@ -200,7 +202,8 @@ func (dt *Table) InsertColumn(tsr tensor.Tensor, name string, idx int) error { // Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. func (dt *Table) AddColumnOfType(typ reflect.Kind, name string) tensor.Tensor { rows := max(1, dt.Rows) - tsr := tensor.NewOfType(typ, []int{rows}, "Row") + tsr := tensor.NewOfType(typ, rows) + tsr.SetNames("Row") dt.AddColumn(tsr, name) return tsr } @@ -210,11 +213,11 @@ func (dt *Table) AddColumnOfType(typ reflect.Kind, name string) tensor.Tensor { // An outer-most Row dimension will be added to this dimensionality to create // the tensor column. // Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. -func (dt *Table) AddTensorColumnOfType(typ reflect.Kind, name string, cellSizes []int, dimNames ...string) tensor.Tensor { +func (dt *Table) AddTensorColumnOfType(typ reflect.Kind, name string, cellSizes ...int) tensor.Tensor { rows := max(1, dt.Rows) sz := append([]int{rows}, cellSizes...) - nms := append([]string{"Row"}, dimNames...) - tsr := tensor.NewOfType(typ, sz, nms...) + tsr := tensor.NewOfType(typ, sz...) + tsr.SetNames("Row") dt.AddColumn(tsr, name) return tsr } @@ -235,8 +238,8 @@ func (dt *Table) AddFloat64Column(name string) *tensor.Float64 { // and dimensionality of each _cell_. // An outer-most Row dimension will be added to this dimensionality to create // the tensor column. -func (dt *Table) AddFloat64TensorColumn(name string, cellSizes []int, dimNames ...string) *tensor.Float64 { - return AddTensorColumn[float64](dt, name, cellSizes, dimNames...).(*tensor.Float64) +func (dt *Table) AddFloat64TensorColumn(name string, cellSizes ...int) *tensor.Float64 { + return AddTensorColumn[float64](dt, name, cellSizes...).(*tensor.Float64) } // AddFloat32Column adds a new float32 column with given name. @@ -249,8 +252,8 @@ func (dt *Table) AddFloat32Column(name string) *tensor.Float32 { // and dimensionality of each _cell_. // An outer-most Row dimension will be added to this dimensionality to create // the tensor column. -func (dt *Table) AddFloat32TensorColumn(name string, cellSizes []int, dimNames ...string) *tensor.Float32 { - return AddTensorColumn[float32](dt, name, cellSizes, dimNames...).(*tensor.Float32) +func (dt *Table) AddFloat32TensorColumn(name string, cellSizes ...int) *tensor.Float32 { + return AddTensorColumn[float32](dt, name, cellSizes...).(*tensor.Float32) } // AddIntColumn adds a new int column with given name. @@ -263,8 +266,8 @@ func (dt *Table) AddIntColumn(name string) *tensor.Int { // and dimensionality of each _cell_. // An outer-most Row dimension will be added to this dimensionality to create // the tensor column. -func (dt *Table) AddIntTensorColumn(name string, cellSizes []int, dimNames ...string) *tensor.Int { - return AddTensorColumn[int](dt, name, cellSizes, dimNames...).(*tensor.Int) +func (dt *Table) AddIntTensorColumn(name string, cellSizes ...int) *tensor.Int { + return AddTensorColumn[int](dt, name, cellSizes...).(*tensor.Int) } // DeleteColumnName deletes column of given name. @@ -503,7 +506,7 @@ func (dt *Table) TensorIndex(column, row int) tensor.Tensor { if ct.NumDims() == 1 { return nil } - return ct.SubSpace([]int{row}) + return ct.SubSpace(row) } // Tensor returns the tensor SubSpace for given column (by name), row index @@ -521,7 +524,7 @@ func (dt *Table) Tensor(column string, row int) tensor.Tensor { if ct.NumDims() == 1 { return nil } - return ct.SubSpace([]int{row}) + return ct.SubSpace(row) } // TensorFloat1D returns the float value of a Tensor cell's cell at given @@ -561,7 +564,7 @@ func (dt *Table) SetFloatIndex(column, row int, val float64) error { if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetFloatIndex: Column %d is a tensor, must use SetTensorFloat1D", column) } - ct.SetFloat1D(row, val) + ct.SetFloat1D(val, row) return nil } @@ -578,7 +581,7 @@ func (dt *Table) SetFloat(column string, row int, val float64) error { if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetFloat: Column %s is a tensor, must use SetTensorFloat1D", column) } - ct.SetFloat1D(row, val) + ct.SetFloat1D(val, row) return nil } @@ -592,7 +595,7 @@ func (dt *Table) SetStringIndex(column, row int, val string) error { if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetStringIndex: Column %d is a tensor, must use SetTensorFloat1D", column) } - ct.SetString1D(row, val) + ct.SetString1D(val, row) return nil } @@ -609,7 +612,7 @@ func (dt *Table) SetString(column string, row int, val string) error { if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetString: Column %s is a tensor, must use SetTensorFloat1D", column) } - ct.SetString1D(row, val) + ct.SetString1D(val, row) return nil } @@ -625,11 +628,11 @@ func (dt *Table) SetTensorIndex(column, row int, val tensor.Tensor) error { sz := min(csz, val.Len()) if ct.IsString() { for j := 0; j < sz; j++ { - ct.SetString1D(st+j, val.String1D(j)) + ct.SetString1D(val.String1D(j), st+j) } } else { for j := 0; j < sz; j++ { - ct.SetFloat1D(st+j, val.Float1D(j)) + ct.SetFloat1D(val.Float1D(j), st+j) } } return nil @@ -664,7 +667,7 @@ func (dt *Table) SetTensorFloat1D(column string, row int, idx int, val float64) return fmt.Errorf("table.Table IsValidRow: index %d is out of valid range [0..%d]", idx, sz) } off := row*sz + idx - ct.SetFloat1D(off, val) + ct.SetFloat1D(val, off) return nil } @@ -686,10 +689,10 @@ func (dt *Table) CopyCell(column string, row int, cpt *Table, cpColNm string, cp _, sz := ct.RowCellSize() if sz == 1 { if ct.IsString() { - ct.SetString1D(row, cpct.String1D(cpRow)) + ct.SetString1D(cpct.String1D(cpRow), row) return nil } - ct.SetFloat1D(row, cpct.Float1D(cpRow)) + ct.SetFloat1D(cpct.Float1D(cpRow), row) return nil } _, cpsz := cpct.RowCellSize() @@ -698,11 +701,11 @@ func (dt *Table) CopyCell(column string, row int, cpt *Table, cpColNm string, cp msz := min(sz, cpsz) if ct.IsString() { for j := 0; j < msz; j++ { - ct.SetString1D(st+j, cpct.String1D(cst+j)) + ct.SetString1D(cpct.String1D(cst+j), st+j) } } else { for j := 0; j < msz; j++ { - ct.SetFloat1D(st+j, cpct.Float1D(cst+j)) + ct.SetFloat1D(cpct.Float1D(cst+j), st+j) } } return nil diff --git a/tensor/table/util.go b/tensor/table/util.go index a631a168cd..1248521af9 100644 --- a/tensor/table/util.go +++ b/tensor/table/util.go @@ -27,7 +27,7 @@ func (dt *Table) InsertKeyColumns(args ...string) *Table { for j := range nc { colNm := args[2*j] val := args[2*j+1] - col := tensor.NewString([]int{c.Rows}) + col := tensor.NewString(c.Rows) c.InsertColumn(col, colNm, 0) for i := range col.Values { col.Values[i] = val diff --git a/tensor/tensor.go b/tensor/tensor.go index 7950be5fd9..010454bb94 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -36,8 +36,10 @@ type Tensor interface { // SetShape sets the sizes parameters of the tensor, and resizes // backing storage appropriately. - // Existing names will be preserved if not presented. - SetShape(sizes []int, names ...string) + SetShape(sizes ...int) + + // SetNames sets the dimension names of the tensor shape. + SetNames(names ...string) // Len returns the number of elements in the tensor, // which is the product of all shape dimensions. @@ -72,24 +74,24 @@ type Tensor interface { IsString() bool // Float returns the value of given index as a float64. - Float(i []int) float64 + Float(i ...int) float64 - // SetFloat sets the value of given index as a float64 - SetFloat(i []int, val float64) + // SetFloat sets the value of given index as a float64. + SetFloat(val float64, i ...int) // NOTE: String conflicts with [fmt.Stringer], so we have to use StringValue - // StringValue returns the value of given index as a string - StringValue(i []int) string + // StringValue returns the value of given index as a string. + StringValue(i ...int) string - // SetString sets the value of given index as a string - SetString(i []int, val string) + // SetString sets the value of given index as a string. + SetString(val string, i ...int) - // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64 + // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. Float1D(i int) float64 - // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64 - SetFloat1D(i int, val float64) + // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. + SetFloat1D(val float64, i int) // FloatRowCell returns the value at given row and cell, where row is outer-most dim, // and cell is 1D index into remaining inner dims. For Table columns. @@ -97,7 +99,7 @@ type Tensor interface { // SetFloatRowCell sets the value at given row and cell, where row is outer-most dim, // and cell is 1D index into remaining inner dims. For Table columns. - SetFloatRowCell(row, cell int, val float64) + SetFloatRowCell(val float64, row, cell int) // Floats sets []float64 slice of all elements in the tensor // (length is ensured to be sufficient). @@ -106,13 +108,13 @@ type Tensor interface { Floats(flt *[]float64) // SetFloats sets tensor values from a []float64 slice (copies values). - SetFloats(vals []float64) + SetFloats(vals ...float64) // String1D returns the value of given 1-dimensional index (0-Len()-1) as a string String1D(i int) string // SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string - SetString1D(i int, val string) + SetString1D(val string, i int) // StringRowCell returns the value at given row and cell, where row is outer-most dim, // and cell is 1D index into remaining inner dims. For Table columns. @@ -120,7 +122,7 @@ type Tensor interface { // SetStringRowCell sets the value at given row and cell, where row is outer-most dim, // and cell is 1D index into remaining inner dims. For Table columns. - SetStringRowCell(row, cell int, val string) + SetStringRowCell(val string, row, cell int) // SubSpace returns a new tensor with innermost subspace at given // offset(s) in outermost dimension(s) (len(offs) < NumDims). @@ -128,7 +130,7 @@ type Tensor interface { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. - SubSpace(offs []int) Tensor + SubSpace(offs ...int) Tensor // Range returns the min, max (and associated indexes, -1 = no values) for the tensor. // This is needed for display and is thus in the core api in optimized form @@ -174,48 +176,48 @@ type Tensor interface { } // New returns a new n-dimensional tensor of given value type -// with the given sizes per dimension (shape), and optional dimension names. -func New[T string | bool | float32 | float64 | int | int32 | byte](sizes []int, names ...string) Tensor { +// with the given sizes per dimension (shape). +func New[T string | bool | float32 | float64 | int | int32 | byte](sizes ...int) Tensor { var v T switch any(v).(type) { case string: - return NewString(sizes, names...) + return NewString(sizes...) case bool: - return NewBits(sizes, names...) + return NewBits(sizes...) case float64: - return NewNumber[float64](sizes, names...) + return NewNumber[float64](sizes...) case float32: - return NewNumber[float32](sizes, names...) + return NewNumber[float32](sizes...) case int: - return NewNumber[int](sizes, names...) + return NewNumber[int](sizes...) case int32: - return NewNumber[int32](sizes, names...) + return NewNumber[int32](sizes...) case byte: - return NewNumber[byte](sizes, names...) + return NewNumber[byte](sizes...) default: panic("tensor.New: unexpected error: type not supported") } } // NewOfType returns a new n-dimensional tensor of given reflect.Kind type -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). // Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. -func NewOfType(typ reflect.Kind, sizes []int, names ...string) Tensor { +func NewOfType(typ reflect.Kind, sizes ...int) Tensor { switch typ { case reflect.String: - return NewString(sizes, names...) + return NewString(sizes...) case reflect.Bool: - return NewBits(sizes, names...) + return NewBits(sizes...) case reflect.Float64: - return NewNumber[float64](sizes, names...) + return NewNumber[float64](sizes...) case reflect.Float32: - return NewNumber[float32](sizes, names...) + return NewNumber[float32](sizes...) case reflect.Int: - return NewNumber[int](sizes, names...) + return NewNumber[int](sizes...) case reflect.Int32: - return NewNumber[int32](sizes, names...) + return NewNumber[int32](sizes...) case reflect.Uint8: - return NewNumber[byte](sizes, names...) + return NewNumber[byte](sizes...) default: panic(fmt.Sprintf("tensor.NewOfType: type not supported: %v", typ)) } @@ -227,7 +229,7 @@ func NewOfType(typ reflect.Kind, sizes []int, names ...string) Tensor { // on the 1D list of values. func New1DViewOf(tsr Tensor) Tensor { vw := tsr.View() - vw.SetShape([]int{tsr.Len()}) + vw.SetShape(tsr.Len()) return vw } @@ -235,12 +237,12 @@ func New1DViewOf(tsr Tensor) Tensor { // using standard Float64 interface func CopyDense(to Tensor, dm *mat.Dense) { nr, nc := dm.Dims() - to.SetShape([]int{nr, nc}) + to.SetShape(nr, nc) idx := 0 for ri := 0; ri < nr; ri++ { for ci := 0; ci < nc; ci++ { v := dm.At(ri, ci) - to.SetFloat1D(idx, v) + to.SetFloat1D(v, idx) idx++ } } diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 1f0992514d..e3652a8518 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -13,21 +13,20 @@ import ( ) func TestTensorString(t *testing.T) { - shp := []int{4, 2} - nms := []string{"Row", "Vals"} - tsr := New[string](shp, nms...) + tsr := New[string](4, 2) + tsr.SetNames("Row", "Vals") assert.Equal(t, 8, tsr.Len()) assert.Equal(t, true, tsr.IsString()) assert.Equal(t, reflect.String, tsr.DataType()) - assert.Equal(t, 2, tsr.SubSpace([]int{0}).Len()) + assert.Equal(t, 2, tsr.SubSpace(0).Len()) r, c := tsr.RowCellSize() assert.Equal(t, 4, r) assert.Equal(t, 2, c) - tsr.SetString([]int{2, 0}, "test") - assert.Equal(t, "test", tsr.StringValue([]int{2, 0})) - tsr.SetString1D(5, "testing") - assert.Equal(t, "testing", tsr.StringValue([]int{2, 1})) + tsr.SetString("test", 2, 0) + assert.Equal(t, "test", tsr.StringValue(2, 0)) + tsr.SetString1D("testing", 5) + assert.Equal(t, "testing", tsr.StringValue(2, 1)) assert.Equal(t, "test", tsr.String1D(4)) assert.Equal(t, "test", tsr.StringRowCell(2, 0)) @@ -35,23 +34,24 @@ func TestTensorString(t *testing.T) { assert.Equal(t, "", tsr.StringRowCell(3, 0)) cln := tsr.Clone() - assert.Equal(t, "testing", cln.StringValue([]int{2, 1})) + assert.Equal(t, "testing", cln.StringValue(2, 1)) cln.SetZeros() - assert.Equal(t, "", cln.StringValue([]int{2, 1})) - assert.Equal(t, "testing", tsr.StringValue([]int{2, 1})) + assert.Equal(t, "", cln.StringValue(2, 1)) + assert.Equal(t, "testing", tsr.StringValue(2, 1)) - tsr.SetShape([]int{2, 4}, "Vals", "Row") - assert.Equal(t, "test", tsr.StringValue([]int{1, 0})) - assert.Equal(t, "testing", tsr.StringValue([]int{1, 1})) + tsr.SetShape(2, 4) + tsr.SetNames("Vals", "Row") + assert.Equal(t, "test", tsr.StringValue(1, 0)) + assert.Equal(t, "testing", tsr.StringValue(1, 1)) - cln.SetString1D(5, "ctesting") + cln.SetString1D("ctesting", 5) cln.SetShapeFrom(tsr) - assert.Equal(t, "ctesting", cln.StringValue([]int{1, 1})) + assert.Equal(t, "ctesting", cln.StringValue(1, 1)) cln.CopyCellsFrom(tsr, 5, 4, 2) - assert.Equal(t, "test", cln.StringValue([]int{1, 1})) - assert.Equal(t, "testing", cln.StringValue([]int{1, 2})) + assert.Equal(t, "test", cln.StringValue(1, 1)) + assert.Equal(t, "testing", cln.StringValue(1, 2)) tsr.SetNumRows(5) assert.Equal(t, 20, tsr.Len()) @@ -64,7 +64,7 @@ func TestTensorString(t *testing.T) { assert.Error(t, err) var flt []float64 - cln.SetString1D(0, "3.14") + cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) cln.Floats(&flt) @@ -73,21 +73,20 @@ func TestTensorString(t *testing.T) { } func TestTensorFloat64(t *testing.T) { - shp := []int{4, 2} - nms := []string{"Row", "Vals"} - tsr := New[float64](shp, nms...) + tsr := New[float64](4, 2) + tsr.SetNames("Row", "Vals") assert.Equal(t, 8, tsr.Len()) assert.Equal(t, false, tsr.IsString()) assert.Equal(t, reflect.Float64, tsr.DataType()) - assert.Equal(t, 2, tsr.SubSpace([]int{0}).Len()) + assert.Equal(t, 2, tsr.SubSpace(0).Len()) r, c := tsr.RowCellSize() assert.Equal(t, 4, r) assert.Equal(t, 2, c) - tsr.SetFloat([]int{2, 0}, 3.14) - assert.Equal(t, 3.14, tsr.Float([]int{2, 0})) - tsr.SetFloat1D(5, 2.17) - assert.Equal(t, 2.17, tsr.Float([]int{2, 1})) + tsr.SetFloat(3.14, 2, 0) + assert.Equal(t, 3.14, tsr.Float(2, 0)) + tsr.SetFloat1D(2.17, 5) + assert.Equal(t, 2.17, tsr.Float(2, 1)) assert.Equal(t, 3.14, tsr.Float1D(4)) assert.Equal(t, 3.14, tsr.FloatRowCell(2, 0)) @@ -95,29 +94,30 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 0.0, tsr.FloatRowCell(3, 0)) cln := tsr.Clone() - assert.Equal(t, 2.17, cln.Float([]int{2, 1})) + assert.Equal(t, 2.17, cln.Float(2, 1)) cln.SetZeros() - assert.Equal(t, 0.0, cln.Float([]int{2, 1})) - assert.Equal(t, 2.17, tsr.Float([]int{2, 1})) + assert.Equal(t, 0.0, cln.Float(2, 1)) + assert.Equal(t, 2.17, tsr.Float(2, 1)) - tsr.SetShape([]int{2, 4}, "Vals", "Row") - assert.Equal(t, 3.14, tsr.Float([]int{1, 0})) - assert.Equal(t, 2.17, tsr.Float([]int{1, 1})) + tsr.SetShape(2, 4) + tsr.SetNames("Vals", "Row") + assert.Equal(t, 3.14, tsr.Float(1, 0)) + assert.Equal(t, 2.17, tsr.Float(1, 1)) - cln.SetFloat1D(5, 9.9) + cln.SetFloat1D(9.9, 5) cln.SetShapeFrom(tsr) - assert.Equal(t, 9.9, cln.Float([]int{1, 1})) + assert.Equal(t, 9.9, cln.Float(1, 1)) cln.CopyCellsFrom(tsr, 5, 4, 2) - assert.Equal(t, 3.14, cln.Float([]int{1, 1})) - assert.Equal(t, 2.17, cln.Float([]int{1, 2})) + assert.Equal(t, 3.14, cln.Float(1, 1)) + assert.Equal(t, 2.17, cln.Float(1, 2)) tsr.SetNumRows(5) assert.Equal(t, 20, tsr.Len()) var flt []float64 - cln.SetString1D(0, "3.14") + cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) cln.Floats(&flt) From bb36e990266f9d0faa1390aa91bcf62041be61ca Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 12 Sep 2024 11:55:13 -0700 Subject: [PATCH 021/311] finished major update to use varargs for all index functions --- tensor/bits.go | 26 +++---- tensor/bitslice/bitslice.go | 16 ++--- tensor/bitslice/bitslice_test.go | 6 +- tensor/indexed.go | 16 +++++ tensor/shape.go | 8 ++- tensor/stats/metric/funcs.go | 32 ++++----- tensor/stats/metric/matrix.go | 20 +++--- tensor/stats/metric/metric_test.go | 6 +- tensor/stats/metric/metrics.go | 8 +-- tensor/stats/metric/misc.go | 2 +- tensor/stats/metric/vec.go | 52 +++++++------- tensor/stats/stats/funcs.go | 5 +- tensor/stats/stats/stats.go | 8 +-- tensor/stats/stats/stats_test.go | 40 +---------- tensor/stats/stats/vec.go | 14 ++-- tensor/tmath/math.go | 74 ++++++++++---------- tensor/tmath/{tmath_test.go => math_test.go} | 48 ++----------- tensor/tmath/norm.go | 8 +-- tensor/tmath/ops.go | 38 +++++----- tensor/tmath/ops_test.go | 13 ++-- tensor/tmath/tmath.go | 7 -- 21 files changed, 194 insertions(+), 253 deletions(-) rename tensor/tmath/{tmath_test.go => math_test.go} (55%) delete mode 100644 tensor/tmath/tmath.go diff --git a/tensor/bits.go b/tensor/bits.go index 6590be71b0..6714109520 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -109,10 +109,10 @@ func (tsr *Bits) Value(i ...int) bool { func (tsr *Bits) Value1D(i int) bool { return tsr.Values.Index(i) } func (tsr *Bits) Set(val bool, i ...int) { - tsr.Values.Set(tsr.shape.Offset(i...), val) + tsr.Values.Set(val, tsr.shape.Offset(i...)) } -func (tsr *Bits) Set1D(val bool, i int) { tsr.Values.Set(i, val) } +func (tsr *Bits) Set1D(val bool, i int) { tsr.Values.Set(val, i) } // SetShape sets the shape params, resizing backing storage appropriately func (tsr *Bits) SetShape(sizes ...int) { @@ -145,7 +145,7 @@ func (tsr *Bits) Float(i ...int) float64 { } func (tsr *Bits) SetFloat(val float64, i ...int) { - tsr.Values.Set(tsr.shape.Offset(i...), Float64ToBool(val)) + tsr.Values.Set(Float64ToBool(val), tsr.shape.Offset(i...)) } func (tsr *Bits) StringValue(i ...int) string { @@ -154,7 +154,7 @@ func (tsr *Bits) StringValue(i ...int) string { func (tsr *Bits) SetString(val string, i ...int) { if bv, err := reflectx.ToBool(val); err == nil { - tsr.Values.Set(tsr.shape.Offset(i...), bv) + tsr.Values.Set(bv, tsr.shape.Offset(i...)) } } @@ -163,7 +163,7 @@ func (tsr *Bits) Float1D(off int) float64 { } func (tsr *Bits) SetFloat1D(val float64, off int) { - tsr.Values.Set(off, Float64ToBool(val)) + tsr.Values.Set(Float64ToBool(val), off) } func (tsr *Bits) FloatRowCell(row, cell int) float64 { @@ -173,7 +173,7 @@ func (tsr *Bits) FloatRowCell(row, cell int) float64 { func (tsr *Bits) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.RowCellSize() - tsr.Values.Set(row*sz+cell, Float64ToBool(val)) + tsr.Values.Set(Float64ToBool(val), row*sz+cell) } func (tsr *Bits) Floats(flt *[]float64) { @@ -188,7 +188,7 @@ func (tsr *Bits) Floats(flt *[]float64) { func (tsr *Bits) SetFloats(vals ...float64) { sz := min(tsr.Len(), len(vals)) for j := 0; j < sz; j++ { - tsr.Values.Set(j, Float64ToBool(vals[j])) + tsr.Values.Set(Float64ToBool(vals[j]), j) } } @@ -198,7 +198,7 @@ func (tsr *Bits) String1D(off int) string { func (tsr *Bits) SetString1D(val string, off int) { if bv, err := reflectx.ToBool(val); err == nil { - tsr.Values.Set(off, bv) + tsr.Values.Set(bv, off) } } @@ -210,7 +210,7 @@ func (tsr *Bits) StringRowCell(row, cell int) string { func (tsr *Bits) SetStringRowCell(val string, row, cell int) { if bv, err := reflectx.ToBool(val); err == nil { _, sz := tsr.RowCellSize() - tsr.Values.Set(row*sz+cell, bv) + tsr.Values.Set(bv, row*sz+cell) } } @@ -230,7 +230,7 @@ func (tsr *Bits) Range() (min, max float64, minIndex, maxIndex int) { func (tsr *Bits) SetZeros() { ln := tsr.Len() for j := 0; j < ln; j++ { - tsr.Values.Set(j, false) + tsr.Values.Set(false, j) } } @@ -261,7 +261,7 @@ func (tsr *Bits) CopyFrom(frm Tensor) { } sz := min(len(tsr.Values), frm.Len()) for i := 0; i < sz; i++ { - tsr.Values.Set(i, Float64ToBool(frm.Float1D(i))) + tsr.Values.Set(Float64ToBool(frm.Float1D(i)), i) } } @@ -281,12 +281,12 @@ func (tsr *Bits) SetShapeFrom(frm Tensor) { func (tsr *Bits) CopyCellsFrom(frm Tensor, to, start, n int) { if fsm, ok := frm.(*Bits); ok { for i := 0; i < n; i++ { - tsr.Values.Set(to+i, fsm.Values.Index(start+i)) + tsr.Values.Set(fsm.Values.Index(start+i), to+i) } return } for i := 0; i < n; i++ { - tsr.Values.Set(to+i, Float64ToBool(frm.Float1D(start+i))) + tsr.Values.Set(Float64ToBool(frm.Float1D(start+i)), to+i) } } diff --git a/tensor/bitslice/bitslice.go b/tensor/bitslice/bitslice.go index eaefa05d9b..69f885779a 100644 --- a/tensor/bitslice/bitslice.go +++ b/tensor/bitslice/bitslice.go @@ -76,7 +76,7 @@ func (bs *Slice) SetLen(ln int) { } // Set sets value of given bit index -- no extra range checking is performed -- will panic if out of range -func (bs *Slice) Set(idx int, val bool) { +func (bs *Slice) Set(val bool, idx int) { by, bi := BitIndex(idx) if val { (*bs)[by+1] |= 1 << bi @@ -95,7 +95,7 @@ func (bs *Slice) Index(idx int) bool { func (bs *Slice) Append(val bool) Slice { if len(*bs) == 0 { *bs = Make(1, 0) - bs.Set(0, val) + bs.Set(val, 0) return *bs } ln := bs.Len() @@ -108,7 +108,7 @@ func (bs *Slice) Append(val bool) Slice { } else { (*bs)[0] = 0 } - bs.Set(ln, val) + bs.Set(val, ln) return *bs } @@ -161,7 +161,7 @@ func (bs *Slice) SubSlice(start, end int) Slice { } ss := Make(nln, 0) for i := 0; i < nln; i++ { - ss.Set(i, bs.Index(i+start)) + ss.Set(bs.Index(i+start), i) } return ss } @@ -186,10 +186,10 @@ func (bs *Slice) Delete(start, n int) Slice { } ss := Make(nln, 0) for i := 0; i < start; i++ { - ss.Set(i, bs.Index(i)) + ss.Set(bs.Index(i), i) } for i := end; i < ln; i++ { - ss.Set(i-n, bs.Index(i)) + ss.Set(bs.Index(i), i-n) } return ss } @@ -207,10 +207,10 @@ func (bs *Slice) Insert(start, n int) Slice { nln := ln + n ss := Make(nln, 0) for i := 0; i < start; i++ { - ss.Set(i, bs.Index(i)) + ss.Set(bs.Index(i), i) } for i := start; i < ln; i++ { - ss.Set(i+n, bs.Index(i)) + ss.Set(bs.Index(i), i+n) } return ss } diff --git a/tensor/bitslice/bitslice_test.go b/tensor/bitslice/bitslice_test.go index ceadc96cc6..322e66a5dd 100644 --- a/tensor/bitslice/bitslice_test.go +++ b/tensor/bitslice/bitslice_test.go @@ -24,7 +24,7 @@ func TestBitSlice10(t *testing.T) { t.Errorf("empty != %v", out) } - bs.Set(2, true) + bs.Set(true, 2) // fmt.Printf("2=true: %v\n", bs.String()) ex = "[0 0 1 0 0 0 0 0 0 0 ]" out = bs.String() @@ -32,7 +32,7 @@ func TestBitSlice10(t *testing.T) { t.Errorf("2=true != %v", out) } - bs.Set(4, true) + bs.Set(true, 4) // fmt.Printf("4=true: %v\n", bs.String()) ex = "[0 0 1 0 1 0 0 0 0 0 ]" out = bs.String() @@ -40,7 +40,7 @@ func TestBitSlice10(t *testing.T) { t.Errorf("4=true != %v", out) } - bs.Set(9, true) + bs.Set(true, 9) // fmt.Printf("9=true: %v\n", bs.String()) ex = "[0 0 1 0 1 0 0 0 0 1 ]" out = bs.String() diff --git a/tensor/indexed.go b/tensor/indexed.go index 403e69a6b2..3f61129385 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -499,3 +499,19 @@ func (ix *Indexed) StringRowCell(row, cell int) string { func (ix *Indexed) SetStringRowCell(val string, row, cell int) { ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) } + +// SubSpace returns a new tensor with innermost subspace at given +// offset(s) in outermost dimension(s) (len(offs) < NumDims). +// The new tensor points to the values of the this tensor (i.e., modifications +// will affect both), as its Values slice is a view onto the original (which +// is why only inner-most contiguous supsaces are supported). +// Use Clone() method to separate the two. +// Indexed version does indexed indirection of the outer-most row dimension +// of the offsets. +func (ix *Indexed) SubSpace(offs ...int) Tensor { + if len(offs) == 0 { + return nil + } + offs[0] = ix.Index(offs[0]) + return ix.Tensor.SubSpace(offs...) +} diff --git a/tensor/shape.go b/tensor/shape.go index e878d9b6b6..886e8e3776 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -177,9 +177,11 @@ func (sh *Shape) Index(offset int) []int { func (sh *Shape) String() string { str := "[" for i := range sh.Sizes { - nm := sh.Names[i] - if nm != "" { - str += nm + ": " + if sh.Names != nil { + nm := sh.Names[i] + if nm != "" { + str += nm + ": " + } } str += fmt.Sprintf("%d", sh.Sizes[i]) if i < len(sh.Sizes)-1 { diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 9a757769c2..532fb78d7b 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -55,8 +55,8 @@ func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { } else { v = scale * scale * ss } - scale64.Tensor.SetFloat1D(i, v) - out.Tensor.SetFloat1D(i, v) + scale64.Tensor.SetFloat1D(v, i) + out.Tensor.SetFloat1D(v, i) } return scale64 } @@ -81,8 +81,8 @@ func EuclideanFunc(a, b, out *tensor.Indexed) { } else { v = scale * math.Sqrt(ss) } - scale64.Tensor.SetFloat1D(i, v) - out.Tensor.SetFloat1D(i, v) + scale64.Tensor.SetFloat1D(v, i) + out.Tensor.SetFloat1D(v, i) } } @@ -142,8 +142,8 @@ func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { } else { v = scale * math.Sqrt(ss) } - scale64.Tensor.SetFloat1D(i, v) - out.Tensor.SetFloat1D(i, v) + scale64.Tensor.SetFloat1D(v, i) + out.Tensor.SetFloat1D(v, i) } } @@ -161,8 +161,8 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { } else { v = scale * scale * ss } - scale64.Tensor.SetFloat1D(i, v) - out.Tensor.SetFloat1D(i, v) + scale64.Tensor.SetFloat1D(v, i) + out.Tensor.SetFloat1D(v, i) } } @@ -216,8 +216,8 @@ func CovarianceFunc(a, b, out *tensor.Indexed) { if c == 0 { continue } - cov64.Tensor.SetFloat1D(i, cov64.Tensor.Float1D(i)/c) - out.Tensor.SetFloat1D(i, cov64.Tensor.Float1D(i)) + cov64.Tensor.SetFloat1D(cov64.Tensor.Float1D(i)/c, i) + out.Tensor.SetFloat1D(cov64.Tensor.Float1D(i), i) } } @@ -249,8 +249,8 @@ func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { if vp > 0 { ss /= vp } - ss64.Tensor.SetFloat1D(i, ss) - out.Tensor.SetFloat1D(i, ss) + ss64.Tensor.SetFloat1D(ss, i) + out.Tensor.SetFloat1D(ss, i) } return ss64 } @@ -278,7 +278,7 @@ func InvCorrelationFunc(a, b, out *tensor.Indexed) { nsub := out.Tensor.Len() for i := range nsub { cor := cor64.Tensor.Float1D(i) - out.Tensor.SetFloat1D(i, 1-cor) + out.Tensor.SetFloat1D(1-cor, i) } } @@ -301,8 +301,8 @@ func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { if vp > 0 { ss /= vp } - ss64.Tensor.SetFloat1D(i, ss) - out.Tensor.SetFloat1D(i, ss) + ss64.Tensor.SetFloat1D(ss, i) + out.Tensor.SetFloat1D(ss, i) } return ss64 } @@ -324,6 +324,6 @@ func InvCosineFunc(a, b, out *tensor.Indexed) { nsub := out.Tensor.Len() for i := range nsub { cos := cos64.Tensor.Float1D(i) - out.Tensor.SetFloat1D(i, 1-cos) + out.Tensor.SetFloat1D(1-cos, i) } } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index c081b7e493..6ba9033a6a 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -57,8 +57,7 @@ func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { if rows == 0 || cells == 0 { return } - msz := []int{rows, rows} - out.Tensor.SetShape(msz) + out.Tensor.SetShape(rows, rows) mout := tensor.NewFloatScalar(0.0) coords := TriangularLIndicies(rows) nc := len(coords) @@ -66,16 +65,16 @@ func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { tensor.VectorizeThreaded(cells*3, func(tsr ...*tensor.Indexed) int { return nc }, func(idx int, tsr ...*tensor.Indexed) { c := coords[idx] - sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].Tensor.SubSpace([]int{tsr[0].Index(c.X)}))) - sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].Tensor.SubSpace([]int{tsr[0].Index(c.Y)}))) + sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].SubSpace(c.X))) + sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].SubSpace(c.Y))) mfun(sa, sb, mout) - tsr[1].SetFloat([]int{c.X, c.Y}, mout.Tensor.Float1D(0)) + tsr[1].SetFloat(mout.Tensor.Float1D(0), c.X, c.Y) }, in, out) for _, c := range coords { // copy to upper if c.X == c.Y { // exclude diag continue } - out.Tensor.SetFloat([]int{c.Y, c.X}, out.Tensor.Float([]int{c.X, c.Y})) + out.Tensor.SetFloat(out.Tensor.Float(c.X, c.Y), c.Y, c.X) } } @@ -97,8 +96,7 @@ func CrossMatrix(a, b, out *tensor.Indexed, mfun MetricFunc) { if brows == 0 || bcells == 0 { return } - msz := []int{arows, brows} - out.Tensor.SetShape(msz) + out.Tensor.SetShape(arows, brows) mout := tensor.NewFloatScalar(0.0) // note: flops estimating 3 per item on average -- different for different metrics. flops := min(acells, bcells) * 3 @@ -107,9 +105,9 @@ func CrossMatrix(a, b, out *tensor.Indexed, mfun MetricFunc) { func(idx int, tsr ...*tensor.Indexed) { ar := idx / brows br := idx % brows - sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].Tensor.SubSpace([]int{tsr[0].Index(ar)}))) - sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[1].Tensor.SubSpace([]int{tsr[1].Index(br)}))) + sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].SubSpace(ar))) + sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[1].SubSpace(br))) mfun(sa, sb, mout) - tsr[2].SetFloat([]int{ar, br}, mout.Tensor.Float1D(0)) + tsr[2].SetFloat(mout.Tensor.Float1D(0), ar, br) }, a, b, out) } diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 8b5fcde656..9fcd23c729 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -24,7 +24,7 @@ func TestFuncs(t *testing.T) { atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64)) btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64)) - out := tensor.NewFloat64([]int{1}) + out := tensor.NewFloat64(1) oix := tensor.NewIndexed(out) EuclideanFunc(atsr, btsr, oix) @@ -67,7 +67,7 @@ func TestFuncs(t *testing.T) { assert.InDelta(t, results[InvCosine], out.Values[0], tol) for met := Euclidean; met < MetricsN; met++ { - Standard(met, atsr, btsr, oix) + Metric(met, atsr, btsr, oix) assert.InDelta(t, results[met], out.Values[0], tol) } } @@ -92,7 +92,7 @@ func TestMatrix(t *testing.T) { assert.NoError(t, err) // smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) - out := tensor.NewIndexed(tensor.NewFloat64(nil)) + out := tensor.NewIndexed(tensor.NewFloat64()) Matrix(in, out, EuclideanFunc) // fmt.Println(out.Tensor) assert.Equal(t, simres, out.Tensor.String()) diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 09b94d9fdd..0b22754928 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -27,15 +27,15 @@ func init() { tensor.AddFunc(Cosine.String(), CosineFunc, 1) } -// Standard calls a standard Metrics enum function on given tensors. +// Metric calls a standard Metrics enum function on given tensors. // Output results are in the out tensor. -func Standard(metric Metrics, a, b, out *tensor.Indexed) { +func Metric(metric Metrics, a, b, out *tensor.Indexed) { tensor.Call(metric.String(), a, b, out) } -// StandardOut calls a standard Metrics enum function on given tensors, +// MetricOut calls a standard Metrics enum function on given tensors, // returning output as a newly created tensor. -func StandardOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { +func MetricOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { return errors.Log1(tensor.CallOut(metric.String(), a, b))[0] // note: error should never happen } diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 5c631613d4..05ad6edb71 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -23,7 +23,7 @@ func ClosestRow(probe, vocab *tensor.Indexed, mfun MetricFunc) (int, float64) { mind := math.MaxFloat64 prview := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) for ri := range rows { - sub := tensor.NewIndexed(tensor.New1DViewOf(vocab.Tensor.SubSpace([]int{vocab.Index(ri)}))) + sub := tensor.NewIndexed(tensor.New1DViewOf(vocab.SubSpace(ri))) mfun(prview, sub, out) d := out.Tensor.Float1D(0) if d < mind { diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 287d495285..e280f28b4e 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -21,9 +21,10 @@ func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } nt := len(tsr) out := tsr[nt-1] - out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) - out3 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes)) + osz := out.Tensor.Shape().Sizes + out1 = tensor.NewIndexed(tensor.NewFloat64(osz...)) + out2 = tensor.NewIndexed(tensor.NewFloat64(osz...)) + out3 = tensor.NewIndexed(tensor.NewFloat64(osz...)) tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2, out3) for idx := range n { @@ -52,8 +53,9 @@ func NFunc(tsr ...*tensor.Indexed) int { a, b, out := tsr[0], tsr[1], tsr[nt-1] na, nb := a.Rows(), b.Rows() osh := OutShape(a.Tensor.Shape()) - out.Tensor.SetShape(osh.Sizes, osh.Names...) - out.Indexes = []int{0} + out.Tensor.SetShape(osh.Sizes...) + out.Tensor.SetNames(osh.Names...) + out.Indexes = nil return min(na, nb) } @@ -64,7 +66,7 @@ func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(i, ini) + out.Tensor.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -74,7 +76,7 @@ func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg if math.IsNaN(bv) { continue } - out.Tensor.SetFloat1D(i, fun(av, bv, out.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(fun(av, bv, out.Tensor.Float1D(i)), i) } } @@ -86,8 +88,8 @@ func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu nsub := out2.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(i, ini1) - out2.Tensor.SetFloat1D(i, ini2) + out1.Tensor.SetFloat1D(ini1, i) + out2.Tensor.SetFloat1D(ini2, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -109,8 +111,8 @@ func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu } else { ss = ss + (absxi/scale)*(absxi/scale) } - out1.Tensor.SetFloat1D(i, scale) - out2.Tensor.SetFloat1D(i, ss) + out1.Tensor.SetFloat1D(scale, i) + out2.Tensor.SetFloat1D(ss, i) } } @@ -122,7 +124,7 @@ func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun fun nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(i, ini) + out.Tensor.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -134,7 +136,7 @@ func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun fun } av2 := a2.Tensor.Float1D(i) bv2 := b2.Tensor.Float1D(i) - out.Tensor.SetFloat1D(i, fun(av, bv, av2, bv2, out.Tensor.Float1D(i))) + out.Tensor.SetFloat1D(fun(av, bv, av2, bv2, out.Tensor.Float1D(i)), i) } } @@ -146,9 +148,9 @@ func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini nsub := out1.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(i, ini) - out2.Tensor.SetFloat1D(i, ini) - out3.Tensor.SetFloat1D(i, ini) + out1.Tensor.SetFloat1D(ini, i) + out2.Tensor.SetFloat1D(ini, i) + out3.Tensor.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -164,9 +166,9 @@ func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini o2 := out2.Tensor.Float1D(i) o3 := out3.Tensor.Float1D(i) o1, o2, o3 = fun(av, bv, av2, bv2, o1, o2, o3) - out1.Tensor.SetFloat1D(i, o1) - out2.Tensor.SetFloat1D(i, o2) - out3.Tensor.SetFloat1D(i, o3) + out1.Tensor.SetFloat1D(o1, i) + out2.Tensor.SetFloat1D(o2, i) + out3.Tensor.SetFloat1D(o3, i) } } @@ -177,9 +179,9 @@ func Vec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, f nsub := out1.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(i, ini) - out2.Tensor.SetFloat1D(i, ini) - out3.Tensor.SetFloat1D(i, ini) + out1.Tensor.SetFloat1D(ini, i) + out2.Tensor.SetFloat1D(ini, i) + out3.Tensor.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -193,8 +195,8 @@ func Vec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, f o2 := out2.Tensor.Float1D(i) o3 := out3.Tensor.Float1D(i) o1, o2, o3 = fun(av, bv, o1, o2, o3) - out1.Tensor.SetFloat1D(i, o1) - out2.Tensor.SetFloat1D(i, o2) - out3.Tensor.SetFloat1D(i, o3) + out1.Tensor.SetFloat1D(o1, i) + out2.Tensor.SetFloat1D(o2, i) + out3.Tensor.SetFloat1D(o3, i) } } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 00d979ef76..bc21fe2d1a 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -167,8 +167,9 @@ func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Index if c < 2 { continue } - var64.Tensor.SetFloat1D(var64.Tensor.Float1D(i)/(c-1), i) - out.Tensor.SetFloat1D(var64.Tensor.Float1D(i), i) + vr := var64.Tensor.Float1D(i) / (c - 1) + var64.Tensor.SetFloat1D(vr, i) + out.Tensor.SetFloat1D(vr, i) } return } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 2a86133190..a8333f8955 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -35,15 +35,15 @@ func init() { tensor.AddFunc(Q3.String(), Q3Func, 1) } -// Standard calls a standard Stats enum function on given tensors. +// Stat calls a standard Stats enum function on given tensors. // Output results are in the out tensor. -func Standard(stat Stats, in, out *tensor.Indexed) { +func Stat(stat Stats, in, out *tensor.Indexed) { tensor.Call(stat.String(), in, out) } -// StandardOut calls a standard Stats enum function on given tensor, +// StatOut calls a standard Stats enum function on given tensor, // returning output as a newly created tensor. -func StandardOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { +func StatOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { return errors.Log1(tensor.CallOut(stat.String(), in))[0] // note: error should never happen } diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index a4170ee739..33f2f2b6d1 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -83,7 +83,7 @@ func TestFuncs64(t *testing.T) { assert.InDelta(t, results[Q3], out.Values[0], tol) for stat := Count; stat < StatsN; stat++ { - Standard(stat, ix, oix) + Stat(stat, ix, oix) assert.InDelta(t, results[stat], out.Values[0], tol) } } @@ -149,7 +149,7 @@ func TestFuncsInt(t *testing.T) { assert.Equal(t, results[L2Norm], out.Values[0]) for stat := Count; stat <= SemPop; stat++ { - Standard(stat, ix, oix) + Stat(stat, ix, oix) assert.Equal(t, results[stat], out.Values[0]) } } @@ -182,39 +182,3 @@ func TestFuncsCell(t *testing.T) { assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) } } - -/* -func TestIndexed(t *testing.T) { - desc := DescAll(ix) - assert.Equal(t, len(DescStats), desc.Rows) - assert.Equal(t, 2, desc.NumColumns()) - - for ri, stat := range DescStats { - dv := desc.Float("data", ri) - // fmt.Println(ri, ag.String(), dv, results[ag]) - assert.Equal(t, results[stat], dv) - } - - desc, err := DescColumn(ix, "data") - if err != nil { - t.Error(err) - } - assert.Equal(t, len(DescStats), desc.Rows) - assert.Equal(t, 2, desc.NumColumns()) - for ri, stat := range DescStats { - dv := desc.Float("data", ri) - // fmt.Println(ri, ag.String(), dv, results[ag]) - assert.Equal(t, results[stat], dv) - } - - pcts := PctIfColumn(ix, "data", func(idx int, val float64) bool { - return val > 2 - }) - assert.Equal(t, []float64{60}, pcts) - - props := PropIfColumn(ix, "data", func(idx int, val float64) bool { - return val > 2 - }) - assert.Equal(t, []float64{0.6}, props) -} -*/ diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 9fc932f072..e8da3096aa 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -26,7 +26,8 @@ func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, ts } nt := len(tsr) out := tsr[nt-1] - o64 := tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes...)) + osz := out.Tensor.Shape().Sizes + o64 := tensor.NewIndexed(tensor.NewFloat64(osz...)) etsr := slices.Clone(tsr) etsr[nt-1] = o64 for idx := range n { @@ -49,8 +50,9 @@ func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } nt := len(tsr) out := tsr[nt-1] - out1 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes...)) - out2 = tensor.NewIndexed(tensor.NewFloat64(out.Tensor.Shape().Sizes...)) + osz := out.Tensor.Shape().Sizes + out1 = tensor.NewIndexed(tensor.NewFloat64(osz...)) + out2 = tensor.NewIndexed(tensor.NewFloat64(osz...)) tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2) for idx := range n { @@ -81,7 +83,7 @@ func NFunc(tsr ...*tensor.Indexed) int { osh := OutShape(in.Tensor.Shape()) out.Tensor.SetShape(osh.Sizes...) out.Tensor.SetNames(osh.Names...) - out.Indexes = []int{0} + out.Indexes = nil return n } @@ -113,11 +115,11 @@ func Vec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(va if idx == 0 { out.Tensor.SetFloat1D(ini, i) } - val1 := in1.FloatRowCell(i, idx) + val1 := in1.FloatRowCell(idx, i) if math.IsNaN(val1) { continue } - val2 := in2.Tensor.Float1D(i) + val2 := in2.Tensor.Float1D(i) // output = not nan out.Tensor.SetFloat1D(fun(val1, val2, out.Tensor.Float1D(i)), i) } } diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index 39008ec87b..7562ebac0a 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -54,7 +54,7 @@ func Abs(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Abs(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Abs(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -62,7 +62,7 @@ func Acos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Acos(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Acos(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -70,7 +70,7 @@ func Acosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Acosh(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Acosh(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -78,7 +78,7 @@ func Asin(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Asin(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Asin(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -86,7 +86,7 @@ func Asinh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Asinh(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Asinh(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -94,7 +94,7 @@ func Atan(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Atan(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Atan(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -102,7 +102,7 @@ func Atanh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Atanh(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Atanh(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -110,7 +110,7 @@ func Cbrt(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Cbrt(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Cbrt(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -118,7 +118,7 @@ func Ceil(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Ceil(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Ceil(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -126,7 +126,7 @@ func Cos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Cos(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Cos(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -134,7 +134,7 @@ func Cosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Cosh(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Cosh(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -142,7 +142,7 @@ func Erf(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Erf(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Erf(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -150,7 +150,7 @@ func Erfc(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Erfc(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Erfc(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -158,7 +158,7 @@ func Erfcinv(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Erfcinv(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Erfcinv(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -166,7 +166,7 @@ func Erfinv(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Erfinv(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Erfinv(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -174,7 +174,7 @@ func Exp(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Exp(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Exp(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -182,7 +182,7 @@ func Exp2(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Exp2(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Exp2(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -190,7 +190,7 @@ func Expm1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Expm1(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Expm1(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -198,7 +198,7 @@ func Floor(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Floor(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Floor(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -206,7 +206,7 @@ func Gamma(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Gamma(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Gamma(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -214,7 +214,7 @@ func J0(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.J0(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.J0(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -222,7 +222,7 @@ func J1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.J1(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.J1(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -230,7 +230,7 @@ func Log(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Log(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Log(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -238,7 +238,7 @@ func Log10(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Log10(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Log10(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -246,7 +246,7 @@ func Log1p(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Log1p(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Log1p(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -254,7 +254,7 @@ func Log2(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Log2(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Log2(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -262,7 +262,7 @@ func Logb(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Logb(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Logb(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -270,7 +270,7 @@ func Round(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Round(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Round(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -278,7 +278,7 @@ func RoundToEven(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.RoundToEven(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.RoundToEven(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -286,7 +286,7 @@ func Sin(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Sin(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Sin(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -294,7 +294,7 @@ func Sinh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Sinh(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Sinh(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -302,7 +302,7 @@ func Sqrt(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Sqrt(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Sqrt(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -310,7 +310,7 @@ func Tan(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Tan(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Tan(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -318,7 +318,7 @@ func Tanh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Tanh(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Tanh(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -326,7 +326,7 @@ func Trunc(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Trunc(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Trunc(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -334,7 +334,7 @@ func Y0(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Y0(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Y0(tsr[0].Tensor.Float1D(i)), i) }, in, out) } @@ -342,7 +342,7 @@ func Y1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math.Y1(tsr[0].Tensor.Float1D(i))) + tsr[1].Tensor.SetFloat1D(math.Y1(tsr[0].Tensor.Float1D(i)), i) }, in, out) } diff --git a/tensor/tmath/tmath_test.go b/tensor/tmath/math_test.go similarity index 55% rename from tensor/tmath/tmath_test.go rename to tensor/tmath/math_test.go index 691291174b..f239364f27 100644 --- a/tensor/tmath/tmath_test.go +++ b/tensor/tmath/math_test.go @@ -13,6 +13,7 @@ import ( ) type onef func(x float64) float64 +type tonef func(in, out *tensor.Indexed) // Equal does equal testing taking into account NaN func Equal(t *testing.T, trg, val float64) { @@ -26,8 +27,7 @@ func Equal(t *testing.T, trg, val float64) { } func TestMath(t *testing.T) { - scalar := tensor.NewIndexed(tensor.NewFloat64([]int{1})) - scalar.Tensor.SetFloat1D(0, -5.5) + scalar := tensor.NewFloatScalar(-5.5) scout := scalar.Clone() vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} @@ -35,16 +35,16 @@ func TestMath(t *testing.T) { oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) oneout := oned.Clone() - cell2d := tensor.NewIndexed(tensor.NewFloat32([]int{5, 2, 6})) + cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, ci := cell2d.RowCellIndex(idx) - cell2d.Tensor.SetFloat1D(i1d, oned.Tensor.Float1D(ci)) + i, _, ci := cell2d.RowCellIndex(idx) + cell2d.Tensor.SetFloat1D(oned.Tensor.Float1D(ci), i) }, cell2d) // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() mfuncs := []onef{math.Abs, math.Acos, math.Acosh, math.Asin, math.Asinh, math.Atan, math.Atanh, math.Cbrt, math.Ceil, math.Cos, math.Cosh, math.Erf, math.Erfc, math.Erfcinv, math.Erfinv, math.Exp, math.Exp2, math.Expm1, math.Floor, math.Gamma, math.J0, math.J1, math.Log, math.Log10, math.Log1p, math.Log2, math.Logb, math.Round, math.RoundToEven, math.Sin, math.Sinh, math.Sqrt, math.Tan, math.Tanh, math.Trunc, math.Y0, math.Y1} - tfuncs := []Func1in1out{Abs, Acos, Acosh, Asin, Asinh, Atan, Atanh, Cbrt, Ceil, Cos, Cosh, Erf, Erfc, Erfcinv, Erfinv, Exp, Exp2, Expm1, Floor, Gamma, J0, J1, Log, Log10, Log1p, Log2, Logb, Round, RoundToEven, Sin, Sinh, Sqrt, Tan, Tanh, Trunc, Y0, Y1} + tfuncs := []tonef{Abs, Acos, Acosh, Asin, Asinh, Atan, Atanh, Cbrt, Ceil, Cos, Cosh, Erf, Erfc, Erfcinv, Erfinv, Exp, Exp2, Expm1, Floor, Gamma, J0, J1, Log, Log10, Log1p, Log2, Logb, Round, RoundToEven, Sin, Sinh, Sqrt, Tan, Tanh, Trunc, Y0, Y1} for i, fun := range mfuncs { tf := tfuncs[i] @@ -67,39 +67,3 @@ func TestMath(t *testing.T) { } } } - -/* -func TestNorm64(t *testing.T) { - vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - - zn := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818} - nvals := make([]float64, len(vals)) - copy(nvals, vals) - ZScore64(nvals) - assert.Equal(t, zn, nvals) - - copy(nvals, vals) - Unit64(nvals) - assert.Equal(t, vals, nvals) - - tn := []float64{0.2, 0.2, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.8, 0.8} - copy(nvals, vals) - Thresh64(nvals, true, 0.8, true, 0.2) - assert.Equal(t, tn, nvals) - - bn := []float64{0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1} - copy(nvals, vals) - Binarize64(nvals, 0.5, 1.0, 0.0) - assert.Equal(t, bn, nvals) - - tsr := tensor.New[float64]([]int{11}).(*tensor.Float64) - copy(tsr.Values, vals) - TensorZScore(tsr, 0) - tolassert.EqualTolSlice(t, zn, tsr.Values, 1.0e-6) - - copy(tsr.Values, vals) - TensorUnit(tsr, 0) - tolassert.EqualTolSlice(t, vals, tsr.Values, 1.0e-6) - -} -*/ diff --git a/tensor/tmath/norm.go b/tensor/tmath/norm.go index 9cee1295d2..0f910aeaa3 100644 --- a/tensor/tmath/norm.go +++ b/tensor/tmath/norm.go @@ -13,7 +13,7 @@ import ( // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. func ZScore(a, out *tensor.Indexed) { - mout := tensor.NewIndexed(tensor.NewFloat64(nil)) + mout := tensor.NewIndexed(tensor.NewFloat64()) std, mean, _ := stats.StdFuncOut64(a, mout) Sub(a, mean, out) Div(out, std, out) @@ -22,7 +22,7 @@ func ZScore(a, out *tensor.Indexed) { // UnitNorm computes unit normalized values into given output tensor, // subtracting the Min value and dividing by the Max of the remaining numbers. func UnitNorm(a, out *tensor.Indexed) { - mout := tensor.NewIndexed(tensor.NewFloat64(nil)) + mout := tensor.NewIndexed(tensor.NewFloat64()) stats.MinFunc(a, mout) Sub(a, mout, out) stats.MaxFunc(out, mout) @@ -38,7 +38,7 @@ func Clamp(in, minv, maxv, out *tensor.Indexed) { mx := maxv.Tensor.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(i, math32.Clamp64(tsr[0].Tensor.Float1D(i), mn, mx)) + tsr[1].Tensor.SetFloat1D(math32.Clamp64(tsr[0].Tensor.Float1D(i), mn, mx), i) }, in, out) } @@ -56,6 +56,6 @@ func Binarize(in, threshold, out *tensor.Indexed) { } else { v = 0 } - tsr[1].Tensor.SetFloat1D(i, v) + tsr[1].Tensor.SetFloat1D(v, i) }, in, out) } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 641173192e..11b404fbbd 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -41,7 +41,7 @@ func Add(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)+tsr[1].Tensor.Float1D(ib)) + out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)+tsr[1].Tensor.Float1D(ib), ia) }, a, b, out) } @@ -50,7 +50,7 @@ func AddScalar(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, a.Tensor.Float1D(ia)+scalar) + out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)+scalar, ia) }, a, out) } @@ -59,9 +59,9 @@ func AddScalar(scalar float64, a, out *tensor.Indexed) { func AddSubSpace(a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ai, _, ci := a.RowCellIndex(idx) + ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)+sub.Tensor.Float1D(si)) + out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)+sub.Tensor.Float1D(si), ia) }, a, sub, out) } @@ -101,7 +101,7 @@ func Sub(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)-tsr[1].Tensor.Float1D(ib)) + out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)-tsr[1].Tensor.Float1D(ib), ia) }, a, b, out) } @@ -111,7 +111,7 @@ func SubScalar(sign float64, scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, sign*(a.Tensor.Float1D(ia)-scalar)) + out.Tensor.SetFloat1D(sign*(a.Tensor.Float1D(ia)-scalar), ia) }, a, out) } @@ -121,9 +121,9 @@ func SubScalar(sign float64, scalar float64, a, out *tensor.Indexed) { func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ai, _, ci := a.RowCellIndex(idx) + ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(ai, sign*(a.Tensor.Float1D(ai)-sub.Tensor.Float1D(si))) + out.Tensor.SetFloat1D(sign*(a.Tensor.Float1D(ia)-sub.Tensor.Float1D(si)), ia) }, a, sub, out) } @@ -163,7 +163,7 @@ func Mul(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)*tsr[1].Tensor.Float1D(ib)) + out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)*tsr[1].Tensor.Float1D(ib), ia) }, a, b, out) } @@ -172,7 +172,7 @@ func MulScalar(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, a.Tensor.Float1D(ia)*scalar) + out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)*scalar, ia) }, a, out) } @@ -181,9 +181,9 @@ func MulScalar(scalar float64, a, out *tensor.Indexed) { func MulSubSpace(a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ai, _, ci := a.RowCellIndex(idx) + ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)*sub.Tensor.Float1D(si)) + out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)*sub.Tensor.Float1D(si), ia) }, a, sub, out) } @@ -223,7 +223,7 @@ func Div(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, tsr[0].Tensor.Float1D(ia)/tsr[1].Tensor.Float1D(ib)) + out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)/tsr[1].Tensor.Float1D(ib), ia) }, a, b, out) } @@ -232,7 +232,7 @@ func DivScalar(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, a.Tensor.Float1D(ia)/scalar) + out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)/scalar, ia) }, a, out) } @@ -242,7 +242,7 @@ func DivScalarInv(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(ia, scalar/a.Tensor.Float1D(ia)) + out.Tensor.SetFloat1D(scalar/a.Tensor.Float1D(ia), ia) }, a, out) } @@ -251,9 +251,9 @@ func DivScalarInv(scalar float64, a, out *tensor.Indexed) { func DivSubSpace(a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ai, _, ci := a.RowCellIndex(idx) + ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(ai, a.Tensor.Float1D(ai)/sub.Tensor.Float1D(si)) + out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)/sub.Tensor.Float1D(si), ia) }, a, sub, out) } @@ -262,8 +262,8 @@ func DivSubSpace(a, sub, out *tensor.Indexed) { func DivSubSpaceInv(a, sub, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ai, _, ci := a.RowCellIndex(idx) + ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(ai, sub.Tensor.Float1D(si)/a.Tensor.Float1D(ai)) + out.Tensor.SetFloat1D(sub.Tensor.Float1D(si)/a.Tensor.Float1D(ia), ia) }, a, sub, out) } diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 6bb024be98..d7e2e09ede 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -13,10 +13,9 @@ import ( ) func TestAdd(t *testing.T) { - scalar := tensor.NewIndexed(tensor.NewFloat64([]int{1})) - scalar.Tensor.SetFloat1D(0, -5.5) + scalar := tensor.NewFloatScalar(-5.5) scb := scalar.Clone() - scb.Tensor.SetFloat1D(0, -4.0) + scb.Tensor.SetFloat1D(-4.0, 0) scout := scalar.Clone() vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} @@ -24,10 +23,10 @@ func TestAdd(t *testing.T) { oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) oneout := oned.Clone() - cell2d := tensor.NewIndexed(tensor.NewFloat32([]int{5, 2, 6})) + cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i1d, _, ci := cell2d.RowCellIndex(idx) - cell2d.Tensor.SetFloat1D(i1d, oned.Tensor.Float1D(ci)) + i, _, ci := cell2d.RowCellIndex(idx) + cell2d.Tensor.SetFloat1D(oned.Tensor.Float1D(ci), i) }, cell2d) // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() @@ -129,7 +128,7 @@ func TestAdd(t *testing.T) { } ZScore(oned, oneout) - mout := tensor.NewIndexed(tensor.NewFloat64(nil)) + mout := tensor.NewIndexed(tensor.NewFloat64()) std, mean, _ := stats.StdFuncOut64(oneout, mout) assert.InDelta(t, 1.0, std.Tensor.Float1D(0), 1.0e-6) assert.InDelta(t, 0.0, mean.Tensor.Float1D(0), 1.0e-6) diff --git a/tensor/tmath/tmath.go b/tensor/tmath/tmath.go deleted file mode 100644 index 48896141f1..0000000000 --- a/tensor/tmath/tmath.go +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tmath - -//go:generate core generate From e50bfda1b2b67ab2c03e0e218247e6f618863baf Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 12 Sep 2024 12:51:04 -0700 Subject: [PATCH 022/311] update table readme with new indexed plan -- seems good.. --- tensor/README.md | 13 ++++++-- tensor/base.go | 4 +-- tensor/bits.go | 4 +-- tensor/indexed.go | 23 +++++--------- tensor/projection2d.go | 12 ++++---- tensor/shape.go | 4 +-- tensor/stats/metric/README.md | 2 +- tensor/stats/metric/funcs.go | 2 +- tensor/stats/metric/matrix.go | 4 +-- tensor/stats/metric/misc.go | 2 +- tensor/stats/stats/README.md | 4 +-- tensor/stats/stats/funcs.go | 2 +- tensor/table/README.md | 56 ++++++++++------------------------- tensor/table/table.go | 14 ++++----- tensor/table/typegen.go | 2 +- tensor/tensor.go | 14 ++++----- tensor/typegen.go | 2 +- tensor/vectorize.go | 2 +- 18 files changed, 70 insertions(+), 96 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index f35e14b262..7f6d5ac0bf 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -2,7 +2,7 @@ Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used `numpy` and `pandas` libraries in python, and the commercial MATLAB framework. -The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. The indexes provide a specific view onto the underlying [Tensor] data, applying to the outer-most _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. +The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. The indexes provide a specific view onto the underlying [Tensor] data, applying to the outermost _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. The `float64` ("Float") and `string` types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. @@ -24,10 +24,17 @@ All tensor package functions are registered using a single name to function map * [tmath](tmath) implements all standard math functions on `tensor.Indexed` data, including the standard `+, -, *, /` operators. `cosl` then calls these functions. -* [stats](stats) implements a number of different ways of analyzing tensor and table data - * [plot/plotcore](../plot/plotcore) supports interactive plotting of `Table` data. +* [bitslice](bitslice) is a Go slice of bytes `[]byte` that has methods for setting individual bits, as if it was a slice of bools, while being 8x more memory efficient. This is used for encoding null entries in `etensor`, and as a Tensor of bool / bits there as well, and is generally very useful for binary (boolean) data. + +* [stats](stats) implements a number of different ways of analyzing tensor and table data, including: + - [split](split) supports splitting a Table into any number of indexed sub-views and aggregating over those (i.e., pivot tables), grouping, summarizing data, etc. + - [metric](metric) provides similarity / distance metrics such as `Euclidean`, `Cosine`, or `Correlation` that operate on slices of `[]float64` or `[]float32`. + - TODO: now in metric: [simat](simat) provides similarity / distance matrix computation methods operating on `etensor.Tensor` or `etable.Table` data. The `SimMat` type holds the resulting matrix and labels for the rows and columns, which has a special `SimMatGrid` view in `etview` for visualizing labeled similarity matricies. + - TODO: where? [pca](pca) provides principal-components-analysis (PCA) and covariance matrix computation functions. + - TODO: in metric? [clust](clust) provides standard agglomerative hierarchical clustering including ability to plot results in an eplot. + # History diff --git a/tensor/base.go b/tensor/base.go index b4b1185294..022bfa781d 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -38,7 +38,7 @@ func (tsr *Base[T]) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension. func (tsr *Base[T]) DimSize(dim int) int { return tsr.shape.DimSize(dim) } -// RowCellSize returns the size of the outer-most Row shape dimension, +// RowCellSize returns the size of the outermost Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. func (tsr *Base[T]) RowCellSize() (rows, cells int) { @@ -94,7 +94,7 @@ func (tsr *Base[T]) SetNames(names ...string) { tsr.shape.SetNames(names...) } -// SetNumRows sets the number of rows (outer-most dimension) in a RowMajor organized tensor. +// SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. func (tsr *Base[T]) SetNumRows(rows int) { rows = max(1, rows) // must be > 0 _, cells := tsr.shape.RowCellSize() diff --git a/tensor/bits.go b/tensor/bits.go index 6714109520..da48f336b4 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -93,7 +93,7 @@ func (tsr *Bits) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension func (tsr *Bits) DimSize(dim int) int { return tsr.shape.DimSize(dim) } -// RowCellSize returns the size of the outer-most Row shape dimension, +// RowCellSize returns the size of the outermost Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. func (tsr *Bits) RowCellSize() (rows, cells int) { @@ -126,7 +126,7 @@ func (tsr *Bits) SetNames(names ...string) { tsr.shape.SetNames(names...) } -// SetNumRows sets the number of rows (outer-most dimension) in a RowMajor organized tensor. +// SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. func (tsr *Bits) SetNumRows(rows int) { rows = max(1, rows) // must be > 0 _, cells := tsr.shape.RowCellSize() diff --git a/tensor/indexed.go b/tensor/indexed.go index 3f61129385..7f3ce8d792 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -16,7 +16,7 @@ import ( // Indexed is an indexed wrapper around a tensor.Tensor that provides a // specific view onto the Tensor defined by the set of indexes, which -// apply to the outer-most row dimension (with default row-major indexing). +// apply to the outermost row dimension (with default row-major indexing). // This is the universal representation of a homogenous data type in the // [tensor] package framework, from scalar to vector, matrix, and beyond, // because it can efficiently represent any kind of element with sufficient @@ -87,7 +87,7 @@ func (ix *Indexed) Index(idx int) int { return ix.Indexes[idx] } -// RowCellSize returns the size of the outer-most Row shape dimension +// RowCellSize returns the size of the outermost Row shape dimension // (via [Indexed.Rows] method), and the size of all the remaining // inner dimensions (the "cell" size). func (ix *Indexed) RowCellSize() (rows, cells int) { @@ -142,14 +142,7 @@ func (ix *Indexed) DeleteInvalid() { // Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor. func (ix *Indexed) Sequential() { //types:add - if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { - ix.Indexes = nil - return - } - ix.Indexes = make([]int, ix.Tensor.DimSize(0)) - for i := range ix.Indexes { - ix.Indexes[i] = i - } + ix.Indexes = nil } // IndexesNeeded is called prior to an operation that needs actual indexes, @@ -447,7 +440,7 @@ func (ix *Indexed) SetFloat(val float64, i ...int) { } // FloatRowCell returns the value at given row and cell, -// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) FloatRowCell(row, cell int) float64 { @@ -455,7 +448,7 @@ func (ix *Indexed) FloatRowCell(row, cell int) float64 { } // SetFloatRowCell sets the value at given row and cell, -// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { @@ -485,7 +478,7 @@ func (ix *Indexed) SetString(val string, i ...int) { } // StringRowCell returns the value at given row and cell, -// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) StringRowCell(row, cell int) string { @@ -493,7 +486,7 @@ func (ix *Indexed) StringRowCell(row, cell int) string { } // SetStringRowCell sets the value at given row and cell, -// where row is outer-most dim, and cell is 1D index into remaining inner dims. +// where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetStringRowCell(val string, row, cell int) { @@ -506,7 +499,7 @@ func (ix *Indexed) SetStringRowCell(val string, row, cell int) { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. -// Indexed version does indexed indirection of the outer-most row dimension +// Indexed version does indexed indirection of the outermost row dimension // of the offsets. func (ix *Indexed) SubSpace(offs ...int) Tensor { if len(offs) == 0 { diff --git a/tensor/projection2d.go b/tensor/projection2d.go index 62a8e3be3d..d266e1ee34 100644 --- a/tensor/projection2d.go +++ b/tensor/projection2d.go @@ -16,7 +16,7 @@ const ( // Projection2DShape returns the size of a 2D projection of the given tensor Shape, // collapsing higher dimensions down to 2D (and 1D up to 2D). -// For any odd number of dimensions, the remaining outer-most dimension +// For any odd number of dimensions, the remaining outermost dimension // can either be multipliexed across the row or column, given the oddRow arg. // Even multiples of inner-most dimensions are assumed to be row, then column. // rowEx returns the number of "extra" (higher dimensional) rows @@ -55,7 +55,7 @@ func Projection2DShape(shp *Shape, oddRow bool) (rows, cols, rowEx, colEx int) { // Projection2DIndex returns the flat 1D index for given row, col coords for a 2D projection // of the given tensor shape, collapsing higher dimensions down to 2D (and 1D up to 2D). -// For any odd number of dimensions, the remaining outer-most dimension +// For any odd number of dimensions, the remaining outermost dimension // can either be multipliexed across the row or column, given the oddRow arg. // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { @@ -141,7 +141,7 @@ func Projection2DCoords(shp *Shape, oddRow bool, row, col int) (rowCoords, colCo // Projection2DValue returns the float64 value at given row, col coords for a 2D projection // of the given tensor, collapsing higher dimensions down to 2D (and 1D up to 2D). -// For any odd number of dimensions, the remaining outer-most dimension +// For any odd number of dimensions, the remaining outermost dimension // can either be multipliexed across the row or column, given the oddRow arg. // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DValue(tsr Tensor, oddRow bool, row, col int) float64 { @@ -151,7 +151,7 @@ func Projection2DValue(tsr Tensor, oddRow bool, row, col int) float64 { // Projection2DString returns the string value at given row, col coords for a 2D projection // of the given tensor, collapsing higher dimensions down to 2D (and 1D up to 2D). -// For any odd number of dimensions, the remaining outer-most dimension +// For any odd number of dimensions, the remaining outermost dimension // can either be multipliexed across the row or column, given the oddRow arg. // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DString(tsr Tensor, oddRow bool, row, col int) string { @@ -161,7 +161,7 @@ func Projection2DString(tsr Tensor, oddRow bool, row, col int) string { // Projection2DSet sets a float64 value at given row, col coords for a 2D projection // of the given tensor, collapsing higher dimensions down to 2D (and 1D up to 2D). -// For any odd number of dimensions, the remaining outer-most dimension +// For any odd number of dimensions, the remaining outermost dimension // can either be multipliexed across the row or column, given the oddRow arg. // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DSet(tsr Tensor, oddRow bool, row, col int, val float64) { @@ -171,7 +171,7 @@ func Projection2DSet(tsr Tensor, oddRow bool, row, col int, val float64) { // Projection2DSetString sets a string value at given row, col coords for a 2D projection // of the given tensor, collapsing higher dimensions down to 2D (and 1D up to 2D). -// For any odd number of dimensions, the remaining outer-most dimension +// For any odd number of dimensions, the remaining outermost dimension // can either be multipliexed across the row or column, given the oddRow arg. // Even multiples of inner-most dimensions are assumed to be row, then column. func Projection2DSetString(tsr Tensor, oddRow bool, row, col int, val string) { diff --git a/tensor/shape.go b/tensor/shape.go index 886e8e3776..6008a88d78 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -136,7 +136,7 @@ func (sh *Shape) IsEqual(oth *Shape) bool { return true } -// RowCellSize returns the size of the outer-most Row shape dimension, +// RowCellSize returns the size of the outermost Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. func (sh *Shape) RowCellSize() (rows, cells int) { @@ -192,7 +192,7 @@ func (sh *Shape) String() string { return str } -// RowMajorStrides returns strides for sizes where the first dimension is outer-most +// RowMajorStrides returns strides for sizes where the first dimension is outermost // and subsequent dimensions are progressively inner. func RowMajorStrides(sizes []int) []int { rem := int(1) diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 3f40867380..e2e409df13 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -27,7 +27,7 @@ The `Matrix` function returns a distance / similarity matrix computed from the n Here is general info about these functions: -`MetricFunc` is the function signature for a metric function, where the output has the same shape as the inputs but with the outer-most row dimension size of 1, and contains the metric value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. +`MetricFunc` is the function signature for a metric function, where the output has the same shape as the inputs but with the outermost row dimension size of 1, and contains the metric value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. Critically, the metric is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ metric for that cell across the different rows. To compute a metric on the `tensor.SubSpace` cells themselves, must call on a `tensor.New1DViewOf` the sub space. See [simat](../simat) package. diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 532fb78d7b..f8f5bddd18 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -14,7 +14,7 @@ import ( // MetricFunc is the function signature for a metric function, // where the output has the same shape as the inputs but with -// the outer-most row dimension size of 1, and contains +// the outermost row dimension size of 1, and contains // the metric value(s) for the "cells" in higher-dimensional tensors, // and a single scalar value for a 1D input tensor. // Critically, the metric is always computed over the outer row dimension, diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 6ba9033a6a..81190db7a3 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -45,7 +45,7 @@ func TriangularLIndicies(n int) []vecint.Vector2i { // Matrix computes the rows x rows distance / similarity matrix between // all sub-space cells of the given higher dimensional input tensor, -// which must have at least 2 dimensions: the outer-most rows, +// which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns that the given distance metric // function is applied to, with the results filling the elements of the output matrix. // The resulting matrix is symmetric, and only the lower triangular part @@ -81,7 +81,7 @@ func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { // CrossMatrix computes the distance / similarity matrix between // two different sets of patterns in the two input tensors, where // the patterns are in the sub-space cells of the tensors, -// which must have at least 2 dimensions: the outer-most rows, +// which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns that the given distance metric // function is applied to, with the results filling in the cells of the output matrix. // The rows of the output matrix are the rows of the first input tensor, diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 05ad6edb71..a62bc7349d 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -11,7 +11,7 @@ import ( ) // ClosestRow returns the closest fit between probe pattern and patterns in -// a "vocabulary" tensor with outer-most row dimension, using given metric +// a "vocabulary" tensor with outermost row dimension, using given metric // function, *which must have the Increasing property*, i.e., larger = further. // returns the row and metric value for that row. // note: this does _not_ use any existing Indexes for the probe (but does for the vocab). diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index 78dbd0f406..c216ec52c3 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -5,7 +5,7 @@ The `stats` package provides standard statistic computations operating on the `t type StatsFunc func(in, out *tensor.Indexed) ``` -For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outer-most row dimension set to 1. +For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outermost row dimension set to 1. There is a `StatsFuncs` map of named stats funcs, which is initialized with the standard Stats per below, and any additional user-defined functions can be added to. @@ -37,7 +37,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): Here is the general info associated with these function calls: -`StatsFunc` is the function signature for a stats function, where the output has the same shape as the input but with the outer-most row dimension size of 1, and contains the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. +`StatsFunc` is the function signature for a stats function, where the output has the same shape as the input but with the outermost row dimension size of 1, and contains the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. Critically, the stat is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ stat for that cell across the different rows. To compute a stat on the `tensor.SubSpace` cells themselves, must call on a [tensor.New1DViewOf] the sub space. diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index bc21fe2d1a..e4b58ae89a 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -12,7 +12,7 @@ import ( // StatsFunc is the function signature for a stats function, // where the output has the same shape as the input but with -// the outer-most row dimension size of 1, and contains +// the outermost row dimension size of 1, and contains // the stat value(s) for the "cells" in higher-dimensional tensors, // and a single scalar value for a 1D input tensor. // Critically, the stat is always computed over the outer row dimension, diff --git a/tensor/table/README.md b/tensor/table/README.md index 17f5d78e02..ef3a8e1d31 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -6,82 +6,56 @@ See [examples/dataproc](examples/dataproc) for a demo of how to use this system for data analysis, paralleling the example in [Python Data Science](https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html) using pandas, to see directly how that translates into this framework. -As a general convention, it is safest, clearest, and quite fast to access columns by name instead of index (there is a map that caches the column indexes), so the base access method names generally take a column name argument, and those that take a column index have an `Index` suffix. In addition, we use the `Try` suffix for versions that return an error message. It is a bit painful for the writer of these methods but very convenient for the users. +Whereas an individual `Tensor` can only hold one data type, the `table` allows coordinated storage and processing of heterogeneous data types, aligned by the outermost row dimension. While the main data processing functions are defined on the individual tensors (which are the universal computational element in the `tensor` system), the coordinated row-wise indexing in the table is important for sorting or filtering a collection of data in the same way, and grouping data by a common set of "splits" for data analysis. Plotting is also driven by the table, with one column providing a shared X axis for the rest of the columns. -The following packages are included: +As a general convention, it is safest, clearest, and quite fast to access columns by name instead of index (there is a map that caches the column indexes), so the base access method names generally take a column name argument, and those that take a column index have an `Index` suffix. -* [bitslice](bitslice) is a Go slice of bytes `[]byte` that has methods for setting individual bits, as if it was a slice of bools, while being 8x more memory efficient. This is used for encoding null entries in `etensor`, and as a Tensor of bool / bits there as well, and is generally very useful for binary (boolean) data. - -* [etensor](etensor) is a Tensor (n-dimensional array) object. `etensor.Tensor` is an interface that applies to many different type-specific instances, such as `etensor.Float32`. A tensor is just a `etensor.Shape` plus a slice holding the specific data type. Our tensor is based directly on the [Apache Arrow](https://github.com/apache/arrow/tree/master/go) project's tensor, and it fully interoperates with it. Arrow tensors are designed to be read-only, and we needed some extra support to make our `etable.Table` work well, so we had to roll our own. Our tensors also interoperate fully with Gonum's 2D-specific Matrix type for the 2D case. - -* [etable](etable) has the `etable.Table` DataTable / DataFrame object, which is useful for many different data analysis and database functions, and also for holding patterns to present to a neural network, and logs of output from the models, etc. A `etable.Table` is just a slice of `etensor.Tensor` columns, that are all aligned along the outer-most *row* dimension. Index-based indirection, which is essential for efficient Sort, Filter etc, is provided by the `etable.Indexed` type, which is an indexed view into a Table. All data processing operations are defined on the Indexed. - -* [eplot](eplot) provides an interactive 2D plotting GUI in [GoGi](https://cogentcore.org/core/gi) for Table data, using the [gonum plot](https://github.com/gonum/plot) plotting package. You can select which columns to plot and specify various basic plot parameters. - -* [tensorcore](tensorcore) provides an interactive tabular, spreadsheet-style GUI using [GoGi](https://cogentcore.org/core/gi) for viewing and editing `etable.Table` and `etable.Tensor` objects. The `tensorcore.TensorGrid` also provides a colored grid display higher-dimensional tensor data. - -* [agg](agg) provides standard aggregation functions (`Sum`, `Mean`, `Var`, `Std` etc) operating over `etable.Indexed` views of Table data. It also defines standard `AggFunc` functions such as `SumFunc` which can be used for `Agg` functions on either a Tensor or Indexed. - -* [tsragg](tsragg) provides the same agg functions as in `agg`, but operating on all the values in a given `Tensor`. Because of the indexed, row-based nature of tensors in a Table, these are not the same as the `agg` functions. - -* [split](split) supports splitting a Table into any number of indexed sub-views and aggregating over those (i.e., pivot tables), grouping, summarizing data, etc. - -* [metric](metric) provides similarity / distance metrics such as `Euclidean`, `Cosine`, or `Correlation` that operate on slices of `[]float64` or `[]float32`. - -* [simat](simat) provides similarity / distance matrix computation methods operating on `etensor.Tensor` or `etable.Table` data. The `SimMat` type holds the resulting matrix and labels for the rows and columns, which has a special `SimMatGrid` view in `etview` for visualizing labeled similarity matricies. - -* [pca](pca) provides principal-components-analysis (PCA) and covariance matrix computation functions. - -* [clust](clust) provides standard agglomerative hierarchical clustering including ability to plot results in an eplot. - -* [minmax](minmax) is home of basic Min / Max range struct, and `norm` has lots of good functions for computing standard norms and normalizing vectors. - -* [utils](utils) has various table-related utility command-line utility tools, including `etcat` which combines multiple table files into one file, including option for averaging column data. +The table itself stores raw data `tensor.Tensor` values, and the `Column` (by index) and `ColumnByName` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). It is best to use the table-wise `Sort` and `Filter` methods (and any others that affect the indexes) to ensure the indexes are properly coordinated. Resetting the column tensor indexes to `nil` (via the `Sequential` method) will break any connection to the table indexes, so that any subsequent index-altering operations on that indexed tensor will be fine. # Cheat Sheet -`et` is the etable pointer variable for examples below: +`dt` is the etable pointer variable for examples below: ## Table Access Scalar columns: ```Go -val := et.Float("ColName", row) +val := dt.Float("ColName", row) ``` ```Go -str := et.StringValue("ColName", row) +str := dt.StringValue("ColName", row) ``` Tensor (higher-dimensional) columns: ```Go -tsr := et.Tensor("ColName", row) // entire tensor at cell (a row-level SubSpace of column tensor) +tsr := dt.Tensor("ColName", row) // entire tensor at cell (a row-level SubSpace of column tensor) ``` ```Go -val := et.TensorFloat1D("ColName", row, cellidx) // idx is 1D index into cell tensor +val := dt.TensorFloat1D("ColName", row, cellidx) // idx is 1D index into cell tensor ``` ## Set Table Value ```Go -et.SetFloat("ColName", row, val) +dt.SetFloat("ColName", row, val) ``` ```Go -et.SetString("ColName", row, str) +dt.SetString("ColName", row, str) ``` Tensor (higher-dimensional) columns: ```Go -et.SetTensor("ColName", row, tsr) // set entire tensor at cell +dt.SetTensor("ColName", row, tsr) // set entire tensor at cell ``` ```Go -et.SetTensorFloat1D("ColName", row, cellidx, val) // idx is 1D index into cell tensor +dt.SetTensorFloat1D("ColName", row, cellidx, val) // idx is 1D index into cell tensor ``` ## Find Value(s) in Column @@ -89,7 +63,7 @@ et.SetTensorFloat1D("ColName", row, cellidx, val) // idx is 1D index into cell t Returns all rows where value matches given value, in string form (any number will convert to a string) ```Go -rows := et.RowsByString("ColName", "value", etable.Contains, etable.IgnoreCase) +rows := dt.RowsByString("ColName", "value", etable.Contains, etable.IgnoreCase) ``` Other options are `etable.Equals` instead of `Contains` to search for an exact full string, and `etable.UseCase` if case should be used instead of ignored. @@ -112,7 +86,7 @@ SortedTable := ix.NewTable() // turn an Indexed back into a new Table organized or: ```Go -nmcl := et.ColumnByName("Name") // nmcl is an etensor of the Name column, cached +nmcl := dt.ColumnByName("Name") // nmcl is an etensor of the Name column, cached ix.Sort(func(t *Table, i, j int) bool { return nmcl.StringValue1D(i) < nmcl.StringValue1D(j) }) @@ -121,7 +95,7 @@ ix.Sort(func(t *Table, i, j int) bool { ### Filter ```Go -nmcl := et.ColumnByName("Name") // column we're filtering on +nmcl := dt.ColumnByName("Name") // column we're filtering on ix.Filter(func(t *Table, row int) bool { // filter return value is for what to *keep* (=true), not exclude // here we keep any row with a name that contains the string "in" diff --git a/tensor/table/table.go b/tensor/table/table.go index eb4e1c7cf3..449619d079 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -19,7 +19,7 @@ import ( ) // Table is a table of data, with columns of tensors, -// each with the same number of Rows (outer-most dimension). +// each with the same number of Rows (outermost dimension). type Table struct { //types:add // columns of data, as tensor.Tensor tensors @@ -28,7 +28,7 @@ type Table struct { //types:add // the names of the columns ColumnNames []string - // number of rows, which is enforced to be the size of the outer-most dimension of the column tensors + // number of rows, which is enforced to be the size of the outermost dimension of the column tensors Rows int `edit:"-"` // the map of column names to column numbers @@ -152,7 +152,7 @@ func InsertColumn[T string | bool | float32 | float64 | int | int32 | byte](dt * // AddTensorColumn adds a new n-dimensional column to the table, of given type, column name // (which must be unique), and dimensionality of each _cell_. -// An outer-most Row dimension will be added to this dimensionality to create +// An outermost Row dimension will be added to this dimensionality to create // the tensor column. func AddTensorColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string, cellSizes ...int) tensor.Tensor { rows := max(1, dt.Rows) @@ -210,7 +210,7 @@ func (dt *Table) AddColumnOfType(typ reflect.Kind, name string) tensor.Tensor { // AddTensorColumnOfType adds a new n-dimensional column to the table, of given reflect type, // column name (which must be unique), and dimensionality of each _cell_. -// An outer-most Row dimension will be added to this dimensionality to create +// An outermost Row dimension will be added to this dimensionality to create // the tensor column. // Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. func (dt *Table) AddTensorColumnOfType(typ reflect.Kind, name string, cellSizes ...int) tensor.Tensor { @@ -236,7 +236,7 @@ func (dt *Table) AddFloat64Column(name string) *tensor.Float64 { // AddFloat64TensorColumn adds a new n-dimensional float64 column with given name // and dimensionality of each _cell_. -// An outer-most Row dimension will be added to this dimensionality to create +// An outermost Row dimension will be added to this dimensionality to create // the tensor column. func (dt *Table) AddFloat64TensorColumn(name string, cellSizes ...int) *tensor.Float64 { return AddTensorColumn[float64](dt, name, cellSizes...).(*tensor.Float64) @@ -250,7 +250,7 @@ func (dt *Table) AddFloat32Column(name string) *tensor.Float32 { // AddFloat32TensorColumn adds a new n-dimensional float32 column with given name // and dimensionality of each _cell_. -// An outer-most Row dimension will be added to this dimensionality to create +// An outermost Row dimension will be added to this dimensionality to create // the tensor column. func (dt *Table) AddFloat32TensorColumn(name string, cellSizes ...int) *tensor.Float32 { return AddTensorColumn[float32](dt, name, cellSizes...).(*tensor.Float32) @@ -264,7 +264,7 @@ func (dt *Table) AddIntColumn(name string) *tensor.Int { // AddIntTensorColumn adds a new n-dimensional int column with given name // and dimensionality of each _cell_. -// An outer-most Row dimension will be added to this dimensionality to create +// An outermost Row dimension will be added to this dimensionality to create // the tensor column. func (dt *Table) AddIntTensorColumn(name string, cellSizes ...int) *tensor.Int { return AddTensorColumn[int](dt, name, cellSizes...).(*tensor.Int) diff --git a/tensor/table/typegen.go b/tensor/table/typegen.go index 64c43d7067..3a020a134b 100644 --- a/tensor/table/typegen.go +++ b/tensor/table/typegen.go @@ -8,4 +8,4 @@ import ( var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Indexed", IDName: "index-view", Doc: "Indexed is an indexed wrapper around an table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}, {Name: "lessFunc", Doc: "current Less function used in sorting"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of data, with columns of tensors,\neach with the same number of Rows (outer-most dimension).", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to each of the columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns\nif rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "columns of data, as tensor.Tensor tensors"}, {Name: "ColumnNames", Doc: "the names of the columns"}, {Name: "Rows", Doc: "number of rows, which is enforced to be the size of the outer-most dimension of the column tensors"}, {Name: "ColumnNameMap", Doc: "the map of column names to column numbers"}, {Name: "MetaData", Doc: "misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of data, with columns of tensors,\neach with the same number of Rows (outermost dimension).", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to each of the columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns\nif rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "columns of data, as tensor.Tensor tensors"}, {Name: "ColumnNames", Doc: "the names of the columns"}, {Name: "Rows", Doc: "number of rows, which is enforced to be the size of the outermost dimension of the column tensors"}, {Name: "ColumnNameMap", Doc: "the map of column names to column numbers"}, {Name: "MetaData", Doc: "misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column"}}}) diff --git a/tensor/tensor.go b/tensor/tensor.go index 010454bb94..e711e66b97 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -16,7 +16,7 @@ import ( // todo: add a conversion function to copy data from Column-Major to a tensor: // It is also possible to use Column-Major order, which is used in R, Julia, and MATLAB -// where the inner-most index is first and outer-most last. +// where the inner-most index is first and outermost last. // Tensor is the interface for n-dimensional tensors. // Per C / Go / Python conventions, indexes are Row-Major, ordered from @@ -51,7 +51,7 @@ type Tensor interface { // DimSize returns size of given dimension DimSize(dim int) int - // RowCellSize returns the size of the outer-most Row shape dimension, + // RowCellSize returns the size of the outermost Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Commonly used to organize multiple instances (rows) of higher-dimensional // patterns (cells), and the [Indexed] type operates on the outer row dimension. @@ -93,11 +93,11 @@ type Tensor interface { // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. SetFloat1D(val float64, i int) - // FloatRowCell returns the value at given row and cell, where row is outer-most dim, + // FloatRowCell returns the value at given row and cell, where row is outermost dim, // and cell is 1D index into remaining inner dims. For Table columns. FloatRowCell(row, cell int) float64 - // SetFloatRowCell sets the value at given row and cell, where row is outer-most dim, + // SetFloatRowCell sets the value at given row and cell, where row is outermost dim, // and cell is 1D index into remaining inner dims. For Table columns. SetFloatRowCell(val float64, row, cell int) @@ -116,11 +116,11 @@ type Tensor interface { // SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string SetString1D(val string, i int) - // StringRowCell returns the value at given row and cell, where row is outer-most dim, + // StringRowCell returns the value at given row and cell, where row is outermost dim, // and cell is 1D index into remaining inner dims. For Table columns. StringRowCell(row, cell int) string - // SetStringRowCell sets the value at given row and cell, where row is outer-most dim, + // SetStringRowCell sets the value at given row and cell, where row is outermost dim, // and cell is 1D index into remaining inner dims. For Table columns. SetStringRowCell(val string, row, cell int) @@ -167,7 +167,7 @@ type Tensor interface { // of the same type, and otherwise it goes through appropriate standard type. CopyCellsFrom(from Tensor, to, start, n int) - // SetNumRows sets the number of rows (outer-most dimension). + // SetNumRows sets the number of rows (outermost dimension). SetNumRows(rows int) // Metadata returns the metadata for this tensor, which can be used diff --git a/tensor/typegen.go b/tensor/typegen.go index 2158c0bfde..9e76dee8dc 100644 --- a/tensor/typegen.go +++ b/tensor/typegen.go @@ -6,4 +6,4 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a tensor.Tensor that provides a\nspecific view onto the Tensor defined by the set of indexes, which\napply to the outer-most row dimension (with default row-major indexing).\nThis provides an efficient way of sorting and filtering a tensor by only\nupdating the indexes while doing nothing to the Tensor itself.\nTo produce a tensor that has data actually organized according to the\nindexed order, call the NewTensor method.\nUse the [Set]FloatRowCell methods wherever possible, for the most efficient\nand natural indirection through the indexes. The 1D methods on underlying\ntensor data do not indirect through the indexes and must be called directly\non the [Tensor].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a tensor.Tensor that provides a\nspecific view onto the Tensor defined by the set of indexes, which\napply to the outermost row dimension (with default row-major indexing).\nThis provides an efficient way of sorting and filtering a tensor by only\nupdating the indexes while doing nothing to the Tensor itself.\nTo produce a tensor that has data actually organized according to the\nindexed order, call the NewTensor method.\nUse the [Set]FloatRowCell methods wherever possible, for the most efficient\nand natural indirection through the indexes. The 1D methods on underlying\ntensor data do not indirect through the indexes and must be called directly\non the [Tensor].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows."}}}) diff --git a/tensor/vectorize.go b/tensor/vectorize.go index cb036cb2af..795b07cbc3 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -32,7 +32,7 @@ var ( // inputs and outputs, and the output could be effectively scalar, // as in a sum operation. The interpretation of the index is // function dependent as well, but often is used to iterate over -// the outer-most row dimension of the tensor. +// the outermost row dimension of the tensor. // This version runs purely sequentially on on this go routine. // See VectorizeThreaded and VectorizeGPU for other versions. func Vectorize(nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { From 9a139c2c0b7f8396168daba5ef43d00160d788b7 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 12 Sep 2024 16:08:28 -0700 Subject: [PATCH 023/311] cluster updated to new apis --- tensor/funcs.go | 103 ++++++++++-- tensor/indexed.go | 7 + tensor/stats/clust/README.md | 7 - tensor/stats/clust/clust.go | 159 ------------------ tensor/stats/clust/dist.go | 124 -------------- tensor/stats/clust/enumgen.go | 48 ------ tensor/stats/cluster/README.md | 14 ++ tensor/stats/{clust => cluster}/clust_test.go | 19 +-- tensor/stats/cluster/cluster.go | 158 +++++++++++++++++ tensor/stats/cluster/enumgen.go | 48 ++++++ tensor/stats/cluster/funcs.go | 135 +++++++++++++++ tensor/stats/{clust => cluster}/plot.go | 16 +- .../{clust => cluster}/testdata/faces.dat | 0 tensor/stats/metric/matrix.go | 101 ++++++----- tensor/stats/metric/metric_test.go | 4 +- tensor/stats/metric/misc.go | 31 ++-- tensor/stats/simat/README.md | 8 - tensor/stats/simat/doc.go | 16 -- tensor/stats/simat/simat.go | 150 ----------------- tensor/stats/simat/simat_test.go | 43 ----- tensor/stats/simat/tensor.go | 134 --------------- 21 files changed, 546 insertions(+), 779 deletions(-) delete mode 100644 tensor/stats/clust/README.md delete mode 100644 tensor/stats/clust/clust.go delete mode 100644 tensor/stats/clust/dist.go delete mode 100644 tensor/stats/clust/enumgen.go create mode 100644 tensor/stats/cluster/README.md rename tensor/stats/{clust => cluster}/clust_test.go (79%) create mode 100644 tensor/stats/cluster/cluster.go create mode 100644 tensor/stats/cluster/enumgen.go create mode 100644 tensor/stats/cluster/funcs.go rename tensor/stats/{clust => cluster}/plot.go (85%) rename tensor/stats/{clust => cluster}/testdata/faces.dat (100%) delete mode 100644 tensor/stats/simat/README.md delete mode 100644 tensor/stats/simat/doc.go delete mode 100644 tensor/stats/simat/simat.go delete mode 100644 tensor/stats/simat/simat_test.go delete mode 100644 tensor/stats/simat/tensor.go diff --git a/tensor/funcs.go b/tensor/funcs.go index 77652e00b7..79036c445e 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -12,16 +12,20 @@ import ( "cogentcore.org/core/base/errors" ) +// StringFirstArg should be used to set StringFirst functions +const StringFirstArg = true + // Func represents a registered tensor function, which has // In number of input *tensor.Indexed arguments, and Out -// number of output arguments. This quantification of the -// argument count is important for allowing the cosl script -// language to properly parse expressions involving these functions. +// number of output arguments. There can also be an optional +// string first argument, which is used to specify the name of +// another function in some cases (e.g., a stat or metric function). type Func struct { // Name is the original CamelCase Go name for function Name string - // Fun is the function, which must only take some number of *tensor.Indexed args + // Fun is the function, which must _only_ take some number of *tensor.Indexed + // args, with an optional first string arg per [StringFirst]. Fun any // In is number of input args @@ -29,19 +33,27 @@ type Func struct { // Out is number of output args Out int + + // StringFirst indicates if there is a string first argument, which can be + // used for many purposes including passing the name of another function to use + // in computation. + StringFirst bool } // NewFunc creates a new Func desciption of the given -// function, with specified number of output arguments. +// function, with specified number of output arguments, +// and an optional first string argument. // The remaining arguments in the function (automatically // determined) are classified as input arguments. -func NewFunc(name string, fun any, out int) (*Func, error) { - fn := &Func{Name: name, Fun: fun} +func NewFunc(name string, fun any, out int, stringFirst ...bool) (*Func, error) { + fn := &Func{Name: name, Fun: fun, Out: out} + if len(stringFirst) == 1 && stringFirst[0] { + fn.StringFirst = true + } nargs := fn.ArgCount() if out > nargs { return nil, fmt.Errorf("tensor.NewFunc: too many output args for function %q, which takes %d (-1 means function signature is not recognized)", name, nargs) } - fn.Out = out fn.In = 1 - out return fn, nil } @@ -56,9 +68,10 @@ var Funcs map[string]*Func // registry, which is used in cosl to call functions by name, and // in specific packages to call functions by enum String() names. // Use the standard Go CamelCase name -- will be auto-lowercased. -// The number of output arguments must be provided here; +// The number of output arguments must be provided here, +// along with an optional first string argument if present; // the number of input arguments is automatically set from that. -func AddFunc(name string, fun any, out int) error { +func AddFunc(name string, fun any, out int, stringFirst ...bool) error { if Funcs == nil { Funcs = make(map[string]*Func) } @@ -67,7 +80,7 @@ func AddFunc(name string, fun any, out int) error { if ok { return errors.Log(fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name)) } - fn, err := NewFunc(name, fun, out) + fn, err := NewFunc(name, fun, out, stringFirst...) if errors.Log(err) != nil { return err } @@ -90,6 +103,21 @@ func Call(name string, tsr ...*Indexed) error { return fn.Call(tsr...) } +// CallString calls function of given name, with given set of arguments +// (string, input and output) appropriate for the given function. +// An error is returned if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match. This version of [Call] is for functions that +// have an initial string argument +func CallString(name, first string, tsr ...*Indexed) error { + nm := strings.ToLower(name) + fn, ok := Funcs[nm] + if !ok { + return errors.Log(fmt.Errorf("tensor.Call: function of name %q not registered", name)) + } + return fn.CallString(first, tsr...) +} + // CallOut calls function of given name, with given set of _input_ // arguments appropriate for the given function, returning newly created // output tensors. @@ -105,7 +133,7 @@ func CallOut(name string, tsr ...*Indexed) ([]*Indexed, error) { return fn.CallOut(tsr...) } -// ArgCount returns the number of arguments the function takes, +// ArgCount returns the number of tensor arguments the function takes, // using a type switch. func (fn *Func) ArgCount() int { nargs := -1 @@ -120,6 +148,17 @@ func (fn *Func) ArgCount() int { nargs = 4 case func(a, b, c, d, e *Indexed): nargs = 5 + // string cases: + case func(s string, a *Indexed): + nargs = 1 + case func(s string, a, b *Indexed): + nargs = 2 + case func(s string, a, b, c *Indexed): + nargs = 3 + case func(s string, a, b, c, d *Indexed): + nargs = 4 + case func(s string, a, b, c, d, e *Indexed): + nargs = 5 } return nargs } @@ -136,6 +175,9 @@ func (fn *Func) ArgCheck(n int, tsr ...*Indexed) error { // Call calls function with given set of input & output arguments // appropriate for the given function (error if not). func (fn *Func) Call(tsr ...*Indexed) error { + if fn.StringFirst { + return fmt.Errorf("tensor.Call: function %q: requires a first string argument", fn.Name) + } switch f := fn.Fun.(type) { case func(a *Indexed): if err := fn.ArgCheck(1, tsr...); err != nil { @@ -166,6 +208,43 @@ func (fn *Func) Call(tsr ...*Indexed) error { return nil } +// CallString calls function with given set of input & output arguments +// appropriate for the given function (error if not), +// with an initial string argument. +func (fn *Func) CallString(s string, tsr ...*Indexed) error { + if !fn.StringFirst { + return fmt.Errorf("tensor.CallString: function %q: does not take a first string argument", fn.Name) + } + switch f := fn.Fun.(type) { + case func(s string, a *Indexed): + if err := fn.ArgCheck(1, tsr...); err != nil { + return err + } + f(s, tsr[0]) + case func(s string, a, b *Indexed): + if err := fn.ArgCheck(2, tsr...); err != nil { + return err + } + f(s, tsr[0], tsr[1]) + case func(s string, a, b, c *Indexed): + if err := fn.ArgCheck(3, tsr...); err != nil { + return err + } + f(s, tsr[0], tsr[1], tsr[2]) + case func(s string, a, b, c, d *Indexed): + if err := fn.ArgCheck(4, tsr...); err != nil { + return err + } + f(s, tsr[0], tsr[1], tsr[2], tsr[3]) + case func(s string, a, b, c, d, e *Indexed): + if err := fn.ArgCheck(5, tsr...); err != nil { + return err + } + f(s, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) + } + return nil +} + // CallOut calls function with given set of _input_ arguments // appropriate for the given function (error if not). // Newly-created output values are returned. diff --git a/tensor/indexed.go b/tensor/indexed.go index 7f3ce8d792..2a7d660991 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -508,3 +508,10 @@ func (ix *Indexed) SubSpace(offs ...int) Tensor { offs[0] = ix.Index(offs[0]) return ix.Tensor.SubSpace(offs...) } + +// Cells1D returns a flat 1D [tensor.Indexed] view of the cells for given row +// index (indirected through our Indexes). This is useful for passing to +// other functions e.g., in stats or metrics that process a 1D tensor. +func (ix *Indexed) Cells1D(row int) *Indexed { + return NewIndexed(New1DViewOf(ix.SubSpace(row))) +} diff --git a/tensor/stats/clust/README.md b/tensor/stats/clust/README.md deleted file mode 100644 index a11a82a5d9..0000000000 --- a/tensor/stats/clust/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# clust - -`clust` implements agglomerative clustering of items based on [simat](../simat) similarity matrix data. - -`GlomClust` is the main function, taking different `DistFunc` options for comparing distance between items. - - diff --git a/tensor/stats/clust/clust.go b/tensor/stats/clust/clust.go deleted file mode 100644 index 666990b250..0000000000 --- a/tensor/stats/clust/clust.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package clust - -//go:generate core generate - -import ( - "fmt" - "math" - "math/rand" - - "cogentcore.org/core/base/indent" - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/simat" - "cogentcore.org/core/tensor/stats/stats" -) - -// Node is one node in the cluster -type Node struct { - - // index into original distance matrix -- only valid for for terminal leaves - Index int - - // distance for this node -- how far apart were all the kids from each other when this node was created -- is 0 for leaf nodes - Dist float64 - - // total aggregate distance from parents -- the X axis offset at which our cluster starts - ParDist float64 - - // y-axis value for this node -- if a parent, it is the average of its kids Y's, otherwise it counts down - Y float64 - - // child nodes under this one - Kids []*Node -} - -// IsLeaf returns true if node is a leaf of the tree with no kids -func (nn *Node) IsLeaf() bool { - return len(nn.Kids) == 0 -} - -// Sprint prints to string -func (nn *Node) Sprint(smat *simat.SimMat, depth int) string { - if nn.IsLeaf() { - return smat.Rows[nn.Index] + " " - } - sv := fmt.Sprintf("\n%v%v: ", indent.Tabs(depth), nn.Dist) - for _, kn := range nn.Kids { - sv += kn.Sprint(smat, depth+1) - } - return sv -} - -// Indexes collects all the indexes in this node -func (nn *Node) Indexes(ix []int, ctr *int) { - if nn.IsLeaf() { - ix[*ctr] = nn.Index - (*ctr)++ - } else { - for _, kn := range nn.Kids { - kn.Indexes(ix, ctr) - } - } -} - -// NewNode merges two nodes into a new node -func NewNode(na, nb *Node, dst float64) *Node { - nn := &Node{Dist: dst} - nn.Kids = []*Node{na, nb} - return nn -} - -// Glom implements basic agglomerative clustering, based on a raw similarity matrix as given. -// This calls GlomInit to initialize the root node with all of the leaves, and the calls -// GlomClust to do the iterative clustering process. If you want to start with pre-defined -// initial clusters, then call GlomClust with a root node so-initialized. -// The smat.Mat matrix must be an tensor.Float64. -func Glom(smat *simat.SimMat, dfunc DistFunc) *Node { - ntot := smat.Mat.DimSize(0) // number of leaves - root := GlomInit(ntot) - return GlomClust(root, smat, dfunc) -} - -// GlomStd implements basic agglomerative clustering, based on a raw similarity matrix as given. -// This calls GlomInit to initialize the root node with all of the leaves, and the calls -// GlomClust to do the iterative clustering process. If you want to start with pre-defined -// initial clusters, then call GlomClust with a root node so-initialized. -// The smat.Mat matrix must be an tensor.Float64. -// Std version uses std distance functions -func GlomStd(smat *simat.SimMat, std StdDists) *Node { - return Glom(smat, StdFunc(std)) -} - -// GlomInit returns a standard root node initialized with all of the leaves -func GlomInit(ntot int) *Node { - root := &Node{} - root.Kids = make([]*Node, ntot) - for i := 0; i < ntot; i++ { - root.Kids[i] = &Node{Index: i} - } - return root -} - -// GlomClust does the iterative agglomerative clustering, based on a raw similarity matrix as given, -// using a root node that has already been initialized with the starting clusters (all of the -// leaves by default, but could be anything if you want to start with predefined clusters). -// The smat.Mat matrix must be an tensor.Float64. -func GlomClust(root *Node, smat *simat.SimMat, dfunc DistFunc) *Node { - ntot := smat.Mat.DimSize(0) // number of leaves - smatf := smat.Mat.(*tensor.Float64).Values - maxd := stats.Max64(smatf) - // indexes in each group - aidx := make([]int, ntot) - bidx := make([]int, ntot) - for { - var ma, mb []int - mval := math.MaxFloat64 - for ai, ka := range root.Kids { - actr := 0 - ka.Indexes(aidx, &actr) - aix := aidx[0:actr] - for bi := 0; bi < ai; bi++ { - kb := root.Kids[bi] - bctr := 0 - kb.Indexes(bidx, &bctr) - bix := bidx[0:bctr] - dv := dfunc(aix, bix, ntot, maxd, smatf) - if dv < mval { - mval = dv - ma = []int{ai} - mb = []int{bi} - } else if dv == mval { // do all ties at same time - ma = append(ma, ai) - mb = append(mb, bi) - } - } - } - ni := 0 - if len(ma) > 1 { - ni = rand.Intn(len(ma)) - } - na := ma[ni] - nb := mb[ni] - // fmt.Printf("merging nodes at dist: %v: %v and %v\nA: %v\nB: %v\n", mval, na, nb, root.Kids[na].Sprint(smat, 0), root.Kids[nb].Sprint(smat, 0)) - nn := NewNode(root.Kids[na], root.Kids[nb], mval) - for i := len(root.Kids) - 1; i >= 0; i-- { - if i == na || i == nb { - root.Kids = append(root.Kids[:i], root.Kids[i+1:]...) - } - } - root.Kids = append(root.Kids, nn) - if len(root.Kids) == 1 { - break - } - } - return root -} diff --git a/tensor/stats/clust/dist.go b/tensor/stats/clust/dist.go deleted file mode 100644 index a60acf62b9..0000000000 --- a/tensor/stats/clust/dist.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package clust - -import ( - "math" -) - -// DistFunc is a clustering distance function that evaluates aggregate distance -// between nodes, given the indexes of leaves in a and b clusters -// which are indexs into an ntot x ntot similarity (distance) matrix smat. -// maxd is the maximum distance value in the smat, which is needed by the -// ContrastDist function and perhaps others. -type DistFunc func(aix, bix []int, ntot int, maxd float64, smat []float64) float64 - -// MinDist is the minimum-distance or single-linkage weighting function for comparing -// two clusters a and b, given by their list of indexes. -// ntot is total number of nodes, and smat is the square similarity matrix [ntot x ntot]. -func MinDist(aix, bix []int, ntot int, maxd float64, smat []float64) float64 { - md := math.MaxFloat64 - for _, ai := range aix { - for _, bi := range bix { - d := smat[ai*ntot+bi] - if d < md { - md = d - } - } - } - return md -} - -// MaxDist is the maximum-distance or complete-linkage weighting function for comparing -// two clusters a and b, given by their list of indexes. -// ntot is total number of nodes, and smat is the square similarity matrix [ntot x ntot]. -func MaxDist(aix, bix []int, ntot int, maxd float64, smat []float64) float64 { - md := -math.MaxFloat64 - for _, ai := range aix { - for _, bi := range bix { - d := smat[ai*ntot+bi] - if d > md { - md = d - } - } - } - return md -} - -// AvgDist is the average-distance or average-linkage weighting function for comparing -// two clusters a and b, given by their list of indexes. -// ntot is total number of nodes, and smat is the square similarity matrix [ntot x ntot]. -func AvgDist(aix, bix []int, ntot int, maxd float64, smat []float64) float64 { - md := 0.0 - n := 0 - for _, ai := range aix { - for _, bi := range bix { - d := smat[ai*ntot+bi] - md += d - n++ - } - } - if n > 0 { - md /= float64(n) - } - return md -} - -// ContrastDist computes maxd + (average within distance - average between distance) -// for two clusters a and b, given by their list of indexes. -// avg between is average distance between all items in a & b versus all outside that. -// ntot is total number of nodes, and smat is the square similarity matrix [ntot x ntot]. -// maxd is the maximum distance and is needed to ensure distances are positive. -func ContrastDist(aix, bix []int, ntot int, maxd float64, smat []float64) float64 { - wd := AvgDist(aix, bix, ntot, maxd, smat) - nab := len(aix) + len(bix) - abix := append(aix, bix...) - abmap := make(map[int]struct{}, ntot-nab) - for _, ix := range abix { - abmap[ix] = struct{}{} - } - oix := make([]int, ntot-nab) - octr := 0 - for ix := 0; ix < ntot; ix++ { - if _, has := abmap[ix]; !has { - oix[octr] = ix - octr++ - } - } - bd := AvgDist(abix, oix, ntot, maxd, smat) - return maxd + (wd - bd) -} - -// StdDists are standard clustering distance functions -type StdDists int32 //enums:enum - -const ( - // Min is the minimum-distance or single-linkage weighting function - Min StdDists = iota - - // Max is the maximum-distance or complete-linkage weighting function - Max - - // Avg is the average-distance or average-linkage weighting function - Avg - - // Contrast computes maxd + (average within distance - average between distance) - Contrast -) - -// StdFunc returns a standard distance function as specified -func StdFunc(std StdDists) DistFunc { - switch std { - case Min: - return MinDist - case Max: - return MaxDist - case Avg: - return AvgDist - case Contrast: - return ContrastDist - } - return nil -} diff --git a/tensor/stats/clust/enumgen.go b/tensor/stats/clust/enumgen.go deleted file mode 100644 index 1cbbc383b5..0000000000 --- a/tensor/stats/clust/enumgen.go +++ /dev/null @@ -1,48 +0,0 @@ -// Code generated by "core generate"; DO NOT EDIT. - -package clust - -import ( - "cogentcore.org/core/enums" -) - -var _StdDistsValues = []StdDists{0, 1, 2, 3} - -// StdDistsN is the highest valid value for type StdDists, plus one. -const StdDistsN StdDists = 4 - -var _StdDistsValueMap = map[string]StdDists{`Min`: 0, `Max`: 1, `Avg`: 2, `Contrast`: 3} - -var _StdDistsDescMap = map[StdDists]string{0: `Min is the minimum-distance or single-linkage weighting function`, 1: `Max is the maximum-distance or complete-linkage weighting function`, 2: `Avg is the average-distance or average-linkage weighting function`, 3: `Contrast computes maxd + (average within distance - average between distance)`} - -var _StdDistsMap = map[StdDists]string{0: `Min`, 1: `Max`, 2: `Avg`, 3: `Contrast`} - -// String returns the string representation of this StdDists value. -func (i StdDists) String() string { return enums.String(i, _StdDistsMap) } - -// SetString sets the StdDists value from its string representation, -// and returns an error if the string is invalid. -func (i *StdDists) SetString(s string) error { - return enums.SetString(i, s, _StdDistsValueMap, "StdDists") -} - -// Int64 returns the StdDists value as an int64. -func (i StdDists) Int64() int64 { return int64(i) } - -// SetInt64 sets the StdDists value from an int64. -func (i *StdDists) SetInt64(in int64) { *i = StdDists(in) } - -// Desc returns the description of the StdDists value. -func (i StdDists) Desc() string { return enums.Desc(i, _StdDistsDescMap) } - -// StdDistsValues returns all possible values for the type StdDists. -func StdDistsValues() []StdDists { return _StdDistsValues } - -// Values returns all possible values for the type StdDists. -func (i StdDists) Values() []enums.Enum { return enums.Values(_StdDistsValues) } - -// MarshalText implements the [encoding.TextMarshaler] interface. -func (i StdDists) MarshalText() ([]byte, error) { return []byte(i.String()), nil } - -// UnmarshalText implements the [encoding.TextUnmarshaler] interface. -func (i *StdDists) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "StdDists") } diff --git a/tensor/stats/cluster/README.md b/tensor/stats/cluster/README.md new file mode 100644 index 0000000000..9057a68d40 --- /dev/null +++ b/tensor/stats/cluster/README.md @@ -0,0 +1,14 @@ +# cluster + +`cluster` implements agglomerative clustering of items based on [metric](../metric) distance `Matrix` data (which is provided as an input, and must have been generated with a distance-like metric (increasing with dissimiliarity). + +There are different standard ways of accumulating the aggregate distance of a node based on its leaves: + +* `Min`: the minimum-distance across leaves, i.e., the single-linkage weighting function. +* `Max`: the maximum-distance across leaves, i.e,. the complete-linkage weighting function. +* `Avg`: the average-distance across leaves, i.e., the average-linkage weighting function. +* `Contrast`: is Max + (average within distance - average between distance). + +`GlomCluster` is the main function, taking different `ClusterFunc` options for comparing distance between items. + + diff --git a/tensor/stats/clust/clust_test.go b/tensor/stats/cluster/clust_test.go similarity index 79% rename from tensor/stats/clust/clust_test.go rename to tensor/stats/cluster/clust_test.go index 139e603c7e..90f4b70138 100644 --- a/tensor/stats/clust/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -2,14 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package clust +package cluster import ( "testing" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/tolassert" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/stats/simat" "cogentcore.org/core/tensor/table" ) @@ -33,15 +34,11 @@ func TestClust(t *testing.T) { if err != nil { t.Error(err) } - ix := table.NewIndexed(dt) - smat := &simat.SimMat{} - smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) - - // fmt.Printf("%v\n", smat.Mat) - // cl := Glom(smat, MinDist) - cl := Glom(smat, AvgDist) - // s := cl.Sprint(smat, 0) - // fmt.Println(s) + in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) + out := tensor.NewIndexed(tensor.NewFloat64()) + metric.Matrix(metric.Euclidean.String(), in, out) + + cl := Cluster(Avg.String(), out, tensor.NewIndexed(errors.Log1(dt.ColumnByName("Name")))) var dists []float64 diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go new file mode 100644 index 0000000000..1ae426fda5 --- /dev/null +++ b/tensor/stats/cluster/cluster.go @@ -0,0 +1,158 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cluster + +//go:generate core generate + +import ( + "fmt" + "math" + "math/rand" + + "cogentcore.org/core/base/indent" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" +) + +// Node is one node in the cluster +type Node struct { + // index into original distance matrix; only valid for for terminal leaves. + Index int + + // Distance value for this node, i.e., how far apart were all the kids from + // each other when this node was created. is 0 for leaf nodes + Dist float64 + + // ParDist is total aggregate distance from parents; The X axis offset at which our cluster starts. + ParDist float64 + + // Y is y-axis value for this node; if a parent, it is the average of its kids Y's, + // otherwise it counts down. + Y float64 + + // Kids are child nodes under this one. + Kids []*Node +} + +// IsLeaf returns true if node is a leaf of the tree with no kids +func (nn *Node) IsLeaf() bool { + return len(nn.Kids) == 0 +} + +// Sprint prints to string +func (nn *Node) Sprint(labels *tensor.Indexed, depth int) string { + if nn.IsLeaf() && labels != nil { + return labels.Tensor.String1D(nn.Index) + " " + } + sv := fmt.Sprintf("\n%v%v: ", indent.Tabs(depth), nn.Dist) + for _, kn := range nn.Kids { + sv += kn.Sprint(labels, depth+1) + } + return sv +} + +// Indexes collects all the indexes in this node +func (nn *Node) Indexes(ix []int, ctr *int) { + if nn.IsLeaf() { + ix[*ctr] = nn.Index + (*ctr)++ + } else { + for _, kn := range nn.Kids { + kn.Indexes(ix, ctr) + } + } +} + +// NewNode merges two nodes into a new node +func NewNode(na, nb *Node, dst float64) *Node { + nn := &Node{Dist: dst} + nn.Kids = []*Node{na, nb} + return nn +} + +// TODO: this call signature does not fit with standard +// not sure how one might pack Node into a tensor + +// Cluster implements agglomerative clustering, based on a +// distance matrix dmat, e.g., as computed by metric.Matrix method, +// using a metric that increases in value with greater dissimilarity. +// labels provides an optional String tensor list of labels for the elements +// of the distance matrix. +// This calls InitAllLeaves to initialize the root node with all of the leaves, +// and then Glom to do the iterative agglomerative clustering process. +// If you want to start with pre-defined initial clusters, +// then call Glom with a root node so-initialized. +func Cluster(funcName string, dmat, labels *tensor.Indexed) *Node { + ntot := dmat.Tensor.DimSize(0) // number of leaves + root := InitAllLeaves(ntot) + return Glom(root, funcName, dmat) +} + +// InitAllLeaves returns a standard root node initialized with all of the leaves. +func InitAllLeaves(ntot int) *Node { + root := &Node{} + root.Kids = make([]*Node, ntot) + for i := 0; i < ntot; i++ { + root.Kids[i] = &Node{Index: i} + } + return root +} + +// Glom does the iterative agglomerative clustering, +// based on a raw similarity matrix as given, +// using a root node that has already been initialized +// with the starting clusters, which is all of the +// leaves by default, but could be anything if you want +// to start with predefined clusters. +func Glom(root *Node, funcName string, dmat *tensor.Indexed) *Node { + ntot := dmat.Tensor.DimSize(0) // number of leaves + mout := tensor.NewFloatScalar(0) + stats.MaxFunc(tensor.NewIndexed(tensor.New1DViewOf(dmat.Tensor)), mout) + maxd := mout.Tensor.Float1D(0) + // indexes in each group + aidx := make([]int, ntot) + bidx := make([]int, ntot) + for { + var ma, mb []int + mval := math.MaxFloat64 + for ai, ka := range root.Kids { + actr := 0 + ka.Indexes(aidx, &actr) + aix := aidx[0:actr] + for bi := 0; bi < ai; bi++ { + kb := root.Kids[bi] + bctr := 0 + kb.Indexes(bidx, &bctr) + bix := bidx[0:bctr] + dv := Call(funcName, aix, bix, ntot, maxd, dmat.Tensor) + if dv < mval { + mval = dv + ma = []int{ai} + mb = []int{bi} + } else if dv == mval { // do all ties at same time + ma = append(ma, ai) + mb = append(mb, bi) + } + } + } + ni := 0 + if len(ma) > 1 { + ni = rand.Intn(len(ma)) + } + na := ma[ni] + nb := mb[ni] + nn := NewNode(root.Kids[na], root.Kids[nb], mval) + for i := len(root.Kids) - 1; i >= 0; i-- { + if i == na || i == nb { + root.Kids = append(root.Kids[:i], root.Kids[i+1:]...) + } + } + root.Kids = append(root.Kids, nn) + if len(root.Kids) == 1 { + break + } + } + return root +} diff --git a/tensor/stats/cluster/enumgen.go b/tensor/stats/cluster/enumgen.go new file mode 100644 index 0000000000..d13044ae4e --- /dev/null +++ b/tensor/stats/cluster/enumgen.go @@ -0,0 +1,48 @@ +// Code generated by "core generate"; DO NOT EDIT. + +package cluster + +import ( + "cogentcore.org/core/enums" +) + +var _MetricsValues = []Metrics{0, 1, 2, 3} + +// MetricsN is the highest valid value for type Metrics, plus one. +const MetricsN Metrics = 4 + +var _MetricsValueMap = map[string]Metrics{`Min`: 0, `Max`: 1, `Avg`: 2, `Contrast`: 3} + +var _MetricsDescMap = map[Metrics]string{0: `Min is the minimum-distance or single-linkage weighting function.`, 1: `Max is the maximum-distance or complete-linkage weighting function.`, 2: `Avg is the average-distance or average-linkage weighting function.`, 3: `Contrast computes maxd + (average within distance - average between distance).`} + +var _MetricsMap = map[Metrics]string{0: `Min`, 1: `Max`, 2: `Avg`, 3: `Contrast`} + +// String returns the string representation of this Metrics value. +func (i Metrics) String() string { return enums.String(i, _MetricsMap) } + +// SetString sets the Metrics value from its string representation, +// and returns an error if the string is invalid. +func (i *Metrics) SetString(s string) error { + return enums.SetString(i, s, _MetricsValueMap, "Metrics") +} + +// Int64 returns the Metrics value as an int64. +func (i Metrics) Int64() int64 { return int64(i) } + +// SetInt64 sets the Metrics value from an int64. +func (i *Metrics) SetInt64(in int64) { *i = Metrics(in) } + +// Desc returns the description of the Metrics value. +func (i Metrics) Desc() string { return enums.Desc(i, _MetricsDescMap) } + +// MetricsValues returns all possible values for the type Metrics. +func MetricsValues() []Metrics { return _MetricsValues } + +// Values returns all possible values for the type Metrics. +func (i Metrics) Values() []enums.Enum { return enums.Values(_MetricsValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i Metrics) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *Metrics) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Metrics") } diff --git a/tensor/stats/cluster/funcs.go b/tensor/stats/cluster/funcs.go new file mode 100644 index 0000000000..a46ac8ccda --- /dev/null +++ b/tensor/stats/cluster/funcs.go @@ -0,0 +1,135 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cluster + +import ( + "log/slog" + "math" + + "cogentcore.org/core/tensor" +) + +// Metrics are standard clustering distance metric functions, +// specifying how a node computes its distance based on its leaves. +type Metrics int32 //enums:enum + +const ( + // Min is the minimum-distance or single-linkage weighting function. + Min Metrics = iota + + // Max is the maximum-distance or complete-linkage weighting function. + Max + + // Avg is the average-distance or average-linkage weighting function. + Avg + + // Contrast computes maxd + (average within distance - average between distance). + Contrast +) + +// MetricFunc is a clustering distance metric function that evaluates aggregate distance +// between nodes, given the indexes of leaves in a and b clusters +// which are indexs into an ntot x ntot distance matrix dmat. +// maxd is the maximum distance value in the dmat, which is needed by the +// ContrastDist function and perhaps others. +type MetricFunc func(aix, bix []int, ntot int, maxd float64, dmat tensor.Tensor) float64 + +// Funcs is a registry of clustering metric functions, +// initialized with the standard options. +var Funcs map[string]MetricFunc + +func init() { + Funcs = make(map[string]MetricFunc) + Funcs[Min.String()] = MinFunc + Funcs[Max.String()] = MaxFunc + Funcs[Avg.String()] = AvgFunc + Funcs[Contrast.String()] = ContrastFunc +} + +// Call calls a cluster metric function by name. +func Call(funcName string, aix, bix []int, ntot int, maxd float64, dmat tensor.Tensor) float64 { + fun, ok := Funcs[funcName] + if !ok { + slog.Error("cluster.Call: function not found", "function:", funcName) + return 0 + } + return fun(aix, bix, ntot, maxd, dmat) +} + +// MinFunc is the minimum-distance or single-linkage weighting function for comparing +// two clusters a and b, given by their list of indexes. +// ntot is total number of nodes, and dmat is the square similarity matrix [ntot x ntot]. +func MinFunc(aix, bix []int, ntot int, maxd float64, dmat tensor.Tensor) float64 { + md := math.MaxFloat64 + for _, ai := range aix { + for _, bi := range bix { + d := dmat.Float(ai, bi) + if d < md { + md = d + } + } + } + return md +} + +// MaxFunc is the maximum-distance or complete-linkage weighting function for comparing +// two clusters a and b, given by their list of indexes. +// ntot is total number of nodes, and dmat is the square similarity matrix [ntot x ntot]. +func MaxFunc(aix, bix []int, ntot int, maxd float64, dmat tensor.Tensor) float64 { + md := -math.MaxFloat64 + for _, ai := range aix { + for _, bi := range bix { + d := dmat.Float(ai, bi) + if d > md { + md = d + } + } + } + return md +} + +// AvgFunc is the average-distance or average-linkage weighting function for comparing +// two clusters a and b, given by their list of indexes. +// ntot is total number of nodes, and dmat is the square similarity matrix [ntot x ntot]. +func AvgFunc(aix, bix []int, ntot int, maxd float64, dmat tensor.Tensor) float64 { + md := 0.0 + n := 0 + for _, ai := range aix { + for _, bi := range bix { + d := dmat.Float(ai, bi) + md += d + n++ + } + } + if n > 0 { + md /= float64(n) + } + return md +} + +// ContrastFunc computes maxd + (average within distance - average between distance) +// for two clusters a and b, given by their list of indexes. +// avg between is average distance between all items in a & b versus all outside that. +// ntot is total number of nodes, and dmat is the square similarity matrix [ntot x ntot]. +// maxd is the maximum distance and is needed to ensure distances are positive. +func ContrastFunc(aix, bix []int, ntot int, maxd float64, dmat tensor.Tensor) float64 { + wd := AvgFunc(aix, bix, ntot, maxd, dmat) + nab := len(aix) + len(bix) + abix := append(aix, bix...) + abmap := make(map[int]struct{}, ntot-nab) + for _, ix := range abix { + abmap[ix] = struct{}{} + } + oix := make([]int, ntot-nab) + octr := 0 + for ix := 0; ix < ntot; ix++ { + if _, has := abmap[ix]; !has { + oix[octr] = ix + octr++ + } + } + bd := AvgFunc(abix, oix, ntot, maxd, dmat) + return maxd + (wd - bd) +} diff --git a/tensor/stats/clust/plot.go b/tensor/stats/cluster/plot.go similarity index 85% rename from tensor/stats/clust/plot.go rename to tensor/stats/cluster/plot.go index 51f9144926..384f796c70 100644 --- a/tensor/stats/clust/plot.go +++ b/tensor/stats/cluster/plot.go @@ -2,17 +2,17 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package clust +package cluster import ( - "cogentcore.org/core/tensor/stats/simat" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) // Plot sets the rows of given data table to trace out lines with labels that // will render cluster plot starting at root node when plotted with a standard plotting package. // The lines double-back on themselves to form a continuous line to be plotted. -func Plot(pt *table.Table, root *Node, smat *simat.SimMat) { +func Plot(pt *table.Table, root *Node, dmat, labels *tensor.Indexed) { pt.DeleteAll() pt.AddFloat64Column("X") pt.AddFloat64Column("Y") @@ -20,20 +20,20 @@ func Plot(pt *table.Table, root *Node, smat *simat.SimMat) { nextY := 0.5 root.SetYs(&nextY) root.SetParDist(0.0) - root.Plot(pt, smat) + root.Plot(pt, dmat, labels) } // Plot sets the rows of given data table to trace out lines with labels that // will render this node in a cluster plot when plotted with a standard plotting package. // The lines double-back on themselves to form a continuous line to be plotted. -func (nn *Node) Plot(pt *table.Table, smat *simat.SimMat) { +func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { row := pt.Rows if nn.IsLeaf() { pt.SetNumRows(row + 1) pt.SetFloatIndex(0, row, nn.ParDist) pt.SetFloatIndex(1, row, nn.Y) - if len(smat.Rows) > nn.Index { - pt.SetStringIndex(2, row, smat.Rows[nn.Index]) + if labels.Len() > nn.Index { + pt.SetStringIndex(2, row, labels.StringValue(nn.Index)) } } else { for _, kn := range nn.Kids { @@ -43,7 +43,7 @@ func (nn *Node) Plot(pt *table.Table, smat *simat.SimMat) { row++ pt.SetFloatIndex(0, row, nn.ParDist+nn.Dist) pt.SetFloatIndex(1, row, kn.Y) - kn.Plot(pt, smat) + kn.Plot(pt, dmat, labels) row = pt.Rows pt.SetNumRows(row + 1) pt.SetFloatIndex(0, row, nn.ParDist) diff --git a/tensor/stats/clust/testdata/faces.dat b/tensor/stats/cluster/testdata/faces.dat similarity index 100% rename from tensor/stats/clust/testdata/faces.dat rename to tensor/stats/cluster/testdata/faces.dat diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 81190db7a3..c4d14e2cf8 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -9,50 +9,20 @@ import ( "cogentcore.org/core/tensor" ) -// note: this answer gives an index into the upper triangular -// https://math.stackexchange.com/questions/2134011/conversion-of-upper-triangle-linear-index-from-index-on-symmetrical-array -// return TriangularN(n) - ((n-c)*(n-c-1))/2 + r - c - 1 <- this works for lower excluding diag -// return (n * (n - 1) / 2) - ((n-r)*(n-r-1))/2 + c <- this works for upper including diag -// but I wasn't able to get an equation for r, c back from index, for this "including diagonal" -// https://stackoverflow.com/questions/27086195/linear-index-upper-triangular-matrix?rq=3 -// python just iterates manually and returns a list -// https://github.com/numpy/numpy/blob/v2.1.0/numpy/lib/_twodim_base_impl.py#L902-L985 - -// TriangularN returns the number of elements in the triangular region -// of a square matrix of given size, where the triangle includes the -// n elements along the diagonal. -func TriangularN(n int) int { - return n + (n*(n-1))/2 -} - -// TriangularLIndicies returns the list of r, c indexes (as X, Y coordinates) -// for the lower triangular portion of a square matrix of size n, -// including the diagonal. -func TriangularLIndicies(n int) []vecint.Vector2i { - trin := TriangularN(n) - coords := make([]vecint.Vector2i, trin) - i := 0 - for r := range n { - for c := range n { - if c <= r { - coords[i] = vecint.Vector2i{X: r, Y: c} - i++ - } - } - } - return coords +func init() { + tensor.AddFunc("metric.Matrix", Matrix, 1, tensor.StringFirstArg) + tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.StringFirstArg) } // Matrix computes the rows x rows distance / similarity matrix between // all sub-space cells of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, -// and within that, 1+dimensional patterns that the given distance metric -// function is applied to, with the results filling the elements of the output matrix. -// The resulting matrix is symmetric, and only the lower triangular part -// is computed, with results copied to the upper triangular region, -// for maximum efficiency. -// See also [LabeledMatrix] struct which can add labels for displaying the matrix. -func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { +// and within that, 1+dimensional patterns. +// The metric function registered in tensor Funcs can be passed as Metrics.String(). +// The results fill in the elements of the output matrix, which is symmetric, +// and only the lower triangular part is computed, with results copied +// to the upper triangular region, for maximum efficiency. +func Matrix(funcName string, in, out *tensor.Indexed) { rows, cells := in.RowCellSize() if rows == 0 || cells == 0 { return @@ -65,9 +35,9 @@ func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { tensor.VectorizeThreaded(cells*3, func(tsr ...*tensor.Indexed) int { return nc }, func(idx int, tsr ...*tensor.Indexed) { c := coords[idx] - sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].SubSpace(c.X))) - sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].SubSpace(c.Y))) - mfun(sa, sb, mout) + sa := tsr[0].Cells1D(c.X) + sb := tsr[0].Cells1D(c.Y) + tensor.Call(funcName, sa, sb, mout) tsr[1].SetFloat(mout.Tensor.Float1D(0), c.X, c.Y) }, in, out) for _, c := range coords { // copy to upper @@ -87,7 +57,7 @@ func Matrix(in, out *tensor.Indexed, mfun MetricFunc) { // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. // See also [LabeledMatrix] struct which can add labels for displaying the matrix. -func CrossMatrix(a, b, out *tensor.Indexed, mfun MetricFunc) { +func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { arows, acells := a.RowCellSize() if arows == 0 || acells == 0 { return @@ -105,9 +75,48 @@ func CrossMatrix(a, b, out *tensor.Indexed, mfun MetricFunc) { func(idx int, tsr ...*tensor.Indexed) { ar := idx / brows br := idx % brows - sa := tensor.NewIndexed(tensor.New1DViewOf(tsr[0].SubSpace(ar))) - sb := tensor.NewIndexed(tensor.New1DViewOf(tsr[1].SubSpace(br))) - mfun(sa, sb, mout) + sa := tsr[0].Cells1D(ar) + sb := tsr[1].Cells1D(br) + tensor.Call(funcName, sa, sb, mout) tsr[2].SetFloat(mout.Tensor.Float1D(0), ar, br) }, a, b, out) } + +//////////////////////////////////////////// +// Triangular square matrix functions + +// TODO: move these somewhere more appropriate + +// note: this answer gives an index into the upper triangular +// https://math.stackexchange.com/questions/2134011/conversion-of-upper-triangle-linear-index-from-index-on-symmetrical-array +// return TriangularN(n) - ((n-c)*(n-c-1))/2 + r - c - 1 <- this works for lower excluding diag +// return (n * (n - 1) / 2) - ((n-r)*(n-r-1))/2 + c <- this works for upper including diag +// but I wasn't able to get an equation for r, c back from index, for this "including diagonal" +// https://stackoverflow.com/questions/27086195/linear-index-upper-triangular-matrix?rq=3 +// python just iterates manually and returns a list +// https://github.com/numpy/numpy/blob/v2.1.0/numpy/lib/_twodim_base_impl.py#L902-L985 + +// TriangularN returns the number of elements in the triangular region +// of a square matrix of given size, where the triangle includes the +// n elements along the diagonal. +func TriangularN(n int) int { + return n + (n*(n-1))/2 +} + +// TriangularLIndicies returns the list of r, c indexes (as X, Y coordinates) +// for the lower triangular portion of a square matrix of size n, +// including the diagonal. +func TriangularLIndicies(n int) []vecint.Vector2i { + trin := TriangularN(n) + coords := make([]vecint.Vector2i, trin) + i := 0 + for r := range n { + for c := range n { + if c <= r { + coords[i] = vecint.Vector2i{X: r, Y: c} + i++ + } + } + } + return coords +} diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 9fcd23c729..a15a126a6b 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -88,12 +88,12 @@ func TestMatrix(t *testing.T) { [11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 ` dt := &table.Table{} - err := dt.OpenCSV("../clust/testdata/faces.dat", table.Tab) + err := dt.OpenCSV("../cluster/testdata/faces.dat", table.Tab) assert.NoError(t, err) // smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) out := tensor.NewIndexed(tensor.NewFloat64()) - Matrix(in, out, EuclideanFunc) + Matrix(Euclidean.String(), in, out) // fmt.Println(out.Tensor) assert.Equal(t, simres, out.Tensor.String()) } diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index a62bc7349d..63a27bf7c6 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -10,26 +10,35 @@ import ( "cogentcore.org/core/tensor" ) +func init() { + tensor.AddFunc("ClosestRow", ClosestRow, 1, tensor.StringFirstArg) +} + // ClosestRow returns the closest fit between probe pattern and patterns in // a "vocabulary" tensor with outermost row dimension, using given metric -// function, *which must have the Increasing property*, i.e., larger = further. -// returns the row and metric value for that row. -// note: this does _not_ use any existing Indexes for the probe (but does for the vocab). -func ClosestRow(probe, vocab *tensor.Indexed, mfun MetricFunc) (int, float64) { +// function registered in tensor Funcs (e.g., use String() method on Metrics enum). +// The metric *must have the Increasing property*, i.e., larger = further. +// Output is a 1D tensor with 2 elements: the row index and metric value for that row. +// Note: this does _not_ use any existing Indexes for the probe, +// but does for the vocab, and the returned index is the logical index +// into any existing Indexes. +func ClosestRow(funcName string, probe, vocab, out *tensor.Indexed) { rows, _ := vocab.Tensor.RowCellSize() mi := -1 - out := tensor.NewFloatScalar(0.0) - // todo: need a 1d view of both spaces + mout := tensor.NewFloatScalar(0.0) mind := math.MaxFloat64 - prview := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) + pr1d := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) for ri := range rows { - sub := tensor.NewIndexed(tensor.New1DViewOf(vocab.SubSpace(ri))) - mfun(prview, sub, out) - d := out.Tensor.Float1D(0) + sub := vocab.Cells1D(ri) + tensor.Call(funcName, pr1d, sub, mout) + d := mout.Tensor.Float1D(0) if d < mind { mi = ri mind = d } } - return mi, mind + out.Tensor.SetShape(2) + out.Sequential() + out.Tensor.SetFloat1D(float64(mi), 0) + out.Tensor.SetFloat1D(mind, 1) } diff --git a/tensor/stats/simat/README.md b/tensor/stats/simat/README.md deleted file mode 100644 index 71b32f2f1e..0000000000 --- a/tensor/stats/simat/README.md +++ /dev/null @@ -1,8 +0,0 @@ -# simat - -`simat` provides similarity / distance matrix functions that create a `SimMat` matrix from Tensor or Table data. Any metric function defined in the [metric](../metric) package (or user-created) can be used. - -The SimMat contains the Tensor of the similarity matrix values, and labels for the Rows and Columns. - -The [tensorcore](../tensorcore) package provides a `SimMatGrid` widget that displays the SimMat with the labels. - diff --git a/tensor/stats/simat/doc.go b/tensor/stats/simat/doc.go deleted file mode 100644 index fcc4953bac..0000000000 --- a/tensor/stats/simat/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package simat provides similarity / distance matrix functions that create -a SimMat matrix from Tensor or Table data. Any metric function defined -in metric package (or user-created) can be used. - -The SimMat contains the Tensor of the similarity matrix values, and -labels for the Rows and Columns. - -The etview package provides a SimMatGrid widget that displays the SimMat -with the labels. -*/ -package simat diff --git a/tensor/stats/simat/simat.go b/tensor/stats/simat/simat.go deleted file mode 100644 index 150a77a730..0000000000 --- a/tensor/stats/simat/simat.go +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package simat - -import ( - "fmt" - - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" -) - -// SimMat is a similarity / distance matrix with additional row and column -// labels for display purposes. -type SimMat struct { - - // the similarity / distance matrix (typically an tensor.Float64) - Mat tensor.Tensor - - // labels for the rows -- blank rows trigger generation of grouping lines - Rows []string - - // labels for the columns -- blank columns trigger generation of grouping lines - Columns []string -} - -// NewSimMat returns a new SimMat similarity matrix -func NewSimMat() *SimMat { - return &SimMat{} -} - -// Init initializes SimMat with default Matrix and nil rows, cols -func (smat *SimMat) Init() { - smat.Mat = &tensor.Float64{} - smat.Mat.Metadata().Set("grid-fill", float32(1)) // best for sim mats -- can override later if need to - smat.Rows = nil - smat.Columns = nil -} - -// TableColumnStd generates a similarity / distance matrix from given column name -// in given Indexed of an table.Table, and given standard metric function. -// if labNm is not empty, uses given column name for labels, which if blankRepeat -// is true are filtered so that any sequentially repeated labels are blank. -// This Std version is usable e.g., in Python where the func cannot be passed. -func (smat *SimMat) TableColumnStd(ix *table.Indexed, column, labNm string, blankRepeat bool, met metric.StdMetrics) error { - return smat.TableColumn(ix, column, labNm, blankRepeat, metric.StdFunc64(met)) -} - -// TableColumn generates a similarity / distance matrix from given column name -// in given Indexed of an table.Table, and given metric function. -// if labNm is not empty, uses given column name for labels, which if blankRepeat -// is true are filtered so that any sequentially repeated labels are blank. -func (smat *SimMat) TableColumn(ix *table.Indexed, column, labNm string, blankRepeat bool, mfun metric.Func64) error { - col, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - smat.Init() - sm := smat.Mat - - rows := ix.Len() - nd := col.NumDims() - if nd < 2 || rows == 0 { - return fmt.Errorf("simat.Tensor: must have 2 or more dims and rows != 0") - } - ln := col.Len() - sz := ln / col.DimSize(0) // size of cell - - sshp := []int{rows, rows} - sm.SetShape(sshp) - - av := make([]float64, sz) - bv := make([]float64, sz) - ardim := []int{0} - brdim := []int{0} - sdim := []int{0, 0} - for ai := 0; ai < rows; ai++ { - ardim[0] = ix.Indexes[ai] - sdim[0] = ai - ar := col.SubSpace(ardim) - ar.Floats(&av) - for bi := 0; bi <= ai; bi++ { // lower diag - brdim[0] = ix.Indexes[bi] - sdim[1] = bi - br := col.SubSpace(brdim) - br.Floats(&bv) - sv := mfun(av, bv) - sm.SetFloat(sdim, sv) - } - } - // now fill in upper diagonal with values from lower diagonal - // note: assumes symmetric distance function - fdim := []int{0, 0} - for ai := 0; ai < rows; ai++ { - sdim[0] = ai - fdim[1] = ai - for bi := ai + 1; bi < rows; bi++ { // upper diag - fdim[0] = bi - sdim[1] = bi - sv := sm.Float(fdim) - sm.SetFloat(sdim, sv) - } - } - - if nm, has := ix.Table.MetaData["name"]; has { - sm.Metadata().Set("name", nm+"_"+column) - } else { - sm.Metadata().Set("name", column) - } - if ds, has := ix.Table.MetaData["desc"]; has { - sm.Metadata().Set("desc", ds) - } - - if labNm == "" { - return nil - } - lc, err := ix.Table.ColumnByName(labNm) - if err != nil { - return err - } - smat.Rows = make([]string, rows) - last := "" - for r := 0; r < rows; r++ { - lbl := lc.String1D(ix.Indexes[r]) - if blankRepeat && lbl == last { - continue - } - smat.Rows[r] = lbl - last = lbl - } - smat.Columns = smat.Rows // identical - return nil -} - -// BlankRepeat returns string slice with any sequentially repeated strings blanked out -func BlankRepeat(str []string) []string { - sz := len(str) - br := make([]string, sz) - last := "" - for r, s := range str { - if s == last { - continue - } - br[r] = s - last = s - } - return br -} diff --git a/tensor/stats/simat/simat_test.go b/tensor/stats/simat/simat_test.go deleted file mode 100644 index 5cc5e24696..0000000000 --- a/tensor/stats/simat/simat_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package simat - -import ( - "testing" - - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" - - "github.com/stretchr/testify/assert" -) - -var simres = `Tensor: [12, 12] -[0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 -[1]: 3.4641016151377544 0 9.38083151964686 8.717797887081348 9.273618495495704 8.831760866327848 5.830951894845301 4.69041575982343 8.717797887081348 7.937253933193772 6.324555320336759 5.291502622129181 -[2]: 8.831760866327848 9.38083151964686 0 3.4641016151377544 4.242640687119285 5.0990195135927845 9.38083151964686 9.899494936611665 4.47213595499958 5.744562646538029 9.38083151964686 9.899494936611665 -[3]: 9.273618495495704 8.717797887081348 3.4641016151377544 0 5.477225575051661 3.7416573867739413 9.797958971132712 9.273618495495704 5.656854249492381 4.58257569495584 9.797958971132712 9.273618495495704 -[4]: 8.717797887081348 9.273618495495704 4.242640687119285 5.477225575051661 0 4 8.831760866327848 9.38083151964686 4.242640687119285 5.5677643628300215 8.831760866327848 9.38083151964686 -[5]: 9.38083151964686 8.831760866327848 5.0990195135927845 3.7416573867739413 4 0 9.486832980505138 8.94427190999916 5.830951894845301 4.795831523312719 9.486832980505138 8.94427190999916 -[6]: 4.69041575982343 5.830951894845301 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 0 3.4641016151377544 9.16515138991168 9.539392014169456 4.242640687119285 5.477225575051661 -[7]: 5.830951894845301 4.69041575982343 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 3.4641016151377544 0 9.695359714832659 9 5.477225575051661 4.242640687119285 -[8]: 8.12403840463596 8.717797887081348 4.47213595499958 5.656854249492381 4.242640687119285 5.830951894845301 9.16515138991168 9.695359714832659 0 3.605551275463989 9.16515138991168 9.695359714832659 -[9]: 8.54400374531753 7.937253933193772 5.744562646538029 4.58257569495584 5.5677643628300215 4.795831523312719 9.539392014169456 9 3.605551275463989 0 9.539392014169456 9 -[10]: 5.291502622129181 6.324555320336759 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 4.242640687119285 5.477225575051661 9.16515138991168 9.539392014169456 0 3.4641016151377544 -[11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 -` - -func TestSimMat(t *testing.T) { - dt := &table.Table{} - err := dt.OpenCSV("../clust/testdata/faces.dat", table.Tab) - if err != nil { - t.Error(err) - } - ix := table.NewIndexed(dt) - smat := &SimMat{} - smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) - - // fmt.Println(smat.Mat) - assert.Equal(t, simres, smat.Mat.String()) -} diff --git a/tensor/stats/simat/tensor.go b/tensor/stats/simat/tensor.go deleted file mode 100644 index fc6d2c0e8a..0000000000 --- a/tensor/stats/simat/tensor.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package simat - -import ( - "fmt" - - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/metric" -) - -// Tensor computes a similarity / distance matrix on tensor -// using given metric function. Outer-most dimension ("rows") is -// used as "indexical" dimension and all other dimensions within that -// are compared. -// Results go in smat which is ensured to have proper square 2D shape -// (rows * rows). -func Tensor(smat tensor.Tensor, a tensor.Tensor, mfun metric.Func64) error { - rows := a.DimSize(0) - nd := a.NumDims() - if nd < 2 || rows == 0 { - return fmt.Errorf("simat.Tensor: must have 2 or more dims and rows != 0") - } - ln := a.Len() - sz := ln / rows - - sshp := []int{rows, rows} - smat.SetShape(sshp) - - av := make([]float64, sz) - bv := make([]float64, sz) - ardim := []int{0} - brdim := []int{0} - sdim := []int{0, 0} - for ai := 0; ai < rows; ai++ { - ardim[0] = ai - sdim[0] = ai - ar := a.SubSpace(ardim) - ar.Floats(&av) - for bi := 0; bi <= ai; bi++ { // lower diag - brdim[0] = bi - sdim[1] = bi - br := a.SubSpace(brdim) - br.Floats(&bv) - sv := mfun(av, bv) - smat.SetFloat(sdim, sv) - } - } - // now fill in upper diagonal with values from lower diagonal - // note: assumes symmetric distance function - fdim := []int{0, 0} - for ai := 0; ai < rows; ai++ { - sdim[0] = ai - fdim[1] = ai - for bi := ai + 1; bi < rows; bi++ { // upper diag - fdim[0] = bi - sdim[1] = bi - sv := smat.Float(fdim) - smat.SetFloat(sdim, sv) - } - } - return nil -} - -// Tensors computes a similarity / distance matrix on two tensors -// using given metric function. Outer-most dimension ("rows") is -// used as "indexical" dimension and all other dimensions within that -// are compared. Resulting reduced 2D shape of two tensors must be -// the same (returns error if not). -// Rows of smat = a, cols = b -func Tensors(smat tensor.Tensor, a, b tensor.Tensor, mfun metric.Func64) error { - arows := a.DimSize(0) - and := a.NumDims() - brows := b.DimSize(0) - bnd := b.NumDims() - if and < 2 || bnd < 2 || arows == 0 || brows == 0 { - return fmt.Errorf("simat.Tensors: must have 2 or more dims and rows != 0") - } - alen := a.Len() - asz := alen / arows - blen := b.Len() - bsz := blen / brows - if asz != bsz { - return fmt.Errorf("simat.Tensors: size of inner dimensions must be same") - } - - sshp := []int{arows, brows} - smat.SetShape(sshp, "a", "b") - - av := make([]float64, asz) - bv := make([]float64, bsz) - ardim := []int{0} - brdim := []int{0} - sdim := []int{0, 0} - for ai := 0; ai < arows; ai++ { - ardim[0] = ai - sdim[0] = ai - ar := a.SubSpace(ardim) - ar.Floats(&av) - for bi := 0; bi < brows; bi++ { - brdim[0] = bi - sdim[1] = bi - br := b.SubSpace(brdim) - br.Floats(&bv) - sv := mfun(av, bv) - smat.SetFloat(sdim, sv) - } - } - return nil -} - -// TensorStd computes a similarity / distance matrix on tensor -// using given Std metric function. Outer-most dimension ("rows") is -// used as "indexical" dimension and all other dimensions within that -// are compared. -// Results go in smat which is ensured to have proper square 2D shape -// (rows * rows). -// This Std version is usable e.g., in Python where the func cannot be passed. -func TensorStd(smat tensor.Tensor, a tensor.Tensor, met metric.StdMetrics) error { - return Tensor(smat, a, metric.StdFunc64(met)) -} - -// TensorsStd computes a similarity / distance matrix on two tensors -// using given Std metric function. Outer-most dimension ("rows") is -// used as "indexical" dimension and all other dimensions within that -// are compared. Resulting reduced 2D shape of two tensors must be -// the same (returns error if not). -// Rows of smat = a, cols = b -// This Std version is usable e.g., in Python where the func cannot be passed. -func TensorsStd(smat tensor.Tensor, a, b tensor.Tensor, met metric.StdMetrics) error { - return Tensors(smat, a, b, metric.StdFunc64(met)) -} From f8869963a1b568ca6778a3b278b9e3bfbf770b67 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 12 Sep 2024 17:02:57 -0700 Subject: [PATCH 024/311] metric.CovarMatrix sketch, from pca, which goes away.. --- tensor/stats/cluster/cluster.go | 5 + tensor/stats/metric/matrix.go | 66 ++++++++++- tensor/stats/pca/covar.go | 193 -------------------------------- 3 files changed, 67 insertions(+), 197 deletions(-) delete mode 100644 tensor/stats/pca/covar.go diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index 1ae426fda5..73dd570b7b 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -16,6 +16,11 @@ import ( "cogentcore.org/core/tensor/stats/stats" ) +// todo: all of this data goes into the datafs +// Cluster makes a new dir, stuffs results in there! +// need a global "cwd" that it uses, so basically you cd +// to a dir, then cal it. + // Node is one node in the cluster type Node struct { // index into original distance matrix; only valid for for terminal leaves. diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index c4d14e2cf8..edb1bfaa1d 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -14,10 +14,10 @@ func init() { tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.StringFirstArg) } -// Matrix computes the rows x rows distance / similarity matrix between -// all sub-space cells of the given higher dimensional input tensor, +// Matrix computes the rows x rows square distance / similarity matrix +// between the patterns for each row of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, -// and within that, 1+dimensional patterns. +// and within that, 1+dimensional patterns (cells). // The metric function registered in tensor Funcs can be passed as Metrics.String(). // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied @@ -54,9 +54,9 @@ func Matrix(funcName string, in, out *tensor.Indexed) { // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns that the given distance metric // function is applied to, with the results filling in the cells of the output matrix. +// The metric function registered in tensor Funcs can be passed as Metrics.String(). // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. -// See also [LabeledMatrix] struct which can add labels for displaying the matrix. func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { arows, acells := a.RowCellSize() if arows == 0 || acells == 0 { @@ -82,6 +82,64 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { }, a, b, out) } +// CovarMatrix generates the cells x cells square covariance matrix +// for all per-row cells of the given higher dimensional input tensor, +// which must have at least 2 dimensions: the outermost rows, +// and within that, 1+dimensional patterns (cells). +// Each value in the resulting matrix represents the extent to which the +// value of a given cell covaries across the rows of the tensor with the +// value of another cell. +// Uses the given metric function, typically [Covariance] or [Correlation], +// which must be registered in tensor Funcs, and can be passed as Metrics.String(). +// Use Covariance if vars have similar overall scaling, +// which is typical in neural network models, and use +// Correlation if they are on very different scales, because it effectively rescales). +// The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. +func CovarMatrix(funcName string, in, out *tensor.Indexed) { + rows, cells := in.RowCellSize() + if rows == 0 || cells == 0 { + return + } + mout := tensor.NewFloatScalar(0.0) + out.Tensor.SetShape(cells, cells) + av := tensor.NewIndexed(tensor.NewFloat64(rows)) + bv := tensor.NewIndexed(tensor.NewFloat64(rows)) + + coords := TriangularLIndicies(cells) + nc := len(coords) + // note: flops estimating 3 per item on average -- different for different metrics. + tensor.VectorizeThreaded(rows*3, func(tsr ...*tensor.Indexed) int { return nc }, + func(idx int, tsr ...*tensor.Indexed) { + // c := coords[idx] + // todo: vectorize: only get new data if diff! + for ai := 0; ai < cells; ai++ { + // todo: extract data from given cell into av + // TableColumnRowsVec(av, ix, col, ai) + for bi := 0; bi <= ai; bi++ { // lower diag + // TableColumnRowsVec(bv, ix, col, bi) + // likewise + tensor.Call(funcName, av, bv, mout) + tsr[1].SetFloat(mout.Tensor.Float1D(0), ai, bi) + } + } + }, in, out) + for _, c := range coords { // copy to upper + if c.X == c.Y { // exclude diag + continue + } + out.Tensor.SetFloat(out.Tensor.Float(c.X, c.Y), c.Y, c.X) + } + // if nm, has := ix.Table.MetaData["name"]; has { + // cmat.SetMetaData("name", nm+"_"+column) + // } else { + // cmat.SetMetaData("name", column) + // } + // if ds, has := ix.Table.MetaData["desc"]; has { + // cmat.SetMetaData("desc", ds) + // } + // return nil +} + //////////////////////////////////////////// // Triangular square matrix functions diff --git a/tensor/stats/pca/covar.go b/tensor/stats/pca/covar.go deleted file mode 100644 index 7afc2b09ec..0000000000 --- a/tensor/stats/pca/covar.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package pca - -import ( - "fmt" - - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" -) - -// CovarTableColumn generates a covariance matrix from given column name -// in given Indexed of an table.Table, and given metric function -// (typically Covariance or Correlation -- use Covar if vars have similar -// overall scaling, which is typical in neural network models, and use -// Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix. -func CovarTableColumn(cmat tensor.Tensor, ix *table.Indexed, column string, mfun metric.Func64) error { - col, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - rows := ix.Len() - nd := col.NumDims() - if nd < 2 || rows == 0 { - return fmt.Errorf("pca.CovarTableColumn: must have 2 or more dims and rows != 0") - } - ln := col.Len() - sz := ln / col.DimSize(0) // size of cell - - cshp := []int{sz, sz} - cmat.SetShape(cshp) - - av := make([]float64, rows) - bv := make([]float64, rows) - sdim := []int{0, 0} - for ai := 0; ai < sz; ai++ { - sdim[0] = ai - TableColumnRowsVec(av, ix, col, ai) - for bi := 0; bi <= ai; bi++ { // lower diag - sdim[1] = bi - TableColumnRowsVec(bv, ix, col, bi) - cv := mfun(av, bv) - cmat.SetFloat(sdim, cv) - } - } - // now fill in upper diagonal with values from lower diagonal - // note: assumes symmetric distance function - fdim := []int{0, 0} - for ai := 0; ai < sz; ai++ { - sdim[0] = ai - fdim[1] = ai - for bi := ai + 1; bi < sz; bi++ { // upper diag - fdim[0] = bi - sdim[1] = bi - cv := cmat.Float(fdim) - cmat.SetFloat(sdim, cv) - } - } - - if nm, has := ix.Table.MetaData["name"]; has { - cmat.SetMetaData("name", nm+"_"+column) - } else { - cmat.SetMetaData("name", column) - } - if ds, has := ix.Table.MetaData["desc"]; has { - cmat.SetMetaData("desc", ds) - } - return nil -} - -// CovarTensor generates a covariance matrix from given tensor.Tensor, -// where the outer-most dimension is rows, and all other dimensions within that -// are covaried against each other, using given metric function -// (typically Covariance or Correlation -- use Covar if vars have similar -// overall scaling, which is typical in neural network models, and use -// Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix. -func CovarTensor(cmat tensor.Tensor, tsr tensor.Tensor, mfun metric.Func64) error { - rows := tsr.DimSize(0) - nd := tsr.NumDims() - if nd < 2 || rows == 0 { - return fmt.Errorf("pca.CovarTensor: must have 2 or more dims and rows != 0") - } - ln := tsr.Len() - sz := ln / rows - - cshp := []int{sz, sz} - cmat.SetShape(cshp) - - av := make([]float64, rows) - bv := make([]float64, rows) - sdim := []int{0, 0} - for ai := 0; ai < sz; ai++ { - sdim[0] = ai - TensorRowsVec(av, tsr, ai) - for bi := 0; bi <= ai; bi++ { // lower diag - sdim[1] = bi - TensorRowsVec(bv, tsr, bi) - cv := mfun(av, bv) - cmat.SetFloat(sdim, cv) - } - } - // now fill in upper diagonal with values from lower diagonal - // note: assumes symmetric distance function - fdim := []int{0, 0} - for ai := 0; ai < sz; ai++ { - sdim[0] = ai - fdim[1] = ai - for bi := ai + 1; bi < sz; bi++ { // upper diag - fdim[0] = bi - sdim[1] = bi - cv := cmat.Float(fdim) - cmat.SetFloat(sdim, cv) - } - } - - if nm, has := tsr.MetaData("name"); has { - cmat.SetMetaData("name", nm+"Covar") - } else { - cmat.SetMetaData("name", "Covar") - } - if ds, has := tsr.MetaData("desc"); has { - cmat.SetMetaData("desc", ds) - } - return nil -} - -// TableColumnRowsVec extracts row-wise vector from given cell index into vec. -// vec must be of size ix.Len() -- number of rows -func TableColumnRowsVec(vec []float64, ix *table.Indexed, col tensor.Tensor, cidx int) { - rows := ix.Len() - ln := col.Len() - sz := ln / col.DimSize(0) // size of cell - for ri := 0; ri < rows; ri++ { - coff := ix.Indexes[ri]*sz + cidx - vec[ri] = col.Float1D(coff) - } -} - -// TensorRowsVec extracts row-wise vector from given cell index into vec. -// vec must be of size tsr.DimSize(0) -- number of rows -func TensorRowsVec(vec []float64, tsr tensor.Tensor, cidx int) { - rows := tsr.DimSize(0) - ln := tsr.Len() - sz := ln / rows - for ri := 0; ri < rows; ri++ { - coff := ri*sz + cidx - vec[ri] = tsr.Float1D(coff) - } -} - -// CovarTableColumnStd generates a covariance matrix from given column name -// in given Indexed of an table.Table, and given metric function -// (typically Covariance or Correlation -- use Covar if vars have similar -// overall scaling, which is typical in neural network models, and use -// Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix. -// This Std version is usable e.g., in Python where the func cannot be passed. -func CovarTableColumnStd(cmat tensor.Tensor, ix *table.Indexed, column string, met metric.StdMetrics) error { - return CovarTableColumn(cmat, ix, column, metric.StdFunc64(met)) -} - -// CovarTensorStd generates a covariance matrix from given tensor.Tensor, -// where the outer-most dimension is rows, and all other dimensions within that -// are covaried against each other, using given metric function -// (typically Covariance or Correlation -- use Covar if vars have similar -// overall scaling, which is typical in neural network models, and use -// Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix. -// This Std version is usable e.g., in Python where the func cannot be passed. -func CovarTensorStd(cmat tensor.Tensor, tsr tensor.Tensor, met metric.StdMetrics) error { - return CovarTensor(cmat, tsr, metric.StdFunc64(met)) -} From ad6972636c5ee8e8deb2e0c6b145d6b91f854ef6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 13 Sep 2024 01:23:40 -0700 Subject: [PATCH 025/311] tensor slice function working --- tensor/shape.go | 8 ++-- tensor/slice.go | 104 ++++++++++++++++++++++++++++++++++++++++++ tensor/tensor_test.go | 40 ++++++++++++++++ 3 files changed, 148 insertions(+), 4 deletions(-) create mode 100644 tensor/slice.go diff --git a/tensor/shape.go b/tensor/shape.go index 6008a88d78..62195252b3 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -39,7 +39,7 @@ func NewShape(sizes ...int) *Shape { // RowMajor ordering is used by default. func (sh *Shape) SetShape(sizes ...int) { sh.Sizes = slices.Clone(sizes) - sh.Strides = RowMajorStrides(sizes) + sh.Strides = RowMajorStrides(sizes...) } // SetNames sets the shape dimension names. @@ -113,7 +113,7 @@ func (sh *Shape) DimSizeByName(name string) int { } // IndexIsValid() returns true if given index is valid (within ranges for all dimensions) -func (sh *Shape) IndexIsValid(idx []int) bool { +func (sh *Shape) IndexIsValid(idx ...int) bool { if len(idx) != sh.NumDims() { return false } @@ -194,7 +194,7 @@ func (sh *Shape) String() string { // RowMajorStrides returns strides for sizes where the first dimension is outermost // and subsequent dimensions are progressively inner. -func RowMajorStrides(sizes []int) []int { +func RowMajorStrides(sizes ...int) []int { rem := int(1) for _, v := range sizes { rem *= v @@ -219,7 +219,7 @@ func RowMajorStrides(sizes []int) []int { // ColMajorStrides returns strides for sizes where the first dimension is inner-most // and subsequent dimensions are progressively outer -func ColMajorStrides(sizes []int) []int { +func ColMajorStrides(sizes ...int) []int { total := int(1) for _, v := range sizes { if v == 0 { diff --git a/tensor/slice.go b/tensor/slice.go new file mode 100644 index 0000000000..60ae321d40 --- /dev/null +++ b/tensor/slice.go @@ -0,0 +1,104 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "fmt" + "slices" +) + +// Range represents a range of values, for extracting slices of data, +// using standard for loop logic with a Start and exclusive End value, +// and an increment: for i := Start; i < End; i += Incr. +// The zero value means all values in the dimension. +type Range struct { + // Starting value. + Start int + + // End value. 0 default = size of relevant dimension. + End int + + // Increment. must be positive, 1 or greater. 0 default = 1. + Incr int +} + +// EndActual is the actual end value given the size of the dimension. +func (rn *Range) EndActual(size int) int { + if rn.End == 0 { + return size + } + return min(rn.End, size) // preserves -1 for no values. +} + +// IncrActual is the actual increment value. +func (rn *Range) IncrActual() int { + return max(1, rn.Incr) +} + +// Size is the number of elements in the actual range given +// size of the dimension. +func (rn *Range) Size(size int) int { + e := rn.EndActual(size) + if e <= rn.Start { + return 0 + } + i := rn.IncrActual() + return (e - rn.Start) / i +} + +// Slice returns a new shape applying the ranges, in order, to +// the dimensions. It is important that all dimensions are non-zero, +// otherwise nothing will be included. An error is returned if this +// is the case. Dimensions beyond the ranges specified are +// automatically included. +func (sh *Shape) Slice(ranges ...Range) ([]int, error) { + nsz := slices.Clone(sh.Sizes) + mx := min(len(ranges), len(sh.Sizes)) + var zd []int + for i := range mx { + nsz[i] = ranges[i].Size(sh.Sizes[i]) + if nsz[i] == 0 { + zd = append(zd, i) + } + } + if len(zd) > 0 { + return nsz, fmt.Errorf("tensor.Shape Slice has zero size for following dimensions: %v", zd) + } + return nsz, nil +} + +// Slice extracts a subset of values from the given tensor into the +// output tensor, according to the provided ranges. +// Dimensions beyond the ranges specified are automatically included. +func Slice(tsr, out Tensor, ranges ...Range) error { + nsz, err := tsr.Shape().Slice(ranges...) + if err != nil { + return err + } + ndim := len(nsz) + out.SetShape(nsz...) + nl := out.Len() + oc := make([]int, ndim) // orig coords + nr := len(ranges) + for ni := range nl { + nc := out.Shape().Index(ni) + for i := range ndim { + c := nc[i] + if i < nr { + r := ranges[i] + oc[i] = r.Start + c*r.IncrActual() + } else { + oc[i] = c + } + } + oi := tsr.Shape().Offset(oc...) + if out.IsString() { + out.SetString1D(tsr.String1D(oi), ni) + } else { + out.SetFloat1D(tsr.Float1D(oi), ni) + } + } + return nil +} diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index e3652a8518..d0da67091b 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -124,3 +124,43 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 3.14, flt[0]) assert.Equal(t, 0.0, flt[1]) } + +func TestSlice(t *testing.T) { + ft := NewFloat64(3, 4, 5) + for r := range 3 { + for y := range 4 { + for x := range 5 { + v := (r+1)*100 + y*10 + x + ft.SetFloat(float64(v), r, y, x) + } + } + } + // fmt.Println(ft.String()) + sf := NewFloat64() + Slice(ft, sf, Range{}, Range{Start: 1, End: 2}) + // fmt.Println(sf.String()) + res := `Tensor: [3, 1, 5] +[0 0]: 110 111 112 113 114 +[1 0]: 210 211 212 213 214 +[2 0]: 310 311 312 313 314 +` + assert.Equal(t, res, sf.String()) + + Slice(ft, sf, Range{}, Range{}, Range{Start: 1, End: 2}) + // fmt.Println(sf.String()) + res = `Tensor: [3, 4, 1] +[0 0]: 101 +[0 1]: 111 +[0 2]: 121 +[0 3]: 131 +[1 0]: 201 +[1 1]: 211 +[1 2]: 221 +[1 3]: 231 +[2 0]: 301 +[2 1]: 311 +[2 2]: 321 +[2 3]: 331 +` + assert.Equal(t, res, sf.String()) +} From 7e4dbe4db8f73ed04ddb1da78bcb8eebe3541337 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 13 Sep 2024 02:11:44 -0700 Subject: [PATCH 026/311] SliceSet function for copying values back to source slice. --- tensor/shape.go | 17 ++--------------- tensor/slice.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/tensor/shape.go b/tensor/shape.go index 62195252b3..638030561b 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -127,10 +127,10 @@ func (sh *Shape) IndexIsValid(idx ...int) bool { // IsEqual returns true if this shape is same as other (does not compare names) func (sh *Shape) IsEqual(oth *Shape) bool { - if !EqualInts(sh.Sizes, oth.Sizes) { + if slices.Compare(sh.Sizes, oth.Sizes) != 0 { return false } - if !EqualInts(sh.Strides, oth.Strides) { + if slices.Compare(sh.Strides, oth.Strides) != 0 { return false } return true @@ -239,19 +239,6 @@ func ColMajorStrides(sizes ...int) []int { return strides } -// EqualInts compares two int slices and returns true if they are equal -func EqualInts(a, b []int) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - // AddShapes returns a new shape by adding two shapes one after the other. func AddShapes(shape1, shape2 *Shape) *Shape { sh1 := shape1.Sizes diff --git a/tensor/slice.go b/tensor/slice.go index 60ae321d40..0afe36a1d2 100644 --- a/tensor/slice.go +++ b/tensor/slice.go @@ -69,9 +69,18 @@ func (sh *Shape) Slice(ranges ...Range) ([]int, error) { return nsz, nil } +// note: the only way to allow arbitrary slicing with shared access +// is with a bitmask. but bitmask is not computationally or memory +// efficient, relative to indexes, and it is simpler to only support one. +// also, the need for direct shared access is limited. + // Slice extracts a subset of values from the given tensor into the // output tensor, according to the provided ranges. // Dimensions beyond the ranges specified are automatically included. +// Unlike the [Tensor.SubSlice] function, the values extracted here are +// copies of the original, not a slice pointer into them, +// which is necessary to allow discontinuous ranges to be extracted. +// Use the [SliceSet] function to copy sliced values back to the original. func Slice(tsr, out Tensor, ranges ...Range) error { nsz, err := tsr.Shape().Slice(ranges...) if err != nil { @@ -102,3 +111,40 @@ func Slice(tsr, out Tensor, ranges ...Range) error { } return nil } + +// SliceSet sets values from the slice into the given tensor. +// Slice tensor must have been created with the [Slice] +// function using the same Range sizes (Start offsets +// can be different). +func SliceSet(tsr, slc Tensor, ranges ...Range) error { + nsz, err := tsr.Shape().Slice(ranges...) + if err != nil { + return err + } + if slices.Compare(nsz, slc.Shape().Sizes) != 0 { + return fmt.Errorf("tensor.SliceSet size from ranges is not the same as the slice tensor") + } + ndim := len(nsz) + nl := slc.Len() + oc := make([]int, ndim) // orig coords + nr := len(ranges) + for ni := range nl { + nc := slc.Shape().Index(ni) + for i := range ndim { + c := nc[i] + if i < nr { + r := ranges[i] + oc[i] = r.Start + c*r.IncrActual() + } else { + oc[i] = c + } + } + oi := tsr.Shape().Offset(oc...) + if slc.IsString() { + tsr.SetString1D(slc.String1D(ni), oi) + } else { + tsr.SetFloat1D(slc.Float1D(ni), oi) + } + } + return nil +} From fc291a680ea8e282d0a830be7235d6b309f5d878 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 13 Sep 2024 04:02:45 -0700 Subject: [PATCH 027/311] metric PCA, SVD functions working and tested; removed pca package. need a simple projection function. --- tensor/slice.go | 57 ++-- tensor/stats/metric/matrix.go | 184 +++++++++++-- tensor/stats/metric/metric_test.go | 46 ++++ .../stats/{pca => metric}/testdata/iris.data | 0 tensor/stats/pca/README.md | 7 - tensor/stats/pca/doc.go | 9 - tensor/stats/pca/pca.go | 230 ----------------- tensor/stats/pca/pca_test.go | 55 ---- tensor/stats/pca/svd.go | 243 ------------------ tensor/stats/pca/svd_test.go | 58 ----- tensor/stats/pca/testdata/projection01.csv | 151 ----------- .../stats/pca/testdata/svd_projection01.csv | 151 ----------- 12 files changed, 243 insertions(+), 948 deletions(-) rename tensor/stats/{pca => metric}/testdata/iris.data (100%) delete mode 100644 tensor/stats/pca/README.md delete mode 100644 tensor/stats/pca/doc.go delete mode 100644 tensor/stats/pca/pca.go delete mode 100644 tensor/stats/pca/pca_test.go delete mode 100644 tensor/stats/pca/svd.go delete mode 100644 tensor/stats/pca/svd_test.go delete mode 100644 tensor/stats/pca/testdata/projection01.csv delete mode 100644 tensor/stats/pca/testdata/svd_projection01.csv diff --git a/tensor/slice.go b/tensor/slice.go index 0afe36a1d2..36f2d78cde 100644 --- a/tensor/slice.go +++ b/tensor/slice.go @@ -48,17 +48,17 @@ func (rn *Range) Size(size int) int { return (e - rn.Start) / i } -// Slice returns a new shape applying the ranges, in order, to -// the dimensions. It is important that all dimensions are non-zero, -// otherwise nothing will be included. An error is returned if this -// is the case. Dimensions beyond the ranges specified are -// automatically included. -func (sh *Shape) Slice(ranges ...Range) ([]int, error) { - nsz := slices.Clone(sh.Sizes) - mx := min(len(ranges), len(sh.Sizes)) +// SliceSize returns a set of sizes applying the ranges, in order, to +// the given dimension sizes. It is important that all dimensions +// are non-zero, otherwise nothing will be included. +// An error is returned if this is the case. +// Dimensions beyond the ranges specified are automatically included. +func SliceSize(sizes []int, ranges ...Range) ([]int, error) { + nsz := slices.Clone(sizes) + mx := min(len(ranges), len(sizes)) var zd []int for i := range mx { - nsz[i] = ranges[i].Size(sh.Sizes[i]) + nsz[i] = ranges[i].Size(sizes[i]) if nsz[i] == 0 { zd = append(zd, i) } @@ -81,18 +81,21 @@ func (sh *Shape) Slice(ranges ...Range) ([]int, error) { // copies of the original, not a slice pointer into them, // which is necessary to allow discontinuous ranges to be extracted. // Use the [SliceSet] function to copy sliced values back to the original. -func Slice(tsr, out Tensor, ranges ...Range) error { - nsz, err := tsr.Shape().Slice(ranges...) +func Slice(tsr, out *Indexed, ranges ...Range) error { + sizes := slices.Clone(tsr.Tensor.Shape().Sizes) + sizes[0] = tsr.Rows() // takes into account indexes + nsz, err := SliceSize(sizes, ranges...) if err != nil { return err } ndim := len(nsz) - out.SetShape(nsz...) + out.Tensor.SetShape(nsz...) + out.Sequential() nl := out.Len() oc := make([]int, ndim) // orig coords nr := len(ranges) for ni := range nl { - nc := out.Shape().Index(ni) + nc := out.Tensor.Shape().Index(ni) for i := range ndim { c := nc[i] if i < nr { @@ -102,11 +105,12 @@ func Slice(tsr, out Tensor, ranges ...Range) error { oc[i] = c } } - oi := tsr.Shape().Offset(oc...) - if out.IsString() { - out.SetString1D(tsr.String1D(oi), ni) + oc[0] = tsr.Index(oc[0]) + oi := tsr.Tensor.Shape().Offset(oc...) + if out.Tensor.IsString() { + out.Tensor.SetString1D(tsr.Tensor.String1D(oi), ni) } else { - out.SetFloat1D(tsr.Float1D(oi), ni) + out.Tensor.SetFloat1D(tsr.Tensor.Float1D(oi), ni) } } return nil @@ -116,12 +120,14 @@ func Slice(tsr, out Tensor, ranges ...Range) error { // Slice tensor must have been created with the [Slice] // function using the same Range sizes (Start offsets // can be different). -func SliceSet(tsr, slc Tensor, ranges ...Range) error { - nsz, err := tsr.Shape().Slice(ranges...) +func SliceSet(tsr, slc *Indexed, ranges ...Range) error { + sizes := slices.Clone(tsr.Tensor.Shape().Sizes) + sizes[0] = tsr.Rows() // takes into account indexes + nsz, err := SliceSize(sizes, ranges...) if err != nil { return err } - if slices.Compare(nsz, slc.Shape().Sizes) != 0 { + if slices.Compare(nsz, slc.Tensor.Shape().Sizes) != 0 { return fmt.Errorf("tensor.SliceSet size from ranges is not the same as the slice tensor") } ndim := len(nsz) @@ -129,7 +135,7 @@ func SliceSet(tsr, slc Tensor, ranges ...Range) error { oc := make([]int, ndim) // orig coords nr := len(ranges) for ni := range nl { - nc := slc.Shape().Index(ni) + nc := slc.Tensor.Shape().Index(ni) for i := range ndim { c := nc[i] if i < nr { @@ -139,11 +145,12 @@ func SliceSet(tsr, slc Tensor, ranges ...Range) error { oc[i] = c } } - oi := tsr.Shape().Offset(oc...) - if slc.IsString() { - tsr.SetString1D(slc.String1D(ni), oi) + oc[0] = tsr.Index(oc[0]) + oi := tsr.Tensor.Shape().Offset(oc...) + if slc.Tensor.IsString() { + tsr.Tensor.SetString1D(slc.Tensor.String1D(ni), oi) } else { - tsr.SetFloat1D(slc.Float1D(ni), oi) + tsr.Tensor.SetFloat1D(slc.Tensor.Float1D(ni), oi) } } return nil diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index edb1bfaa1d..91f638983f 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -7,11 +7,13 @@ package metric import ( "cogentcore.org/core/math32/vecint" "cogentcore.org/core/tensor" + "gonum.org/v1/gonum/mat" ) func init() { tensor.AddFunc("metric.Matrix", Matrix, 1, tensor.StringFirstArg) tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.StringFirstArg) + tensor.AddFunc("metric.CovarMatrix", CovarMatrix, 1, tensor.StringFirstArg) } // Matrix computes the rows x rows square distance / similarity matrix @@ -100,45 +102,189 @@ func CovarMatrix(funcName string, in, out *tensor.Indexed) { if rows == 0 || cells == 0 { return } + + flatsz := []int{in.Tensor.DimSize(0), cells} + flatvw := in.Tensor.View() + flatvw.SetShape(flatsz...) + flatix := tensor.NewIndexed(flatvw) + flatix.Indexes = in.Indexes + mout := tensor.NewFloatScalar(0.0) out.Tensor.SetShape(cells, cells) av := tensor.NewIndexed(tensor.NewFloat64(rows)) bv := tensor.NewIndexed(tensor.NewFloat64(rows)) + curCoords := vecint.Vector2i{-1, -1} coords := TriangularLIndicies(cells) nc := len(coords) // note: flops estimating 3 per item on average -- different for different metrics. tensor.VectorizeThreaded(rows*3, func(tsr ...*tensor.Indexed) int { return nc }, func(idx int, tsr ...*tensor.Indexed) { - // c := coords[idx] - // todo: vectorize: only get new data if diff! - for ai := 0; ai < cells; ai++ { - // todo: extract data from given cell into av - // TableColumnRowsVec(av, ix, col, ai) - for bi := 0; bi <= ai; bi++ { // lower diag - // TableColumnRowsVec(bv, ix, col, bi) - // likewise - tensor.Call(funcName, av, bv, mout) - tsr[1].SetFloat(mout.Tensor.Float1D(0), ai, bi) - } + c := coords[idx] + if c.X != curCoords.X { + tensor.Slice(tsr[0], av, tensor.Range{}, tensor.Range{Start: c.X, End: c.X + 1}) + curCoords.X = c.X } - }, in, out) + if c.Y != curCoords.Y { + tensor.Slice(tsr[0], bv, tensor.Range{}, tensor.Range{Start: c.Y, End: c.Y + 1}) + curCoords.Y = c.Y + } + tensor.Call(funcName, av, bv, mout) + tsr[1].SetFloat(mout.Tensor.Float1D(0), c.X, c.Y) + }, flatix, out) for _, c := range coords { // copy to upper if c.X == c.Y { // exclude diag continue } out.Tensor.SetFloat(out.Tensor.Float(c.X, c.Y), c.Y, c.X) } - // if nm, has := ix.Table.MetaData["name"]; has { - // cmat.SetMetaData("name", nm+"_"+column) - // } else { - // cmat.SetMetaData("name", column) +} + +// PCA performs the eigen decomposition of the given CovarMatrix, +// using principal components analysis (PCA), which is slower than [SVD]. +// The eigenvectors are same size as Covar. Each eigenvector is a column +// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// i.e., maximum eigenvector is the last column. +// The eigenvalues are the size of one row, ordered *lowest* to *highest*. +func PCA(covar, eigenvecs, vals *tensor.Indexed) { + n := covar.Tensor.DimSize(0) + cv, ok := covar.Tensor.(*tensor.Float64) + if !ok { + cv = tensor.NewFloat64(covar.Tensor.Shape().Sizes...) + cv.CopyFrom(covar.Tensor) + } + eigenvecs.Tensor.SetShape(n, n) + eigenvecs.Sequential() + vals.Tensor.SetShape(n) + vals.Sequential() + var eig mat.EigenSym + // note: MUST be a Float64 otherwise doesn't have Symmetric function + eig.Factorize(cv, true) + // if !ok { + // return fmt.Errorf("gonum EigenSym Factorize failed") // } - // if ds, has := ix.Table.MetaData["desc"]; has { - // cmat.SetMetaData("desc", ds) + var ev mat.Dense + eig.VectorsTo(&ev) + tensor.CopyDense(eigenvecs.Tensor, &ev) + eig.Values(vals.Tensor.(*tensor.Float64).Values) +} + +// SVD performs the eigen decomposition of the given CovarMatrix, +// using singular value decomposition (SVD), which is faster than [PCA]. +// The eigenvectors are same size as Covar. Each eigenvector is a column +// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// i.e., maximum eigenvector is the last column. +// The eigenvalues are the size of one row, ordered *lowest* to *highest*. +func SVD(covar, eigenvecs, vals *tensor.Indexed) { + n := covar.Tensor.DimSize(0) + cv, ok := covar.Tensor.(*tensor.Float64) + if !ok { + cv = tensor.NewFloat64(covar.Tensor.Shape().Sizes...) + cv.CopyFrom(covar.Tensor) + } + eigenvecs.Tensor.SetShape(n, n) + eigenvecs.Sequential() + vals.Tensor.SetShape(n) + vals.Sequential() + var eig mat.SVD + eig.Factorize(cv, mat.SVDFull) // todo: test weaker versions than SVDFull + // note: MUST be a Float64 otherwise doesn't have Symmetric function + // if !ok { + // return fmt.Errorf("gonum EigenSym Factorize failed") // } - // return nil + var ev mat.Dense + eig.UTo(&ev) + tensor.CopyDense(eigenvecs.Tensor, &ev) + eig.Values(vals.Tensor.(*tensor.Float64).Values) +} + +// TODO: simple projection function + +/* +// ProjectColumn projects values from the given column of given table (via Indexed) +// onto the idx'th eigenvector (0 = largest eigenvalue, 1 = next, etc). +// Must have already called PCA() method. +func (pa *PCA) ProjectColumn(vals *[]float64, ix *table.Indexed, column string, idx int) error { + col, err := ix.Table.ColumnByName(column) + if err != nil { + return err + } + if pa.Vectors == nil { + return fmt.Errorf("PCA.ProjectColumn Vectors are nil -- must call PCA first") + } + nr := pa.Vectors.DimSize(0) + if idx >= nr { + return fmt.Errorf("PCA.ProjectColumn eigenvector index > rank of matrix") + } + cvec := make([]float64, nr) + eidx := nr - 1 - idx // eigens in reverse order + vec := pa.Vectors.(*tensor.Float64) + for ri := 0; ri < nr; ri++ { + cvec[ri] = vec.Value([]int{ri, eidx}) // vecs are in columns, reverse magnitude order + } + rows := ix.Len() + if len(*vals) != rows { + *vals = make([]float64, rows) + } + ln := col.Len() + sz := ln / col.DimSize(0) // size of cell + if sz != nr { + return fmt.Errorf("PCA.ProjectColumn column cell size != pca eigenvectors") + } + rdim := []int{0} + for row := 0; row < rows; row++ { + sum := 0.0 + rdim[0] = ix.Indexes[row] + rt := col.SubSpace(rdim) + for ci := 0; ci < sz; ci++ { + sum += cvec[ci] * rt.Float1D(ci) + } + (*vals)[row] = sum + } + return nil +} + +// ProjectColumnToTable projects values from the given column of given table (via Indexed) +// onto the given set of eigenvectors (idxs, 0 = largest eigenvalue, 1 = next, etc), +// and stores results along with labels from column labNm into results table. +// Must have already called PCA() method. +func (pa *PCA) ProjectColumnToTable(projections *table.Table, ix *table.Indexed, column, labNm string, idxs []int) error { + _, err := ix.Table.ColumnByName(column) + if err != nil { + return err + } + if pa.Vectors == nil { + return fmt.Errorf("PCA.ProjectColumn Vectors are nil -- must call PCA first") + } + rows := ix.Len() + projections.DeleteAll() + pcolSt := 0 + if labNm != "" { + projections.AddStringColumn(labNm) + pcolSt = 1 + } + for _, idx := range idxs { + projections.AddFloat64Column(fmt.Sprintf("Projection%v", idx)) + } + projections.SetNumRows(rows) + + for ii, idx := range idxs { + pcol := projections.Columns[pcolSt+ii].(*tensor.Float64) + pa.ProjectColumn(&pcol.Values, ix, column, idx) + } + + if labNm != "" { + lcol, err := ix.Table.ColumnByName(labNm) + if err == nil { + plcol := projections.Columns[0] + for row := 0; row < rows; row++ { + plcol.SetString1D(row, lcol.String1D(row)) + } + } + } + return nil } +*/ //////////////////////////////////////////// // Triangular square matrix functions diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index a15a126a6b..cd3abd90bb 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -97,3 +97,49 @@ func TestMatrix(t *testing.T) { // fmt.Println(out.Tensor) assert.Equal(t, simres, out.Tensor.String()) } + +func TestPCAIris(t *testing.T) { + // note: these results are verified against this example: + // https://plot.ly/ipython-notebooks/principal-component-analysis/ + + dt := table.NewTable() + dt.AddFloat64TensorColumn("data", 4) + dt.AddStringColumn("class") + err := dt.OpenCSV("testdata/iris.data", table.Comma) + if err != nil { + t.Error(err) + } + // pc.TableColumn(ix, "data", metric.Covariance64) + // fmt.Printf("covar: %v\n", pc.Covar) + data := tensor.NewIndexed(errors.Log1(dt.ColumnByName("data"))) + covar := tensor.NewIndexed(tensor.NewFloat64()) + CovarMatrix("Correlation", data, covar) + // fmt.Printf("correl: %s\n", covar.Tensor.String()) + + vecs := tensor.NewIndexed(tensor.NewFloat64()) + vals := tensor.NewIndexed(tensor.NewFloat64()) + PCA(covar, vecs, vals) + + // fmt.Printf("correl vec: %v\n", vecs) + // fmt.Printf("correl val: %v\n", vals) + errtol := 1.0e-9 + corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} + for i, v := range vals.Tensor.(*tensor.Float64).Values { + assert.InDelta(t, corvals[i], v, errtol) + } + + // prjt := &table.Table{} + // err = pc.ProjectColumnToTable(prjt, ix, "data", "class", []int{0, 1}) + // if err != nil { + // t.Error(err) + // } + // prjt.SaveCSV("test_data/projection01.csv", table.Comma, true) + + SVD(covar, vecs, vals) + // fmt.Printf("correl vec: %v\n", vecs) + // fmt.Printf("correl val: %v\n", vals) + for i, v := range vals.Tensor.(*tensor.Float64).Values { + assert.InDelta(t, corvals[3-i], v, errtol) // opposite order + } + +} diff --git a/tensor/stats/pca/testdata/iris.data b/tensor/stats/metric/testdata/iris.data similarity index 100% rename from tensor/stats/pca/testdata/iris.data rename to tensor/stats/metric/testdata/iris.data diff --git a/tensor/stats/pca/README.md b/tensor/stats/pca/README.md deleted file mode 100644 index 5f3f7b4d63..0000000000 --- a/tensor/stats/pca/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# pca - -This performs principal component's analysis and associated covariance matrix computations, operating on `table.Table` or `tensor.Tensor` data, using the [gonum](https://github.com/gonum/gonum) matrix interface. - -There is support for the SVD version, which is much faster and produces the same results, with options for how much information to compute trading off with compute time. - - diff --git a/tensor/stats/pca/doc.go b/tensor/stats/pca/doc.go deleted file mode 100644 index f1c43f22f3..0000000000 --- a/tensor/stats/pca/doc.go +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package pca performs principal component's analysis and associated covariance -matrix computations, operating on table.Table or tensor.Tensor data. -*/ -package pca diff --git a/tensor/stats/pca/pca.go b/tensor/stats/pca/pca.go deleted file mode 100644 index 47fcf44c87..0000000000 --- a/tensor/stats/pca/pca.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package pca - -//go:generate core generate - -import ( - "fmt" - - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" - "gonum.org/v1/gonum/mat" -) - -// PCA computes the eigenvalue decomposition of a square similarity matrix, -// typically generated using the correlation metric. -type PCA struct { - - // the covariance matrix computed on original data, which is then eigen-factored - Covar tensor.Tensor `display:"no-inline"` - - // the eigenvectors, in same size as Covar - each eigenvector is a column in this 2D square matrix, ordered *lowest* to *highest* across the columns -- i.e., maximum eigenvector is the last column - Vectors tensor.Tensor `display:"no-inline"` - - // the eigenvalues, ordered *lowest* to *highest* - Values []float64 `display:"no-inline"` -} - -func (pa *PCA) Init() { - pa.Covar = &tensor.Float64{} - pa.Vectors = &tensor.Float64{} - pa.Values = nil -} - -// TableColumn is a convenience method that computes a covariance matrix -// on given column of table and then performs the PCA on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -func (pa *PCA) TableColumn(ix *table.Indexed, column string, mfun metric.Func64) error { - if pa.Covar == nil { - pa.Init() - } - err := CovarTableColumn(pa.Covar, ix, column, mfun) - if err != nil { - return err - } - return pa.PCA() -} - -// Tensor is a convenience method that computes a covariance matrix -// on given tensor and then performs the PCA on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -func (pa *PCA) Tensor(tsr tensor.Tensor, mfun metric.Func64) error { - if pa.Covar == nil { - pa.Init() - } - err := CovarTensor(pa.Covar, tsr, mfun) - if err != nil { - return err - } - return pa.PCA() -} - -// TableColumnStd is a convenience method that computes a covariance matrix -// on given column of table and then performs the PCA on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is a Std metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -// This Std version is usable e.g., in Python where the func cannot be passed. -func (pa *PCA) TableColumnStd(ix *table.Indexed, column string, met metric.StdMetrics) error { - return pa.TableColumn(ix, column, metric.StdFunc64(met)) -} - -// TensorStd is a convenience method that computes a covariance matrix -// on given tensor and then performs the PCA on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is Std metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the PCA eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -// This Std version is usable e.g., in Python where the func cannot be passed. -func (pa *PCA) TensorStd(tsr tensor.Tensor, met metric.StdMetrics) error { - return pa.Tensor(tsr, metric.StdFunc64(met)) -} - -// PCA performs the eigen decomposition of the existing Covar matrix. -// Vectors and Values fields contain the results. -func (pa *PCA) PCA() error { - if pa.Covar == nil || pa.Covar.NumDims() != 2 { - return fmt.Errorf("pca.PCA: Covar matrix is nil or not 2D") - } - var eig mat.EigenSym - // note: MUST be a Float64 otherwise doesn't have Symmetric function - ok := eig.Factorize(pa.Covar.(*tensor.Float64), true) - if !ok { - return fmt.Errorf("gonum EigenSym Factorize failed") - } - if pa.Vectors == nil { - pa.Vectors = &tensor.Float64{} - } - var ev mat.Dense - eig.VectorsTo(&ev) - tensor.CopyDense(pa.Vectors, &ev) - nr := pa.Vectors.DimSize(0) - if len(pa.Values) != nr { - pa.Values = make([]float64, nr) - } - eig.Values(pa.Values) - return nil -} - -// ProjectColumn projects values from the given column of given table (via Indexed) -// onto the idx'th eigenvector (0 = largest eigenvalue, 1 = next, etc). -// Must have already called PCA() method. -func (pa *PCA) ProjectColumn(vals *[]float64, ix *table.Indexed, column string, idx int) error { - col, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - if pa.Vectors == nil { - return fmt.Errorf("PCA.ProjectColumn Vectors are nil -- must call PCA first") - } - nr := pa.Vectors.DimSize(0) - if idx >= nr { - return fmt.Errorf("PCA.ProjectColumn eigenvector index > rank of matrix") - } - cvec := make([]float64, nr) - eidx := nr - 1 - idx // eigens in reverse order - vec := pa.Vectors.(*tensor.Float64) - for ri := 0; ri < nr; ri++ { - cvec[ri] = vec.Value([]int{ri, eidx}) // vecs are in columns, reverse magnitude order - } - rows := ix.Len() - if len(*vals) != rows { - *vals = make([]float64, rows) - } - ln := col.Len() - sz := ln / col.DimSize(0) // size of cell - if sz != nr { - return fmt.Errorf("PCA.ProjectColumn column cell size != pca eigenvectors") - } - rdim := []int{0} - for row := 0; row < rows; row++ { - sum := 0.0 - rdim[0] = ix.Indexes[row] - rt := col.SubSpace(rdim) - for ci := 0; ci < sz; ci++ { - sum += cvec[ci] * rt.Float1D(ci) - } - (*vals)[row] = sum - } - return nil -} - -// ProjectColumnToTable projects values from the given column of given table (via Indexed) -// onto the given set of eigenvectors (idxs, 0 = largest eigenvalue, 1 = next, etc), -// and stores results along with labels from column labNm into results table. -// Must have already called PCA() method. -func (pa *PCA) ProjectColumnToTable(projections *table.Table, ix *table.Indexed, column, labNm string, idxs []int) error { - _, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - if pa.Vectors == nil { - return fmt.Errorf("PCA.ProjectColumn Vectors are nil -- must call PCA first") - } - rows := ix.Len() - projections.DeleteAll() - pcolSt := 0 - if labNm != "" { - projections.AddStringColumn(labNm) - pcolSt = 1 - } - for _, idx := range idxs { - projections.AddFloat64Column(fmt.Sprintf("Projection%v", idx)) - } - projections.SetNumRows(rows) - - for ii, idx := range idxs { - pcol := projections.Columns[pcolSt+ii].(*tensor.Float64) - pa.ProjectColumn(&pcol.Values, ix, column, idx) - } - - if labNm != "" { - lcol, err := ix.Table.ColumnByName(labNm) - if err == nil { - plcol := projections.Columns[0] - for row := 0; row < rows; row++ { - plcol.SetString1D(row, lcol.String1D(row)) - } - } - } - return nil -} diff --git a/tensor/stats/pca/pca_test.go b/tensor/stats/pca/pca_test.go deleted file mode 100644 index 136bd95273..0000000000 --- a/tensor/stats/pca/pca_test.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package pca - -import ( - "fmt" - "math" - "testing" - - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" -) - -func TestPCAIris(t *testing.T) { - // note: these results are verified against this example: - // https://plot.ly/ipython-notebooks/principal-component-analysis/ - - dt := table.NewTable() - dt.AddFloat64TensorColumn("data", []int{4}) - dt.AddStringColumn("class") - err := dt.OpenCSV("testdata/iris.data", table.Comma) - if err != nil { - t.Error(err) - } - ix := table.NewIndexed(dt) - pc := &PCA{} - // pc.TableColumn(ix, "data", metric.Covariance64) - // fmt.Printf("covar: %v\n", pc.Covar) - err = pc.TableColumn(ix, "data", metric.Correlation64) - if err != nil { - t.Error(err) - } - // fmt.Printf("correl: %v\n", pc.Covar) - // fmt.Printf("correl vec: %v\n", pc.Vectors) - // fmt.Printf("correl val: %v\n", pc.Values) - - errtol := 1.0e-9 - corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} - for i, v := range pc.Values { - dif := math.Abs(corvals[i] - v) - if dif > errtol { - err = fmt.Errorf("eigenvalue: %v differs from correct: %v was: %v", i, corvals[i], v) - t.Error(err) - } - } - - prjt := &table.Table{} - err = pc.ProjectColumnToTable(prjt, ix, "data", "class", []int{0, 1}) - if err != nil { - t.Error(err) - } - // prjt.SaveCSV("test_data/projection01.csv", table.Comma, true) -} diff --git a/tensor/stats/pca/svd.go b/tensor/stats/pca/svd.go deleted file mode 100644 index 33bdaa192a..0000000000 --- a/tensor/stats/pca/svd.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package pca - -import ( - "fmt" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" - "gonum.org/v1/gonum/mat" -) - -// SVD computes the eigenvalue decomposition of a square similarity matrix, -// typically generated using the correlation metric. -type SVD struct { - - // type of SVD to run: SVDNone is the most efficient if you only need the values which are always computed. Otherwise, SVDThin is the next most efficient for getting approximate vectors - Kind mat.SVDKind - - // condition value -- minimum normalized eigenvalue to return in values - Cond float64 `default:"0.01"` - - // the rank (count) of singular values greater than Cond - Rank int - - // the covariance matrix computed on original data, which is then eigen-factored - Covar tensor.Tensor `display:"no-inline"` - - // the eigenvectors, in same size as Covar - each eigenvector is a column in this 2D square matrix, ordered *lowest* to *highest* across the columns -- i.e., maximum eigenvector is the last column - Vectors tensor.Tensor `display:"no-inline"` - - // the eigenvalues, ordered *lowest* to *highest* - Values []float64 `display:"no-inline"` -} - -func (svd *SVD) Init() { - svd.Kind = mat.SVDNone - svd.Cond = 0.01 - svd.Covar = &tensor.Float64{} - svd.Vectors = &tensor.Float64{} - svd.Values = nil -} - -// TableColumn is a convenience method that computes a covariance matrix -// on given column of table and then performs the SVD on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the SVD eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -func (svd *SVD) TableColumn(ix *table.Indexed, column string, mfun metric.Func64) error { - if svd.Covar == nil { - svd.Init() - } - err := CovarTableColumn(svd.Covar, ix, column, mfun) - if err != nil { - return err - } - return svd.SVD() -} - -// Tensor is a convenience method that computes a covariance matrix -// on given tensor and then performs the SVD on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the SVD eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -func (svd *SVD) Tensor(tsr tensor.Tensor, mfun metric.Func64) error { - if svd.Covar == nil { - svd.Init() - } - err := CovarTensor(svd.Covar, tsr, mfun) - if err != nil { - return err - } - return svd.SVD() -} - -// TableColumnStd is a convenience method that computes a covariance matrix -// on given column of table and then performs the SVD on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is a Std metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the SVD eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -// This Std version is usable e.g., in Python where the func cannot be passed. -func (svd *SVD) TableColumnStd(ix *table.Indexed, column string, met metric.StdMetrics) error { - return svd.TableColumn(ix, column, metric.StdFunc64(met)) -} - -// TensorStd is a convenience method that computes a covariance matrix -// on given tensor and then performs the SVD on the resulting matrix. -// If no error occurs, the results can be read out from Vectors and Values -// or used in Projection methods. -// mfun is Std metric function, typically Covariance or Correlation -- use Covar -// if vars have similar overall scaling, which is typical in neural network models, -// and use Correl if they are on very different scales -- Correl effectively rescales). -// A Covariance matrix computes the *row-wise* vector similarities for each -// pairwise combination of column cells -- i.e., the extent to which each -// cell co-varies in its value with each other cell across the rows of the table. -// This is the input to the SVD eigenvalue decomposition of the resulting -// covariance matrix, which extracts the eigenvectors as directions with maximal -// variance in this matrix. -// This Std version is usable e.g., in Python where the func cannot be passed. -func (svd *SVD) TensorStd(tsr tensor.Tensor, met metric.StdMetrics) error { - return svd.Tensor(tsr, metric.StdFunc64(met)) -} - -// SVD performs the eigen decomposition of the existing Covar matrix. -// Vectors and Values fields contain the results. -func (svd *SVD) SVD() error { - if svd.Covar == nil || svd.Covar.NumDims() != 2 { - return fmt.Errorf("svd.SVD: Covar matrix is nil or not 2D") - } - var eig mat.SVD - // note: MUST be a Float64 otherwise doesn't have Symmetric function - ok := eig.Factorize(svd.Covar, svd.Kind) - if !ok { - return fmt.Errorf("gonum SVD Factorize failed") - } - if svd.Kind > mat.SVDNone { - if svd.Vectors == nil { - svd.Vectors = &tensor.Float64{} - } - var ev mat.Dense - eig.UTo(&ev) - tensor.CopyDense(svd.Vectors, &ev) - } - nr := svd.Covar.DimSize(0) - if len(svd.Values) != nr { - svd.Values = make([]float64, nr) - } - eig.Values(svd.Values) - svd.Rank = eig.Rank(svd.Cond) - return nil -} - -// ProjectColumn projects values from the given column of given table (via Indexed) -// onto the idx'th eigenvector (0 = largest eigenvalue, 1 = next, etc). -// Must have already called SVD() method. -func (svd *SVD) ProjectColumn(vals *[]float64, ix *table.Indexed, column string, idx int) error { - col, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - if svd.Vectors == nil || svd.Vectors.Len() == 0 { - return fmt.Errorf("SVD.ProjectColumn Vectors are nil: must call SVD first, with Kind = mat.SVDFull so that the vectors are returned") - } - nr := svd.Vectors.DimSize(0) - if idx >= nr { - return fmt.Errorf("SVD.ProjectColumn eigenvector index > rank of matrix") - } - cvec := make([]float64, nr) - // eidx := nr - 1 - idx // eigens in reverse order - vec := svd.Vectors.(*tensor.Float64) - for ri := 0; ri < nr; ri++ { - cvec[ri] = vec.Value([]int{ri, idx}) // vecs are in columns, reverse magnitude order - } - rows := ix.Len() - if len(*vals) != rows { - *vals = make([]float64, rows) - } - ln := col.Len() - sz := ln / col.DimSize(0) // size of cell - if sz != nr { - return fmt.Errorf("SVD.ProjectColumn column cell size != svd eigenvectors") - } - rdim := []int{0} - for row := 0; row < rows; row++ { - sum := 0.0 - rdim[0] = ix.Indexes[row] - rt := col.SubSpace(rdim) - for ci := 0; ci < sz; ci++ { - sum += cvec[ci] * rt.Float1D(ci) - } - (*vals)[row] = sum - } - return nil -} - -// ProjectColumnToTable projects values from the given column of given table (via Indexed) -// onto the given set of eigenvectors (idxs, 0 = largest eigenvalue, 1 = next, etc), -// and stores results along with labels from column labNm into results table. -// Must have already called SVD() method. -func (svd *SVD) ProjectColumnToTable(projections *table.Table, ix *table.Indexed, column, labNm string, idxs []int) error { - _, err := ix.Table.ColumnByName(column) - if errors.Log(err) != nil { - return err - } - if svd.Vectors == nil { - return fmt.Errorf("SVD.ProjectColumn Vectors are nil -- must call SVD first") - } - rows := ix.Len() - projections.DeleteAll() - pcolSt := 0 - if labNm != "" { - projections.AddStringColumn(labNm) - pcolSt = 1 - } - for _, idx := range idxs { - projections.AddFloat64Column(fmt.Sprintf("Projection%v", idx)) - } - projections.SetNumRows(rows) - - for ii, idx := range idxs { - pcol := projections.Columns[pcolSt+ii].(*tensor.Float64) - svd.ProjectColumn(&pcol.Values, ix, column, idx) - } - - if labNm != "" { - lcol, err := ix.Table.ColumnByName(labNm) - if errors.Log(err) == nil { - plcol := projections.Columns[0] - for row := 0; row < rows; row++ { - plcol.SetString1D(row, lcol.String1D(row)) - } - } - } - return nil -} diff --git a/tensor/stats/pca/svd_test.go b/tensor/stats/pca/svd_test.go deleted file mode 100644 index 3c65a63dc7..0000000000 --- a/tensor/stats/pca/svd_test.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package pca - -import ( - "fmt" - "math" - "testing" - - "cogentcore.org/core/tensor/stats/metric" - "cogentcore.org/core/tensor/table" - "gonum.org/v1/gonum/mat" -) - -func TestSVDIris(t *testing.T) { - // note: these results are verified against this example: - // https://plot.ly/ipython-notebooks/principal-component-analysis/ - - dt := table.NewTable() - dt.AddFloat64TensorColumn("data", []int{4}) - dt.AddStringColumn("class") - err := dt.OpenCSV("testdata/iris.data", table.Comma) - if err != nil { - t.Error(err) - } - ix := table.NewIndexed(dt) - pc := &SVD{} - pc.Init() - pc.Kind = mat.SVDFull - // pc.TableColumn(ix, "data", metric.Covariance64) - // fmt.Printf("covar: %v\n", pc.Covar) - err = pc.TableColumn(ix, "data", metric.Correlation64) - if err != nil { - t.Error(err) - } - // fmt.Printf("correl: %v\n", pc.Covar) - // fmt.Printf("correl vec: %v\n", pc.Vectors) - // fmt.Printf("correl val: %v\n", pc.Values) - - errtol := 1.0e-9 - corvals := []float64{2.910818083752054, 0.9212209307072254, 0.14735327830509573, 0.020607707235624825} - for i, v := range pc.Values { - dif := math.Abs(corvals[i] - v) - if dif > errtol { - err = fmt.Errorf("eigenvalue: %v differs from correct: %v was: %v", i, corvals[i], v) - t.Error(err) - } - } - - prjt := &table.Table{} - err = pc.ProjectColumnToTable(prjt, ix, "data", "class", []int{0, 1}) - if err != nil { - t.Error(err) - } - // prjt.SaveCSV("test_data/svd_projection01.csv", table.Comma, true) -} diff --git a/tensor/stats/pca/testdata/projection01.csv b/tensor/stats/pca/testdata/projection01.csv deleted file mode 100644 index 49ef2cc2de..0000000000 --- a/tensor/stats/pca/testdata/projection01.csv +++ /dev/null @@ -1,151 +0,0 @@ -_H:,$class,#Prjn0,#Prjn1 -_D:,Iris-setosa,2.669230878293515,5.180887223993903 -_D:,Iris-setosa,2.6964340118689534,4.643645304250262 -_D:,Iris-setosa,2.4811633041648684,4.752183452725602 -_D:,Iris-setosa,2.5715124347750256,4.6266149223441255 -_D:,Iris-setosa,2.5906582247213548,5.236211037073636 -_D:,Iris-setosa,3.0080988099460617,5.682216917525971 -_D:,Iris-setosa,2.490941664609344,4.90871396981208 -_D:,Iris-setosa,2.7014546083439073,5.053209215928301 -_D:,Iris-setosa,2.461583693196517,4.364930473160547 -_D:,Iris-setosa,2.6716628159090594,4.731768854441222 -_D:,Iris-setosa,2.83139678191279,5.479803509512478 -_D:,Iris-setosa,2.6551056848221406,4.980855020942431 -_D:,Iris-setosa,2.5876357448399223,4.599871891007371 -_D:,Iris-setosa,2.152073732956798,4.4073842762800135 -_D:,Iris-setosa,2.786962753802378,5.900069370044279 -_D:,Iris-setosa,2.91688203729186,6.252471718236359 -_D:,Iris-setosa,2.7755972077070026,5.673779006789473 -_D:,Iris-setosa,2.72579198328178,5.187428800901795 -_D:,Iris-setosa,3.134584682611489,5.694815200208339 -_D:,Iris-setosa,2.7049109092473627,5.4672052268301075 -_D:,Iris-setosa,3.0266540576265015,5.206355516636538 -_D:,Iris-setosa,2.787807505767021,5.381191154323272 -_D:,Iris-setosa,2.1492079743192316,5.078845780997149 -_D:,Iris-setosa,3.065961378000392,5.0217290889404955 -_D:,Iris-setosa,2.829481886501435,4.987183453994805 -_D:,Iris-setosa,2.8649219750292487,4.685096095953507 -_D:,Iris-setosa,2.8727022188802023,5.068401847428211 -_D:,Iris-setosa,2.7795934408940464,5.220228538013025 -_D:,Iris-setosa,2.7478035318656753,5.12556341091417 -_D:,Iris-setosa,2.6555395058441627,4.758511885777976 -_D:,Iris-setosa,2.734112159416322,4.703188072698243 -_D:,Iris-setosa,3.023525466483502,5.215219715084075 -_D:,Iris-setosa,2.565019386717418,5.769020857593508 -_D:,Iris-setosa,2.6938310857368215,5.977704115236996 -_D:,Iris-setosa,2.6716628159090594,4.731768854441222 -_D:,Iris-setosa,2.579749389727401,4.8617694840464685 -_D:,Iris-setosa,2.8200541258968146,5.327705091649766 -_D:,Iris-setosa,2.6716628159090594,4.731768854441222 -_D:,Iris-setosa,2.3771228011053585,4.455376644891153 -_D:,Iris-setosa,2.7536917703846737,5.090441052263297 -_D:,Iris-setosa,2.615429420681249,5.148087486882674 -_D:,Iris-setosa,2.670269508854147,3.8512605122309354 -_D:,Iris-setosa,2.3244518180425704,4.640487943720612 -_D:,Iris-setosa,2.959488937325338,5.174040650658726 -_D:,Iris-setosa,2.993973616474687,5.482184714474499 -_D:,Iris-setosa,2.700757954816452,4.612955044823157 -_D:,Iris-setosa,2.706475204818863,5.462773127606339 -_D:,Iris-setosa,2.487051542683867,4.7170610940747295 -_D:,Iris-setosa,2.7791596198720234,5.44257167317748 -_D:,Iris-setosa,2.669664699315537,4.958544088829447 -_D:,Iris-versicolor,6.337614909993668,5.758736852585482 -_D:,Iris-versicolor,5.964502241617807,5.537668456115144 -_D:,Iris-versicolor,6.48452514559209,5.639709899111897 -_D:,Iris-versicolor,5.327637994258105,4.345950542131197 -_D:,Iris-versicolor,6.180206770343914,5.206787172475347 -_D:,Iris-versicolor,5.5910618634814915,4.893739850295462 -_D:,Iris-versicolor,6.058741494153441,5.603752801471018 -_D:,Iris-versicolor,4.411318411598967,4.180724099023395 -_D:,Iris-versicolor,6.092986230876757,5.323491504409287 -_D:,Iris-versicolor,5.064020246438732,4.608909730008894 -_D:,Iris-versicolor,4.685148340884838,3.8519522930677232 -_D:,Iris-versicolor,5.581611212797471,5.160069542558326 -_D:,Iris-versicolor,5.445475981028535,4.419929343667774 -_D:,Iris-versicolor,5.946486926220956,5.1459833773263215 -_D:,Iris-versicolor,4.989360604871448,4.930078364218072 -_D:,Iris-versicolor,6.03286271372347,5.548157261113388 -_D:,Iris-versicolor,5.599275928354467,5.054702466605709 -_D:,Iris-versicolor,5.267449599849797,4.810353395755553 -_D:,Iris-versicolor,6.123382832850215,4.537648289297856 -_D:,Iris-versicolor,5.155956562699788,4.553101045795742 -_D:,Iris-versicolor,6.0473759480580656,5.377462438216211 -_D:,Iris-versicolor,5.509383508845732,5.032119807214825 -_D:,Iris-versicolor,6.329115122535858,4.8609849846135385 -_D:,Iris-versicolor,5.859700207775821,5.040344574095806 -_D:,Iris-versicolor,5.81413570511593,5.242699398686921 -_D:,Iris-versicolor,6.006961043214098,5.4183697753636615 -_D:,Iris-versicolor,6.396607952597476,5.316160059940694 -_D:,Iris-versicolor,6.577633923578247,5.487883208527084 -_D:,Iris-versicolor,5.834560068048925,5.111074162530968 -_D:,Iris-versicolor,4.892795525981836,4.667909043901078 -_D:,Iris-versicolor,5.071929491630652,4.421204082361892 -_D:,Iris-versicolor,4.957242986082623,4.412553027769875 -_D:,Iris-versicolor,5.264321008706797,4.8192175942030895 -_D:,Iris-versicolor,6.292544559458566,4.94516130671415 -_D:,Iris-versicolor,5.4948016042729355,4.980238793935716 -_D:,Iris-versicolor,5.75944371538022,5.580393986512508 -_D:,Iris-versicolor,6.263800020391029,5.561027271073654 -_D:,Iris-versicolor,5.978036892823293,4.652243143547671 -_D:,Iris-versicolor,5.253652116138878,5.033181402053425 -_D:,Iris-versicolor,5.274967011195318,4.531061840960656 -_D:,Iris-versicolor,5.424572016914717,4.625513824203992 -_D:,Iris-versicolor,5.862026034129797,5.236429549056926 -_D:,Iris-versicolor,5.348781900797956,4.728771422472485 -_D:,Iris-versicolor,4.489891065171127,4.1254002859436625 -_D:,Iris-versicolor,5.3907839912928255,4.757623931493361 -_D:,Iris-versicolor,5.307453573751144,5.065981139164654 -_D:,Iris-versicolor,5.390350170270803,4.979967066657817 -_D:,Iris-versicolor,5.709661381034398,5.168235726016927 -_D:,Iris-versicolor,4.3716421474580756,4.347956564963636 -_D:,Iris-versicolor,5.358560261242432,4.885301939558963 -_D:,Iris-virginica,7.323421646324768,5.690050203535673 -_D:,Iris-virginica,6.357753550341829,4.890322364767835 -_D:,Iris-virginica,7.535955596732253,5.6819621606557655 -_D:,Iris-virginica,6.80033427529343,5.265598656785007 -_D:,Iris-virginica,7.2209683289161575,5.463003241869551 -_D:,Iris-virginica,8.204019210854437,5.882887686119622 -_D:,Iris-virginica,5.478415461702605,4.34438451900287 -_D:,Iris-virginica,7.729583699619444,5.652683363923848 -_D:,Iris-virginica,7.230875690701601,5.048522359834326 -_D:,Iris-virginica,7.772675030657245,6.30491315647896 -_D:,Iris-virginica,6.648297331958487,5.620265043094353 -_D:,Iris-virginica,6.7874273237059555,5.1179323381460655 -_D:,Iris-virginica,7.146742508370896,5.561828740914276 -_D:,Iris-virginica,6.356623075792351,4.672411328827146 -_D:,Iris-virginica,6.614223583751759,5.015585898722028 -_D:,Iris-virginica,6.881994286002045,5.606876892851283 -_D:,Iris-virginica,6.820347707283804,5.430508501185606 -_D:,Iris-virginica,8.160258946192082,6.669215772364471 -_D:,Iris-virginica,8.649096750676604,5.56930851166386 -_D:,Iris-virginica,6.309535511567507,4.473732005048484 -_D:,Iris-virginica,7.375681698444933,5.801473985262766 -_D:,Iris-virginica,6.167254038597639,4.910736963052213 -_D:,Iris-virginica,8.310491651529492,5.7305761244013915 -_D:,Iris-virginica,6.446127454437865,5.065721014166677 -_D:,Iris-virginica,7.131749672855478,5.806482808191717 -_D:,Iris-virginica,7.423963861305202,5.8867900427806665 -_D:,Iris-virginica,6.30942940030594,5.1189353495622845 -_D:,Iris-virginica,6.262646655762151,5.2689242897408715 -_D:,Iris-virginica,7.048590243830385,5.229899574428954 -_D:,Iris-virginica,7.247261833271931,5.6843766347671725 -_D:,Iris-virginica,7.748466657060339,5.59968217238376 -_D:,Iris-virginica,7.9772348586177895,6.724267858166305 -_D:,Iris-virginica,7.10515134881865,5.236441151336846 -_D:,Iris-virginica,6.366359449061206,5.142870888225977 -_D:,Iris-virginica,6.548622005853021,4.887301728239255 -_D:,Iris-virginica,8.07875158007291,5.92265528784978 -_D:,Iris-virginica,7.00802344756605,5.767626365306011 -_D:,Iris-virginica,6.7417750537116445,5.4858323142653385 -_D:,Iris-virginica,6.15228409316162,5.2295829757217485 -_D:,Iris-virginica,7.114518778320504,5.689506748979877 -_D:,Iris-virginica,7.295978570323296,5.638886762401811 -_D:,Iris-virginica,7.0532647866177385,5.6962614697432885 -_D:,Iris-virginica,6.357753550341829,4.890322364767835 -_D:,Iris-virginica,7.439695337523697,5.768461104296018 -_D:,Iris-virginica,7.3579940928085374,5.832649115823288 -_D:,Iris-virginica,7.0332513546273665,5.5313516253426895 -_D:,Iris-virginica,6.613484943048682,4.889260769929234 -_D:,Iris-virginica,6.75909371558104,5.437263221949017 -_D:,Iris-virginica,6.782974379417489,5.719633996694872 -_D:,Iris-virginica,6.274423132800148,5.198679572439127 diff --git a/tensor/stats/pca/testdata/svd_projection01.csv b/tensor/stats/pca/testdata/svd_projection01.csv deleted file mode 100644 index bef23d7481..0000000000 --- a/tensor/stats/pca/testdata/svd_projection01.csv +++ /dev/null @@ -1,151 +0,0 @@ -$class,#Prjn0,#Prjn1 -Iris-setosa,-2.6692308782935164,-5.180887223993902 -Iris-setosa,-2.6964340118689543,-4.643645304250262 -Iris-setosa,-2.4811633041648697,-4.752183452725602 -Iris-setosa,-2.5715124347750264,-4.626614922344125 -Iris-setosa,-2.5906582247213565,-5.236211037073635 -Iris-setosa,-3.0080988099460635,-5.682216917525971 -Iris-setosa,-2.4909416646093447,-4.908713969812081 -Iris-setosa,-2.701454608343909,-5.053209215928301 -Iris-setosa,-2.461583693196518,-4.364930473160547 -Iris-setosa,-2.6716628159090603,-4.731768854441222 -Iris-setosa,-2.8313967819127916,-5.479803509512477 -Iris-setosa,-2.655105684822142,-4.980855020942431 -Iris-setosa,-2.5876357448399236,-4.599871891007371 -Iris-setosa,-2.152073732956799,-4.4073842762800135 -Iris-setosa,-2.7869627538023796,-5.900069370044279 -Iris-setosa,-2.9168820372918622,-6.25247171823636 -Iris-setosa,-2.7755972077070044,-5.673779006789473 -Iris-setosa,-2.7257919832817814,-5.187428800901794 -Iris-setosa,-3.1345846826114907,-5.694815200208339 -Iris-setosa,-2.7049109092473644,-5.467205226830108 -Iris-setosa,-3.0266540576265033,-5.206355516636537 -Iris-setosa,-2.7878075057670233,-5.381191154323272 -Iris-setosa,-2.149207974319233,-5.078845780997149 -Iris-setosa,-3.0659613780003934,-5.0217290889404955 -Iris-setosa,-2.829481886501436,-4.987183453994805 -Iris-setosa,-2.86492197502925,-4.685096095953507 -Iris-setosa,-2.872702218880204,-5.068401847428211 -Iris-setosa,-2.7795934408940473,-5.220228538013024 -Iris-setosa,-2.747803531865676,-5.12556341091417 -Iris-setosa,-2.655539505844164,-4.758511885777976 -Iris-setosa,-2.734112159416324,-4.703188072698243 -Iris-setosa,-3.0235254664835036,-5.215219715084074 -Iris-setosa,-2.565019386717419,-5.769020857593508 -Iris-setosa,-2.6938310857368224,-5.977704115236996 -Iris-setosa,-2.6716628159090603,-4.731768854441222 -Iris-setosa,-2.579749389727403,-4.8617694840464685 -Iris-setosa,-2.820054125896816,-5.327705091649766 -Iris-setosa,-2.6716628159090603,-4.731768854441222 -Iris-setosa,-2.3771228011053593,-4.455376644891153 -Iris-setosa,-2.753691770384675,-5.090441052263298 -Iris-setosa,-2.615429420681251,-5.148087486882674 -Iris-setosa,-2.6702695088541484,-3.851260512230936 -Iris-setosa,-2.3244518180425717,-4.640487943720612 -Iris-setosa,-2.95948893732534,-5.174040650658726 -Iris-setosa,-2.9939736164746886,-5.4821847144745 -Iris-setosa,-2.7007579548164533,-4.612955044823157 -Iris-setosa,-2.7064752048188643,-5.46277312760634 -Iris-setosa,-2.4870515426838677,-4.7170610940747295 -Iris-setosa,-2.779159619872025,-5.44257167317748 -Iris-setosa,-2.6696646993155384,-4.958544088829447 -Iris-versicolor,-6.33761490999367,-5.758736852585481 -Iris-versicolor,-5.96450224161781,-5.537668456115145 -Iris-versicolor,-6.484525145592094,-5.639709899111898 -Iris-versicolor,-5.327637994258107,-4.345950542131198 -Iris-versicolor,-6.180206770343917,-5.206787172475347 -Iris-versicolor,-5.591061863481493,-4.8937398502954625 -Iris-versicolor,-6.058741494153444,-5.6037528014710185 -Iris-versicolor,-4.411318411598969,-4.180724099023395 -Iris-versicolor,-6.092986230876758,-5.323491504409287 -Iris-versicolor,-5.064020246438733,-4.608909730008894 -Iris-versicolor,-4.68514834088484,-3.8519522930677237 -Iris-versicolor,-5.581611212797473,-5.160069542558327 -Iris-versicolor,-5.445475981028536,-4.419929343667774 -Iris-versicolor,-5.946486926220958,-5.1459833773263215 -Iris-versicolor,-4.98936060487145,-4.930078364218073 -Iris-versicolor,-6.032862713723472,-5.548157261113388 -Iris-versicolor,-5.599275928354468,-5.05470246660571 -Iris-versicolor,-5.267449599849799,-4.810353395755553 -Iris-versicolor,-6.123382832850217,-4.537648289297856 -Iris-versicolor,-5.15595656269979,-4.553101045795744 -Iris-versicolor,-6.047375948058068,-5.377462438216212 -Iris-versicolor,-5.509383508845733,-5.032119807214826 -Iris-versicolor,-6.32911512253586,-4.860984984613539 -Iris-versicolor,-5.8597002077758225,-5.040344574095807 -Iris-versicolor,-5.814135705115932,-5.242699398686921 -Iris-versicolor,-6.006961043214099,-5.418369775363661 -Iris-versicolor,-6.396607952597479,-5.316160059940693 -Iris-versicolor,-6.577633923578249,-5.4878832085270846 -Iris-versicolor,-5.834560068048927,-5.111074162530969 -Iris-versicolor,-4.892795525981839,-4.667909043901078 -Iris-versicolor,-5.0719294916306525,-4.421204082361892 -Iris-versicolor,-4.9572429860826235,-4.412553027769875 -Iris-versicolor,-5.2643210087067995,-4.8192175942030895 -Iris-versicolor,-6.292544559458569,-4.94516130671415 -Iris-versicolor,-5.494801604272937,-4.980238793935717 -Iris-versicolor,-5.759443715380222,-5.580393986512508 -Iris-versicolor,-6.263800020391032,-5.561027271073655 -Iris-versicolor,-5.978036892823295,-4.652243143547671 -Iris-versicolor,-5.25365211613888,-5.033181402053426 -Iris-versicolor,-5.274967011195319,-4.531061840960657 -Iris-versicolor,-5.42457201691472,-4.625513824203992 -Iris-versicolor,-5.862026034129799,-5.2364295490569255 -Iris-versicolor,-5.348781900797959,-4.728771422472485 -Iris-versicolor,-4.489891065171128,-4.1254002859436625 -Iris-versicolor,-5.390783991292826,-4.757623931493362 -Iris-versicolor,-5.307453573751146,-5.065981139164655 -Iris-versicolor,-5.390350170270805,-4.979967066657818 -Iris-versicolor,-5.7096613810344,-5.168235726016928 -Iris-versicolor,-4.371642147458076,-4.347956564963637 -Iris-versicolor,-5.358560261242434,-4.885301939558964 -Iris-virginica,-7.32342164632477,-5.690050203535675 -Iris-virginica,-6.357753550341831,-4.890322364767834 -Iris-virginica,-7.535955596732257,-5.681962160655765 -Iris-virginica,-6.800334275293433,-5.265598656785008 -Iris-virginica,-7.22096832891616,-5.463003241869552 -Iris-virginica,-8.204019210854439,-5.882887686119622 -Iris-virginica,-5.478415461702606,-4.344384519002871 -Iris-virginica,-7.729583699619447,-5.652683363923849 -Iris-virginica,-7.230875690701603,-5.048522359834328 -Iris-virginica,-7.772675030657247,-6.30491315647896 -Iris-virginica,-6.648297331958489,-5.620265043094353 -Iris-virginica,-6.787427323705957,-5.117932338146065 -Iris-virginica,-7.146742508370899,-5.561828740914276 -Iris-virginica,-6.356623075792353,-4.672411328827147 -Iris-virginica,-6.614223583751762,-5.015585898722027 -Iris-virginica,-6.881994286002047,-5.606876892851284 -Iris-virginica,-6.820347707283807,-5.4305085011856065 -Iris-virginica,-8.160258946192084,-6.669215772364471 -Iris-virginica,-8.649096750676605,-5.56930851166386 -Iris-virginica,-6.309535511567509,-4.473732005048484 -Iris-virginica,-7.375681698444936,-5.801473985262767 -Iris-virginica,-6.16725403859764,-4.910736963052214 -Iris-virginica,-8.310491651529494,-5.730576124401391 -Iris-virginica,-6.4461274544378675,-5.065721014166677 -Iris-virginica,-7.131749672855481,-5.806482808191716 -Iris-virginica,-7.423963861305205,-5.886790042780667 -Iris-virginica,-6.309429400305942,-5.1189353495622845 -Iris-virginica,-6.262646655762153,-5.2689242897408715 -Iris-virginica,-7.048590243830388,-5.229899574428954 -Iris-virginica,-7.247261833271932,-5.684376634767173 -Iris-virginica,-7.748466657060342,-5.599682172383759 -Iris-virginica,-7.977234858617792,-6.724267858166306 -Iris-virginica,-7.105151348818652,-5.236441151336847 -Iris-virginica,-6.366359449061208,-5.142870888225977 -Iris-virginica,-6.548622005853022,-4.887301728239255 -Iris-virginica,-8.078751580072911,-5.922655287849781 -Iris-virginica,-7.0080234475660514,-5.767626365306012 -Iris-virginica,-6.741775053711646,-5.485832314265339 -Iris-virginica,-6.152284093161622,-5.229582975721749 -Iris-virginica,-7.114518778320507,-5.689506748979878 -Iris-virginica,-7.295978570323298,-5.638886762401812 -Iris-virginica,-7.053264786617741,-5.696261469743289 -Iris-virginica,-6.357753550341831,-4.890322364767834 -Iris-virginica,-7.4396953375237,-5.768461104296019 -Iris-virginica,-7.35799409280854,-5.832649115823288 -Iris-virginica,-7.033251354627368,-5.53135162534269 -Iris-virginica,-6.613484943048684,-4.889260769929234 -Iris-virginica,-6.7590937155810416,-5.437263221949018 -Iris-virginica,-6.7829743794174915,-5.719633996694874 -Iris-virginica,-6.274423132800151,-5.1986795724391275 From 7faebbc9151a4092fda55678c612167c21fa1dca Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 13 Sep 2024 12:42:40 -0700 Subject: [PATCH 028/311] AsFloat64 etc instead of direct float access; package qualified FuncName() for stats, matrix; ProjectionOnMatrixColumn -- but not getting same results as previous.. --- tensor/bits.go | 16 --- tensor/{table => }/enumgen.go | 2 +- tensor/indexed.go | 10 +- tensor/io.go | 45 ++++++-- tensor/number.go | 61 +++++----- tensor/stats/metric/matrix.go | 178 ++++++++++------------------- tensor/stats/metric/metric_test.go | 29 ++--- tensor/stats/metric/metrics.go | 36 +++--- tensor/stats/metric/misc.go | 2 +- tensor/stats/metric/vec.go | 6 +- tensor/stats/stats/stats.go | 72 ++++++------ tensor/string.go | 22 +--- tensor/table/io.go | 51 ++------- tensor/table/typegen.go | 2 +- tensor/tensor.go | 65 ++++++----- tensor/tensorcore/table.go | 4 +- tensor/tensorcore/tensoreditor.go | 4 +- tensor/tmath/math_test.go | 2 +- tensor/tmath/ops_test.go | 8 +- tensor/typegen.go | 2 +- 20 files changed, 272 insertions(+), 345 deletions(-) rename tensor/{table => }/enumgen.go (99%) diff --git a/tensor/bits.go b/tensor/bits.go index da48f336b4..6a416b5926 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -176,22 +176,6 @@ func (tsr *Bits) SetFloatRowCell(val float64, row, cell int) { tsr.Values.Set(Float64ToBool(val), row*sz+cell) } -func (tsr *Bits) Floats(flt *[]float64) { - sz := tsr.Len() - *flt = slicesx.SetLength(*flt, sz) - for j := 0; j < sz; j++ { - (*flt)[j] = BoolToFloat64(tsr.Values.Index(j)) - } -} - -// SetFloats sets tensor values from a []float64 slice (copies values). -func (tsr *Bits) SetFloats(vals ...float64) { - sz := min(tsr.Len(), len(vals)) - for j := 0; j < sz; j++ { - tsr.Values.Set(Float64ToBool(vals[j]), j) - } -} - func (tsr *Bits) String1D(off int) string { return reflectx.ToString(tsr.Values.Index(off)) } diff --git a/tensor/table/enumgen.go b/tensor/enumgen.go similarity index 99% rename from tensor/table/enumgen.go rename to tensor/enumgen.go index c4eda161e5..eaeb7bfb3b 100644 --- a/tensor/table/enumgen.go +++ b/tensor/enumgen.go @@ -1,6 +1,6 @@ // Code generated by "core generate"; DO NOT EDIT. -package table +package tensor import ( "cogentcore.org/core/enums" diff --git a/tensor/indexed.go b/tensor/indexed.go index 2a7d660991..60f46861a3 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -54,9 +54,15 @@ func NewIndexed(tsr Tensor, idxs ...[]int) *Indexed { return ix } -// NewFloatScalar is a convenience method to quickly get an Indexed +// NewFloat64Indexed is a convenience method to quickly get an Indexed +// representation of [Float64] tensor of given shape, for use in math routines etc. +func NewFloat64Indexed(sizes ...int) *Indexed { + return &Indexed{Tensor: NewFloat64(sizes...)} +} + +// NewFloat64Scalar is a convenience method to quickly get an Indexed // representation of a single float64 scalar value, for use in math routines etc. -func NewFloatScalar(val float64) *Indexed { +func NewFloat64Scalar(val float64) *Indexed { return &Indexed{Tensor: NewNumberFromSlice([]float64{val})} } diff --git a/tensor/io.go b/tensor/io.go index f10a31010e..be3ac42f18 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -15,11 +15,40 @@ import ( "cogentcore.org/core/core" ) +// Delim are standard CSV delimiter options (Tab, Comma, Space) +type Delims int32 //enums:enum + +const ( + // Tab is the tab rune delimiter, for TSV tab separated values + Tab Delims = iota + + // Comma is the comma rune delimiter, for CSV comma separated values + Comma + + // Space is the space rune delimiter, for SSV space separated value + Space + + // Detect is used during reading a file -- reads the first line and detects tabs or commas + Detect +) + +func (dl Delims) Rune() rune { + switch dl { + case Tab: + return '\t' + case Comma: + return ',' + case Space: + return ' ' + } + return '\t' +} + // SaveCSV writes a tensor to a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg). // Outer-most dims are rows in the file, and inner-most is column -- // Reading just grabs all values and doesn't care about shape. -func SaveCSV(tsr Tensor, filename core.Filename, delim rune) error { +func SaveCSV(tsr Tensor, filename core.Filename, delim Delims) error { fp, err := os.Create(string(filename)) defer fp.Close() if err != nil { @@ -35,7 +64,7 @@ func SaveCSV(tsr Tensor, filename core.Filename, delim rune) error { // using the Go standard encoding/csv reader conforming // to the official CSV standard. // Reads all values and assigns as many as fit. -func OpenCSV(tsr Tensor, filename core.Filename, delim rune) error { +func OpenCSV(tsr Tensor, filename core.Filename, delim Delims) error { fp, err := os.Open(string(filename)) defer fp.Close() if err != nil { @@ -52,15 +81,13 @@ func OpenCSV(tsr Tensor, filename core.Filename, delim rune) error { // (where comma = any delimiter, specified in the delim arg). // Outer-most dims are rows in the file, and inner-most is column -- // Reading just grabs all values and doesn't care about shape. -func WriteCSV(tsr Tensor, w io.Writer, delim rune) error { +func WriteCSV(tsr Tensor, w io.Writer, delim Delims) error { prec := -1 if ps, err := metadata.Get[int](*tsr.Metadata(), "precision"); err == nil { prec = ps } cw := csv.NewWriter(w) - if delim != 0 { - cw.Comma = delim - } + cw.Comma = delim.Rune() nrow := tsr.DimSize(0) nin := tsr.Len() / nrow rec := make([]string, nin) @@ -89,11 +116,9 @@ func WriteCSV(tsr Tensor, w io.Writer, delim rune) error { // using the Go standard encoding/csv reader conforming // to the official CSV standard. // Reads all values and assigns as many as fit. -func ReadCSV(tsr Tensor, r io.Reader, delim rune) error { +func ReadCSV(tsr Tensor, r io.Reader, delim Delims) error { cr := csv.NewReader(r) - if delim != 0 { - cr.Comma = delim - } + cr.Comma = delim.Rune() rec, err := cr.ReadAll() // todo: lazy, avoid resizing if err != nil || len(rec) == 0 { return err diff --git a/tensor/number.go b/tensor/number.go index f398c2e014..4911e85e94 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -12,7 +12,6 @@ import ( "strings" "cogentcore.org/core/base/num" - "cogentcore.org/core/base/slicesx" "gonum.org/v1/gonum/mat" ) @@ -36,18 +35,46 @@ type Int32 = Number[int32] // Byte is an alias for Number[byte]. type Byte = Number[byte] -// NewFloat32 returns a new Float32 tensor +// NewFloat32 returns a new [Float32] tensor // with the given sizes per dimension (shape), and optional dimension names. func NewFloat32(sizes ...int) *Float32 { return New[float32](sizes...).(*Float32) } -// NewFloat64 returns a new Float64 tensor +// AsFloat32 returns the tensor as a [Float32] tensor. +// If already is a Float32, it is returned as such. +// Otherwise, a new Float32 tensor is created and values are copied. +func AsFloat32(tsr Tensor) *Float32 { + if f, ok := tsr.(*Float32); ok { + return f + } + f := NewFloat32(tsr.Shape().Sizes...) + f.SetNames(tsr.Shape().Names...) + f.CopyFrom(tsr) + return f +} + +// NewFloat64 returns a new [Float64] tensor // with the given sizes per dimension (shape), and optional dimension names. func NewFloat64(sizes ...int) *Float64 { return New[float64](sizes...).(*Float64) } +// AsFloat64 returns the tensor as a [Float64] tensor. +// If already is a Float64, it is returned as such. +// Otherwise, a new Float64 tensor is created and values are copied. +// Use this function for interfacing with gonum or other apis that +// only operate on float64 types. +func AsFloat64(tsr Tensor) *Float64 { + if f, ok := tsr.(*Float64); ok { + return f + } + f := NewFloat64(tsr.Shape().Sizes...) + f.SetNames(tsr.Shape().Names...) + f.CopyFrom(tsr) + return f +} + // NewInt returns a new Int tensor // with the given sizes per dimension (shape), and optional dimension names. func NewInt(sizes ...int) *Int { @@ -169,34 +196,6 @@ func (tsr *Number[T]) SetFloatRowCell(val float64, row, cell int) { tsr.Values[row*sz+cell] = T(val) } -// Floats sets []float64 slice of all elements in the tensor -// (length is ensured to be sufficient). -// This can be used for all of the gonum/floats methods -// for basic math, gonum/stats, etc. -func (tsr *Number[T]) Floats(flt *[]float64) { - *flt = slicesx.SetLength(*flt, len(tsr.Values)) - switch vals := any(tsr.Values).(type) { - case []float64: - copy(*flt, vals) - default: - for i, v := range tsr.Values { - (*flt)[i] = float64(v) - } - } -} - -// SetFloats sets tensor values from a []float64 slice (copies values). -func (tsr *Number[T]) SetFloats(flt ...float64) { - switch vals := any(tsr.Values).(type) { - case []float64: - copy(vals, flt) - default: - for i, v := range flt { - tsr.Values[i] = T(v) - } - } -} - // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. func (tsr *Number[T]) At(i, j int) float64 { diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 91f638983f..1e786dc058 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -5,8 +5,12 @@ package metric import ( + "log/slog" + "cogentcore.org/core/math32/vecint" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" + "cogentcore.org/core/tensor/tmath" "gonum.org/v1/gonum/mat" ) @@ -20,7 +24,7 @@ func init() { // between the patterns for each row of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns (cells). -// The metric function registered in tensor Funcs can be passed as Metrics.String(). +// The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied // to the upper triangular region, for maximum efficiency. @@ -30,7 +34,7 @@ func Matrix(funcName string, in, out *tensor.Indexed) { return } out.Tensor.SetShape(rows, rows) - mout := tensor.NewFloatScalar(0.0) + mout := tensor.NewFloat64Scalar(0.0) coords := TriangularLIndicies(rows) nc := len(coords) // note: flops estimating 3 per item on average -- different for different metrics. @@ -56,7 +60,7 @@ func Matrix(funcName string, in, out *tensor.Indexed) { // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns that the given distance metric // function is applied to, with the results filling in the cells of the output matrix. -// The metric function registered in tensor Funcs can be passed as Metrics.String(). +// The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { @@ -69,7 +73,7 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { return } out.Tensor.SetShape(arows, brows) - mout := tensor.NewFloatScalar(0.0) + mout := tensor.NewFloat64Scalar(0.0) // note: flops estimating 3 per item on average -- different for different metrics. flops := min(acells, bcells) * 3 nc := arows * brows @@ -92,7 +96,7 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { // value of a given cell covaries across the rows of the tensor with the // value of another cell. // Uses the given metric function, typically [Covariance] or [Correlation], -// which must be registered in tensor Funcs, and can be passed as Metrics.String(). +// which must be registered in tensor Funcs, and can be passed as Metrics.FuncName(). // Use Covariance if vars have similar overall scaling, // which is typical in neural network models, and use // Correlation if they are on very different scales, because it effectively rescales). @@ -109,10 +113,10 @@ func CovarMatrix(funcName string, in, out *tensor.Indexed) { flatix := tensor.NewIndexed(flatvw) flatix.Indexes = in.Indexes - mout := tensor.NewFloatScalar(0.0) + mout := tensor.NewFloat64Scalar(0.0) out.Tensor.SetShape(cells, cells) - av := tensor.NewIndexed(tensor.NewFloat64(rows)) - bv := tensor.NewIndexed(tensor.NewFloat64(rows)) + av := tensor.NewFloat64Indexed(rows) + bv := tensor.NewFloat64Indexed(rows) curCoords := vecint.Vector2i{-1, -1} coords := TriangularLIndicies(cells) @@ -146,145 +150,89 @@ func CovarMatrix(funcName string, in, out *tensor.Indexed) { // in this 2D square matrix, ordered *lowest* to *highest* across the columns, // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *lowest* to *highest*. +// Note that PCA produces results in the *opposite* order of [SVD]. func PCA(covar, eigenvecs, vals *tensor.Indexed) { n := covar.Tensor.DimSize(0) - cv, ok := covar.Tensor.(*tensor.Float64) - if !ok { - cv = tensor.NewFloat64(covar.Tensor.Shape().Sizes...) - cv.CopyFrom(covar.Tensor) - } + cv := tensor.AsFloat64(covar.Tensor) eigenvecs.Tensor.SetShape(n, n) eigenvecs.Sequential() vals.Tensor.SetShape(n) vals.Sequential() var eig mat.EigenSym - // note: MUST be a Float64 otherwise doesn't have Symmetric function - eig.Factorize(cv, true) - // if !ok { - // return fmt.Errorf("gonum EigenSym Factorize failed") - // } + ok := eig.Factorize(cv, true) + if !ok { + slog.Error("gonum mat.EigenSym Factorize failed") + return + } var ev mat.Dense eig.VectorsTo(&ev) tensor.CopyDense(eigenvecs.Tensor, &ev) - eig.Values(vals.Tensor.(*tensor.Float64).Values) + fv := tensor.AsFloat64(vals.Tensor) + eig.Values(fv.Values) + if fv != vals.Tensor { + vals.Tensor.CopyFrom(fv) + } } // SVD performs the eigen decomposition of the given CovarMatrix, // using singular value decomposition (SVD), which is faster than [PCA]. // The eigenvectors are same size as Covar. Each eigenvector is a column -// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// in this 2D square matrix, ordered *highest* to *lowest* across the columns, // i.e., maximum eigenvector is the last column. -// The eigenvalues are the size of one row, ordered *lowest* to *highest*. +// The eigenvalues are the size of one row, ordered *highest* to *lowest*. +// Note that SVD produces results in the *opposite* order of [PCA]. func SVD(covar, eigenvecs, vals *tensor.Indexed) { n := covar.Tensor.DimSize(0) - cv, ok := covar.Tensor.(*tensor.Float64) - if !ok { - cv = tensor.NewFloat64(covar.Tensor.Shape().Sizes...) - cv.CopyFrom(covar.Tensor) - } + cv := tensor.AsFloat64(covar.Tensor) eigenvecs.Tensor.SetShape(n, n) eigenvecs.Sequential() vals.Tensor.SetShape(n) vals.Sequential() var eig mat.SVD - eig.Factorize(cv, mat.SVDFull) // todo: test weaker versions than SVDFull - // note: MUST be a Float64 otherwise doesn't have Symmetric function - // if !ok { - // return fmt.Errorf("gonum EigenSym Factorize failed") - // } + ok := eig.Factorize(cv, mat.SVDFull) // todo: benchmark different versions + if !ok { + slog.Error("gonum mat.SVD Factorize failed") + return + } var ev mat.Dense eig.UTo(&ev) tensor.CopyDense(eigenvecs.Tensor, &ev) - eig.Values(vals.Tensor.(*tensor.Float64).Values) -} - -// TODO: simple projection function - -/* -// ProjectColumn projects values from the given column of given table (via Indexed) -// onto the idx'th eigenvector (0 = largest eigenvalue, 1 = next, etc). -// Must have already called PCA() method. -func (pa *PCA) ProjectColumn(vals *[]float64, ix *table.Indexed, column string, idx int) error { - col, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - if pa.Vectors == nil { - return fmt.Errorf("PCA.ProjectColumn Vectors are nil -- must call PCA first") - } - nr := pa.Vectors.DimSize(0) - if idx >= nr { - return fmt.Errorf("PCA.ProjectColumn eigenvector index > rank of matrix") - } - cvec := make([]float64, nr) - eidx := nr - 1 - idx // eigens in reverse order - vec := pa.Vectors.(*tensor.Float64) - for ri := 0; ri < nr; ri++ { - cvec[ri] = vec.Value([]int{ri, eidx}) // vecs are in columns, reverse magnitude order - } - rows := ix.Len() - if len(*vals) != rows { - *vals = make([]float64, rows) - } - ln := col.Len() - sz := ln / col.DimSize(0) // size of cell - if sz != nr { - return fmt.Errorf("PCA.ProjectColumn column cell size != pca eigenvectors") - } - rdim := []int{0} - for row := 0; row < rows; row++ { - sum := 0.0 - rdim[0] = ix.Indexes[row] - rt := col.SubSpace(rdim) - for ci := 0; ci < sz; ci++ { - sum += cvec[ci] * rt.Float1D(ci) - } - (*vals)[row] = sum + fv := tensor.AsFloat64(vals.Tensor) + eig.Values(fv.Values) + if fv != vals.Tensor { + vals.Tensor.CopyFrom(fv) } - return nil } -// ProjectColumnToTable projects values from the given column of given table (via Indexed) -// onto the given set of eigenvectors (idxs, 0 = largest eigenvalue, 1 = next, etc), -// and stores results along with labels from column labNm into results table. -// Must have already called PCA() method. -func (pa *PCA) ProjectColumnToTable(projections *table.Table, ix *table.Indexed, column, labNm string, idxs []int) error { - _, err := ix.Table.ColumnByName(column) - if err != nil { - return err - } - if pa.Vectors == nil { - return fmt.Errorf("PCA.ProjectColumn Vectors are nil -- must call PCA first") - } - rows := ix.Len() - projections.DeleteAll() - pcolSt := 0 - if labNm != "" { - projections.AddStringColumn(labNm) - pcolSt = 1 - } - for _, idx := range idxs { - projections.AddFloat64Column(fmt.Sprintf("Projection%v", idx)) - } - projections.SetNumRows(rows) - - for ii, idx := range idxs { - pcol := projections.Columns[pcolSt+ii].(*tensor.Float64) - pa.ProjectColumn(&pcol.Values, ix, column, idx) - } - - if labNm != "" { - lcol, err := ix.Table.ColumnByName(labNm) - if err == nil { - plcol := projections.Columns[0] - for row := 0; row < rows; row++ { - plcol.SetString1D(row, lcol.String1D(row)) - } +// ProjectOnMatrixColumn is a convenience function for projecting given vector +// of values along a specific column (2nd dimension) of the given 2D matrix, +// specified by the scalar colindex, putting results into out. +// If the vec is more than 1 dimensional, then it is treated as rows x cells, +// and each row of cells is projected through the matrix column, producing a +// 1D output with the number of rows. Otherwise a single number is produced. +// This is typically done with results from SVD or PCA. +func ProjectOnMatrixColumn(mtx, vec, colindex, out *tensor.Indexed) { + ci := int(colindex.Tensor.Float1D(0)) + col := tensor.NewFloat64Indexed() + tensor.Slice(mtx, col, tensor.Range{}, tensor.Range{Start: ci, End: ci + 1}) + // fmt.Println(mtx.Tensor.String(), col.Tensor.String()) + rows, cells := vec.RowCellSize() + mout := tensor.NewFloat64Indexed() + if rows > 0 && cells > 0 { + msum := tensor.NewFloat64Scalar(0) + out.Tensor.SetShape(rows) + out.Sequential() + for i := range rows { + tmath.Mul(vec.Cells1D(i), col, mout) + stats.SumFunc(mout, msum) + // fmt.Println(vec.Cells1D(i).Tensor.String(), mout.Tensor.String(), msum.Tensor.String()) + out.Tensor.SetFloat1D(msum.Tensor.Float1D(0), i) } + } else { + tmath.Mul(vec, col, mout) + stats.SumFunc(mout, out) } - return nil } -*/ //////////////////////////////////////////// // Triangular square matrix functions diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index cd3abd90bb..89d2bffe96 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -88,12 +88,12 @@ func TestMatrix(t *testing.T) { [11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 ` dt := &table.Table{} - err := dt.OpenCSV("../cluster/testdata/faces.dat", table.Tab) + err := dt.OpenCSV("../cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) // smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) - out := tensor.NewIndexed(tensor.NewFloat64()) - Matrix(Euclidean.String(), in, out) + out := tensor.NewFloat64Indexed() + Matrix(Euclidean.FuncName(), in, out) // fmt.Println(out.Tensor) assert.Equal(t, simres, out.Tensor.String()) } @@ -105,19 +105,19 @@ func TestPCAIris(t *testing.T) { dt := table.NewTable() dt.AddFloat64TensorColumn("data", 4) dt.AddStringColumn("class") - err := dt.OpenCSV("testdata/iris.data", table.Comma) + err := dt.OpenCSV("testdata/iris.data", tensor.Comma) if err != nil { t.Error(err) } // pc.TableColumn(ix, "data", metric.Covariance64) // fmt.Printf("covar: %v\n", pc.Covar) data := tensor.NewIndexed(errors.Log1(dt.ColumnByName("data"))) - covar := tensor.NewIndexed(tensor.NewFloat64()) - CovarMatrix("Correlation", data, covar) + covar := tensor.NewFloat64Indexed() + CovarMatrix(Correlation.FuncName(), data, covar) // fmt.Printf("correl: %s\n", covar.Tensor.String()) - vecs := tensor.NewIndexed(tensor.NewFloat64()) - vals := tensor.NewIndexed(tensor.NewFloat64()) + vecs := tensor.NewFloat64Indexed() + vals := tensor.NewFloat64Indexed() PCA(covar, vecs, vals) // fmt.Printf("correl vec: %v\n", vecs) @@ -128,12 +128,13 @@ func TestPCAIris(t *testing.T) { assert.InDelta(t, corvals[i], v, errtol) } - // prjt := &table.Table{} - // err = pc.ProjectColumnToTable(prjt, ix, "data", "class", []int{0, 1}) - // if err != nil { - // t.Error(err) - // } - // prjt.SaveCSV("test_data/projection01.csv", table.Comma, true) + colidx := tensor.NewFloat64Scalar(3) // strongest at end + prjns := tensor.NewFloat64Indexed() + ProjectOnMatrixColumn(covar, data, colidx, prjns) + tensor.SaveCSV(prjns.Tensor, "testdata/pca_projection.csv", tensor.Comma) + + //////////////////////////////////////////////////////////// + // SVD SVD(covar, vecs, vals) // fmt.Printf("correl vec: %v\n", vecs) diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 0b22754928..ff04b8a1c6 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -12,31 +12,31 @@ import ( ) func init() { - tensor.AddFunc(Euclidean.String(), EuclideanFunc, 1) - tensor.AddFunc(SumSquares.String(), SumSquaresFunc, 1) - tensor.AddFunc(Abs.String(), AbsFunc, 1) - tensor.AddFunc(Hamming.String(), HammingFunc, 1) - tensor.AddFunc(EuclideanBinTol.String(), EuclideanBinTolFunc, 1) - tensor.AddFunc(SumSquaresBinTol.String(), SumSquaresBinTolFunc, 1) - tensor.AddFunc(InvCosine.String(), InvCosineFunc, 1) - tensor.AddFunc(InvCorrelation.String(), InvCorrelationFunc, 1) - tensor.AddFunc(InnerProduct.String(), InnerProductFunc, 1) - tensor.AddFunc(CrossEntropy.String(), CrossEntropyFunc, 1) - tensor.AddFunc(Covariance.String(), CovarianceFunc, 1) - tensor.AddFunc(Correlation.String(), CorrelationFunc, 1) - tensor.AddFunc(Cosine.String(), CosineFunc, 1) + tensor.AddFunc(Euclidean.FuncName(), EuclideanFunc, 1) + tensor.AddFunc(SumSquares.FuncName(), SumSquaresFunc, 1) + tensor.AddFunc(Abs.FuncName(), AbsFunc, 1) + tensor.AddFunc(Hamming.FuncName(), HammingFunc, 1) + tensor.AddFunc(EuclideanBinTol.FuncName(), EuclideanBinTolFunc, 1) + tensor.AddFunc(SumSquaresBinTol.FuncName(), SumSquaresBinTolFunc, 1) + tensor.AddFunc(InvCosine.FuncName(), InvCosineFunc, 1) + tensor.AddFunc(InvCorrelation.FuncName(), InvCorrelationFunc, 1) + tensor.AddFunc(InnerProduct.FuncName(), InnerProductFunc, 1) + tensor.AddFunc(CrossEntropy.FuncName(), CrossEntropyFunc, 1) + tensor.AddFunc(Covariance.FuncName(), CovarianceFunc, 1) + tensor.AddFunc(Correlation.FuncName(), CorrelationFunc, 1) + tensor.AddFunc(Cosine.FuncName(), CosineFunc, 1) } // Metric calls a standard Metrics enum function on given tensors. // Output results are in the out tensor. func Metric(metric Metrics, a, b, out *tensor.Indexed) { - tensor.Call(metric.String(), a, b, out) + tensor.Call(metric.FuncName(), a, b, out) } // MetricOut calls a standard Metrics enum function on given tensors, // returning output as a newly created tensor. func MetricOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { - return errors.Log1(tensor.CallOut(metric.String(), a, b))[0] // note: error should never happen + return errors.Log1(tensor.CallOut(metric.FuncName(), a, b))[0] // note: error should never happen } // Metrics are standard metric functions @@ -107,6 +107,12 @@ const ( Cosine ) +// FuncName returns the package-qualified function name to use +// in tensor.Call to call this function. +func (m Metrics) FuncName() string { + return "metric." + m.String() +} + // Increasing returns true if the distance metric is such that metric // values increase as a function of distance (e.g., Euclidean) // and false if metric values decrease as a function of distance diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 63a27bf7c6..f95fe26da0 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -25,7 +25,7 @@ func init() { func ClosestRow(funcName string, probe, vocab, out *tensor.Indexed) { rows, _ := vocab.Tensor.RowCellSize() mi := -1 - mout := tensor.NewFloatScalar(0.0) + mout := tensor.NewFloat64Scalar(0.0) mind := math.MaxFloat64 pr1d := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) for ri := range rows { diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index e280f28b4e..9fdf032402 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -22,9 +22,9 @@ func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t nt := len(tsr) out := tsr[nt-1] osz := out.Tensor.Shape().Sizes - out1 = tensor.NewIndexed(tensor.NewFloat64(osz...)) - out2 = tensor.NewIndexed(tensor.NewFloat64(osz...)) - out3 = tensor.NewIndexed(tensor.NewFloat64(osz...)) + out1 = tensor.NewFloat64Indexed(osz...) + out2 = tensor.NewFloat64Indexed(osz...) + out3 = tensor.NewFloat64Indexed(osz...) tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2, out3) for idx := range n { diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index a8333f8955..f7bd6fc319 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -12,39 +12,27 @@ import ( //go:generate core generate func init() { - tensor.AddFunc(Count.String(), CountFunc, 1) - tensor.AddFunc(Sum.String(), SumFunc, 1) - tensor.AddFunc(SumAbs.String(), SumAbsFunc, 1) - tensor.AddFunc(L1Norm.String(), SumAbsFunc, 1) - tensor.AddFunc(Prod.String(), ProdFunc, 1) - tensor.AddFunc(Min.String(), MinFunc, 1) - tensor.AddFunc(Max.String(), MaxFunc, 1) - tensor.AddFunc(MinAbs.String(), MinAbsFunc, 1) - tensor.AddFunc(MaxAbs.String(), MaxAbsFunc, 1) - tensor.AddFunc(Mean.String(), MeanFunc, 1) - tensor.AddFunc(Var.String(), VarFunc, 1) - tensor.AddFunc(Std.String(), StdFunc, 1) - tensor.AddFunc(Sem.String(), SemFunc, 1) - tensor.AddFunc(SumSq.String(), SumSqFunc, 1) - tensor.AddFunc(L2Norm.String(), L2NormFunc, 1) - tensor.AddFunc(VarPop.String(), VarPopFunc, 1) - tensor.AddFunc(StdPop.String(), StdPopFunc, 1) - tensor.AddFunc(SemPop.String(), SemPopFunc, 1) - tensor.AddFunc(Median.String(), MedianFunc, 1) - tensor.AddFunc(Q1.String(), Q1Func, 1) - tensor.AddFunc(Q3.String(), Q3Func, 1) -} - -// Stat calls a standard Stats enum function on given tensors. -// Output results are in the out tensor. -func Stat(stat Stats, in, out *tensor.Indexed) { - tensor.Call(stat.String(), in, out) -} - -// StatOut calls a standard Stats enum function on given tensor, -// returning output as a newly created tensor. -func StatOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { - return errors.Log1(tensor.CallOut(stat.String(), in))[0] // note: error should never happen + tensor.AddFunc(Count.FuncName(), CountFunc, 1) + tensor.AddFunc(Sum.FuncName(), SumFunc, 1) + tensor.AddFunc(SumAbs.FuncName(), SumAbsFunc, 1) + tensor.AddFunc(L1Norm.FuncName(), SumAbsFunc, 1) + tensor.AddFunc(Prod.FuncName(), ProdFunc, 1) + tensor.AddFunc(Min.FuncName(), MinFunc, 1) + tensor.AddFunc(Max.FuncName(), MaxFunc, 1) + tensor.AddFunc(MinAbs.FuncName(), MinAbsFunc, 1) + tensor.AddFunc(MaxAbs.FuncName(), MaxAbsFunc, 1) + tensor.AddFunc(Mean.FuncName(), MeanFunc, 1) + tensor.AddFunc(Var.FuncName(), VarFunc, 1) + tensor.AddFunc(Std.FuncName(), StdFunc, 1) + tensor.AddFunc(Sem.FuncName(), SemFunc, 1) + tensor.AddFunc(SumSq.FuncName(), SumSqFunc, 1) + tensor.AddFunc(L2Norm.FuncName(), L2NormFunc, 1) + tensor.AddFunc(VarPop.FuncName(), VarPopFunc, 1) + tensor.AddFunc(StdPop.FuncName(), StdPopFunc, 1) + tensor.AddFunc(SemPop.FuncName(), SemPopFunc, 1) + tensor.AddFunc(Median.FuncName(), MedianFunc, 1) + tensor.AddFunc(Q1.FuncName(), Q1Func, 1) + tensor.AddFunc(Q3.FuncName(), Q3Func, 1) } // Stats is a list of different standard aggregation functions, which can be used @@ -115,3 +103,21 @@ const ( // Q3 third quartile = 75%ile value = .75 quantile value. Q3 ) + +// FuncName returns the package-qualified function name to use +// in tensor.Call to call this function. +func (s Stats) FuncName() string { + return "stats." + s.String() +} + +// Stat calls a standard Stats enum function on given tensors. +// Output results are in the out tensor. +func Stat(stat Stats, in, out *tensor.Indexed) { + tensor.Call(stat.FuncName(), in, out) +} + +// StatOut calls a standard Stats enum function on given tensor, +// returning output as a newly created tensor. +func StatOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { + return errors.Log1(tensor.CallOut(stat.FuncName(), in))[0] // note: error should never happen +} diff --git a/tensor/string.go b/tensor/string.go index 0f631992be..f66d5f520c 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" - "cogentcore.org/core/base/slicesx" "gonum.org/v1/gonum/mat" ) @@ -132,24 +131,6 @@ func (tsr *String) SetFloatRowCell(val float64, row, cell int) { tsr.Values[row*sz+cell] = Float64ToString(val) } -// Floats sets []float64 slice of all elements in the tensor -// (length is ensured to be sufficient). -// This can be used for all of the gonum/floats methods -// for basic math, gonum/stats, etc. -func (tsr *String) Floats(flt *[]float64) { - *flt = slicesx.SetLength(*flt, len(tsr.Values)) - for i, v := range tsr.Values { - (*flt)[i] = StringToFloat64(v) - } -} - -// SetFloats sets tensor values from a []float64 slice (copies values). -func (tsr *String) SetFloats(flt ...float64) { - for i, v := range flt { - tsr.Values[i] = Float64ToString(v) - } -} - // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. func (tsr *String) At(i, j int) float64 { @@ -196,7 +177,8 @@ func (tsr *String) Range() (min, max float64, minIndex, maxIndex int) { return } -// SetZeros is simple convenience function initialize all values to 0 +// SetZeros is a simple convenience function initialize all values to the +// zero value of the type (empty strings for string type). func (tsr *String) SetZeros() { for j := range tsr.Values { tsr.Values[j] = "" diff --git a/tensor/table/io.go b/tensor/table/io.go index 197745bdee..a2f30dcb8a 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -22,35 +22,6 @@ import ( "cogentcore.org/core/tensor" ) -// Delim are standard CSV delimiter options (Tab, Comma, Space) -type Delims int32 //enums:enum - -const ( - // Tab is the tab rune delimiter, for TSV tab separated values - Tab Delims = iota - - // Comma is the comma rune delimiter, for CSV comma separated values - Comma - - // Space is the space rune delimiter, for SSV space separated value - Space - - // Detect is used during reading a file -- reads the first line and detects tabs or commas - Detect -) - -func (dl Delims) Rune() rune { - switch dl { - case Tab: - return '\t' - case Comma: - return ',' - case Space: - return ' ' - } - return '\t' -} - const ( // Headers is passed to CSV methods for the headers arg, to use headers // that capture full type and tensor shape information. @@ -66,7 +37,7 @@ const ( // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (dt *Table) SaveCSV(filename core.Filename, delim Delims, headers bool) error { //types:add +func (dt *Table) SaveCSV(filename core.Filename, delim tensor.Delims, headers bool) error { //types:add fp, err := os.Create(string(filename)) defer fp.Close() if err != nil { @@ -85,7 +56,7 @@ func (dt *Table) SaveCSV(filename core.Filename, delim Delims, headers bool) err // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (ix *Indexed) SaveCSV(filename core.Filename, delim Delims, headers bool) error { //types:add +func (ix *Indexed) SaveCSV(filename core.Filename, delim tensor.Delims, headers bool) error { //types:add fp, err := os.Create(string(filename)) defer fp.Close() if err != nil { @@ -107,7 +78,7 @@ func (ix *Indexed) SaveCSV(filename core.Filename, delim Delims, headers bool) e // information for tensor type and dimensionality. // If the table DOES have existing columns, then those are used robustly // for whatever information fits from each row of the file. -func (dt *Table) OpenCSV(filename core.Filename, delim Delims) error { //types:add +func (dt *Table) OpenCSV(filename core.Filename, delim tensor.Delims) error { //types:add fp, err := os.Open(string(filename)) if err != nil { return errors.Log(err) @@ -117,7 +88,7 @@ func (dt *Table) OpenCSV(filename core.Filename, delim Delims) error { //types:a } // OpenFS is the version of [Table.OpenCSV] that uses an [fs.FS] filesystem. -func (dt *Table) OpenFS(fsys fs.FS, filename string, delim Delims) error { +func (dt *Table) OpenFS(fsys fs.FS, filename string, delim tensor.Delims) error { fp, err := fsys.Open(filename) if err != nil { return errors.Log(err) @@ -135,14 +106,14 @@ func (dt *Table) OpenFS(fsys fs.FS, filename string, delim Delims) error { // information for tensor type and dimensionality. // If the table DOES have existing columns, then those are used robustly // for whatever information fits from each row of the file. -func (ix *Indexed) OpenCSV(filename core.Filename, delim Delims) error { //types:add +func (ix *Indexed) OpenCSV(filename core.Filename, delim tensor.Delims) error { //types:add err := ix.Table.OpenCSV(filename, delim) ix.Sequential() return err } // OpenFS is the version of [Indexed.OpenCSV] that uses an [fs.FS] filesystem. -func (ix *Indexed) OpenFS(fsys fs.FS, filename string, delim Delims) error { +func (ix *Indexed) OpenFS(fsys fs.FS, filename string, delim tensor.Delims) error { err := ix.Table.OpenFS(fsys, filename, delim) ix.Sequential() return err @@ -157,7 +128,7 @@ func (ix *Indexed) OpenFS(fsys fs.FS, filename string, delim Delims) error { // information for tensor type and dimensionality. // If the table DOES have existing columns, then those are used robustly // for whatever information fits from each row of the file. -func (dt *Table) ReadCSV(r io.Reader, delim Delims) error { +func (dt *Table) ReadCSV(r io.Reader, delim tensor.Delims) error { cr := csv.NewReader(r) cr.Comma = delim.Rune() rec, err := cr.ReadAll() // todo: lazy, avoid resizing @@ -393,7 +364,7 @@ func InferDataType(str string) reflect.Kind { // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (dt *Table) WriteCSV(w io.Writer, delim Delims, headers bool) error { +func (dt *Table) WriteCSV(w io.Writer, delim tensor.Delims, headers bool) error { ncol := 0 var err error if headers { @@ -422,7 +393,7 @@ func (dt *Table) WriteCSV(w io.Writer, delim Delims, headers bool) error { // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (ix *Indexed) WriteCSV(w io.Writer, delim Delims, headers bool) error { +func (ix *Indexed) WriteCSV(w io.Writer, delim tensor.Delims, headers bool) error { ncol := 0 var err error if headers { @@ -449,7 +420,7 @@ func (ix *Indexed) WriteCSV(w io.Writer, delim Delims, headers bool) error { // WriteCSVHeaders writes headers to a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg). // Returns number of columns in header -func (dt *Table) WriteCSVHeaders(w io.Writer, delim Delims) (int, error) { +func (dt *Table) WriteCSVHeaders(w io.Writer, delim tensor.Delims) (int, error) { cw := csv.NewWriter(w) cw.Comma = delim.Rune() hdrs := dt.TableHeaders() @@ -464,7 +435,7 @@ func (dt *Table) WriteCSVHeaders(w io.Writer, delim Delims) (int, error) { // WriteCSVRow writes given row to a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg) -func (dt *Table) WriteCSVRow(w io.Writer, row int, delim Delims) error { +func (dt *Table) WriteCSVRow(w io.Writer, row int, delim tensor.Delims) error { cw := csv.NewWriter(w) cw.Comma = delim.Rune() err := dt.WriteCSVRowWriter(cw, row, 0) diff --git a/tensor/table/typegen.go b/tensor/table/typegen.go index 3a020a134b..eb0a757516 100644 --- a/tensor/table/typegen.go +++ b/tensor/table/typegen.go @@ -6,6 +6,6 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Indexed", IDName: "index-view", Doc: "Indexed is an indexed wrapper around an table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}, {Name: "lessFunc", Doc: "current Less function used in sorting"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "IndexedColumnName", Doc: "IndexedColumnName returns a tensor.Indexed view of given column name,\nusing our current indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column"}, Returns: []string{"Indexed", "error"}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of data, with columns of tensors,\neach with the same number of Rows (outermost dimension).", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to each of the columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns\nif rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "columns of data, as tensor.Tensor tensors"}, {Name: "ColumnNames", Doc: "the names of the columns"}, {Name: "Rows", Doc: "number of rows, which is enforced to be the size of the outermost dimension of the column tensors"}, {Name: "ColumnNameMap", Doc: "the map of column names to column numbers"}, {Name: "MetaData", Doc: "misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column"}}}) diff --git a/tensor/tensor.go b/tensor/tensor.go index e711e66b97..8d7a1e4957 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -35,7 +35,7 @@ type Tensor interface { Shape() *Shape // SetShape sets the sizes parameters of the tensor, and resizes - // backing storage appropriately. + // backing storage appropriately, retaining all existing data that fits. SetShape(sizes ...int) // SetNames sets the dimension names of the tensor shape. @@ -48,7 +48,7 @@ type Tensor interface { // NumDims returns the total number of dimensions. NumDims() int - // DimSize returns size of given dimension + // DimSize returns size of given dimension. DimSize(dim int) int // RowCellSize returns the size of the outermost Row shape dimension, @@ -62,7 +62,7 @@ type Tensor interface { DataType() reflect.Kind // Sizeof returns the number of bytes contained in the Values of this tensor. - // for String types, this is just the string pointers. + // For String types, this is just the string pointers, not the string content. Sizeof() int64 // Bytes returns the underlying byte representation of the tensor values. @@ -70,23 +70,17 @@ type Tensor interface { // unintentionally modified or retained more than for immediate use. Bytes() []byte - // returns true if the data type is a String. otherwise is numeric. + // IsString returns true if the data type is a String; otherwise it is numeric. IsString() bool - // Float returns the value of given index as a float64. + // Float returns the value of given n-dimensional index (matching Shape) as a float64. Float(i ...int) float64 - // SetFloat sets the value of given index as a float64. + // SetFloat sets the value of given n-dimensional index (matching Shape) as a float64. SetFloat(val float64, i ...int) // NOTE: String conflicts with [fmt.Stringer], so we have to use StringValue - // StringValue returns the value of given index as a string. - StringValue(i ...int) string - - // SetString sets the value of given index as a string. - SetString(val string, i ...int) - // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. Float1D(i int) float64 @@ -94,50 +88,55 @@ type Tensor interface { SetFloat1D(val float64, i int) // FloatRowCell returns the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. For Table columns. + // and cell is 1D index into remaining inner dims. This is useful for lists of + // patterns, and the [table.Table] container. [Indexed] tensors index along the row, + // and use this interface extensively. FloatRowCell(row, cell int) float64 // SetFloatRowCell sets the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. For Table columns. + // and cell is 1D index into remaining inner dims. This is useful for lists of + // patterns, and the [table.Table] container. [Indexed] tensors index along the row, + // and use this interface extensively. SetFloatRowCell(val float64, row, cell int) - // Floats sets []float64 slice of all elements in the tensor - // (length is ensured to be sufficient). - // This can be used for all of the gonum/floats methods - // for basic math, gonum/stats, etc. - Floats(flt *[]float64) + // StringValue returns the value of given n-dimensional index (matching Shape) as a string. + StringValue(i ...int) string - // SetFloats sets tensor values from a []float64 slice (copies values). - SetFloats(vals ...float64) + // SetString sets the value of given n-dimensional index (matching Shape) as a string. + SetString(val string, i ...int) - // String1D returns the value of given 1-dimensional index (0-Len()-1) as a string + // String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. String1D(i int) string - // SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string + // SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. SetString1D(val string, i int) // StringRowCell returns the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. For Table columns. + // and cell is 1D index into remaining inner dims. [Indexed] tensors index along the row, + // and use this interface extensively. StringRowCell(row, cell int) string // SetStringRowCell sets the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. For Table columns. + // and cell is 1D index into remaining inner dims. [Indexed] tensors index along the row, + // and use this interface extensively. SetStringRowCell(val string, row, cell int) // SubSpace returns a new tensor with innermost subspace at given - // offset(s) in outermost dimension(s) (len(offs) < NumDims). + // offset(s) in outermost dimension(s) (len(offs) < [NumDims]). // The new tensor points to the values of the this tensor (i.e., modifications // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). - // Use Clone() method to separate the two. + // Use Clone() method to separate the two. See [Slice] function to + // extract arbitrary subspaces along ranges of each dimension. SubSpace(offs ...int) Tensor // Range returns the min, max (and associated indexes, -1 = no values) for the tensor. - // This is needed for display and is thus in the core api in optimized form - // Other math operations can be done using gonum/floats package. + // This is needed for display and is thus in the core api in optimized form. + // See the [stats] package for many more statistics functions. Range() (min, max float64, minIndex, maxIndex int) - // SetZeros is simple convenience function initialize all values to 0. + // SetZeros is a simple convenience function initialize all values to the + // zero value of the type (empty strings for string type). SetZeros() // Clone clones this tensor, creating a duplicate copy of itself with its @@ -146,14 +145,14 @@ type Tensor interface { Clone() Tensor // View clones this tensor, *keeping the same underlying Values slice*, - // instead of making a copy like Clone() does. The main point of this + // instead of making a copy like Clone() does. A major use of this // is to then change the shape of the view to provide a different way // of accessing the same data. See [New1DViewOf] for example. View() Tensor - // CopyFrom copies all avail values from other tensor into this tensor, with an + // CopyFrom copies all values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and - // otherwise it goes through appropriate standard type. + // otherwise it goes through the appropriate standard type (Float or String). CopyFrom(from Tensor) // SetShapeFrom sets our shape from given source tensor, calling diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 7083d81f89..9904362bbd 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -678,7 +678,7 @@ func (tb *Table) CopySelectToMime() mimedata.Mimes { } ix.Indexes = iidx var b bytes.Buffer - ix.WriteCSV(&b, table.Tab, table.Headers) + ix.WriteCSV(&b, tensor.Tab, table.Headers) md := mimedata.NewTextBytes(b.Bytes()) md[0].Type = fileinfo.DataCsv return md @@ -691,7 +691,7 @@ func (tb *Table) FromMimeData(md mimedata.Mimes) [][]string { if d.Type == fileinfo.DataCsv { b := bytes.NewBuffer(d.Data) cr := csv.NewReader(b) - cr.Comma = table.Tab.Rune() + cr.Comma = tensor.Tab.Rune() rec, err := cr.ReadAll() if err != nil || len(rec) == 0 { log.Printf("Error reading CSV from clipboard: %s\n", err) diff --git a/tensor/tensorcore/tensoreditor.go b/tensor/tensorcore/tensoreditor.go index 26af50675a..7c57bea909 100644 --- a/tensor/tensorcore/tensoreditor.go +++ b/tensor/tensorcore/tensoreditor.go @@ -409,7 +409,7 @@ func (tb *TensorEditor) CopySelectToMime() mimedata.Mimes { } // idx := tb.SelectedIndexesList(false) // ascending // var b bytes.Buffer - // ix.WriteCSV(&b, table.Tab, table.Headers) + // ix.WriteCSV(&b, tensor.Tab, table.Headers) // md := mimedata.NewTextBytes(b.Bytes()) // md[0].Type = fileinfo.DataCsv // return md @@ -423,7 +423,7 @@ func (tb *TensorEditor) FromMimeData(md mimedata.Mimes) [][]string { if d.Type == fileinfo.DataCsv { // b := bytes.NewBuffer(d.Data) // cr := csv.NewReader(b) - // cr.Comma = table.Tab.Rune() + // cr.Comma = tensor.Tab.Rune() // rec, err := cr.ReadAll() // if err != nil || len(rec) == 0 { // log.Printf("Error reading CSV from clipboard: %s\n", err) diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index f239364f27..ac9b0029db 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -27,7 +27,7 @@ func Equal(t *testing.T, trg, val float64) { } func TestMath(t *testing.T) { - scalar := tensor.NewFloatScalar(-5.5) + scalar := tensor.NewFloat64Scalar(-5.5) scout := scalar.Clone() vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index d7e2e09ede..2f0c76f945 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -13,7 +13,7 @@ import ( ) func TestAdd(t *testing.T) { - scalar := tensor.NewFloatScalar(-5.5) + scalar := tensor.NewFloat64Scalar(-5.5) scb := scalar.Clone() scb.Tensor.SetFloat1D(-4.0, 0) scout := scalar.Clone() @@ -140,8 +140,8 @@ func TestAdd(t *testing.T) { assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) // fmt.Println(oneout.Tensor) - minv := tensor.NewFloatScalar(0) - maxv := tensor.NewFloatScalar(1) + minv := tensor.NewFloat64Scalar(0) + maxv := tensor.NewFloat64Scalar(1) Clamp(oned, minv, maxv, oneout) stats.MinFunc(oneout, mout) assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) @@ -149,7 +149,7 @@ func TestAdd(t *testing.T) { assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) // fmt.Println(oneout.Tensor) - thr := tensor.NewFloatScalar(0.5) + thr := tensor.NewFloat64Scalar(0.5) Binarize(oned, thr, oneout) stats.MinFunc(oneout, mout) assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) diff --git a/tensor/typegen.go b/tensor/typegen.go index 9e76dee8dc..e6bc08a311 100644 --- a/tensor/typegen.go +++ b/tensor/typegen.go @@ -6,4 +6,4 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a tensor.Tensor that provides a\nspecific view onto the Tensor defined by the set of indexes, which\napply to the outermost row dimension (with default row-major indexing).\nThis provides an efficient way of sorting and filtering a tensor by only\nupdating the indexes while doing nothing to the Tensor itself.\nTo produce a tensor that has data actually organized according to the\nindexed order, call the NewTensor method.\nUse the [Set]FloatRowCell methods wherever possible, for the most efficient\nand natural indirection through the indexes. The 1D methods on underlying\ntensor data do not indirect through the indexes and must be called directly\non the [Tensor].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a tensor.Tensor that provides a\nspecific view onto the Tensor defined by the set of indexes, which\napply to the outermost row dimension (with default row-major indexing).\nThis is the universal representation of a homogenous data type in the\n[tensor] package framework, from scalar to vector, matrix, and beyond,\nbecause it can efficiently represent any kind of element with sufficient\nflexibility to enable a full range of computations to be elegantly expressed.\nFor example, sorting and filtering a tensor only requires\nupdating the indexes while doing nothing to the Tensor itself.\nTo produce a new [Tensor] that has its raw data actually organized according\nto the indexed order, call the [NewTensor] method.\nUse the [Set]FloatRowCell methods wherever possible, for the most efficient\nand natural indirection through the indexes. The 1D methods on underlying\ntensor data do not indirect through the indexes and must be called directly\non the [Tensor].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "IndexesNeeded", Doc: "IndexesNeeded is called prior to an operation that needs actual indexes,\ne.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise\ncurrent indexes are left as is. Use Sequential, then IndexesNeeded to ensure\nall rows are represented.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing1D", Doc: "ExcludeMissing1D deletes indexes for a 1D tensor (only) where\nthe values are missing, as indicated by NaN.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows.\nOnly set if order is different from default sequential order.\nUse the Index() method for nil-aware logic."}}}) From dd0f26e48b25db6c69cfb74e5fc2b21c36342ef4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 13 Sep 2024 13:54:00 -0700 Subject: [PATCH 029/311] fixed test to use actual eigenvectors.. --- tensor/stats/metric/metric_test.go | 33 ++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 89d2bffe96..20d6e74546 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -130,8 +130,21 @@ func TestPCAIris(t *testing.T) { colidx := tensor.NewFloat64Scalar(3) // strongest at end prjns := tensor.NewFloat64Indexed() - ProjectOnMatrixColumn(covar, data, colidx, prjns) - tensor.SaveCSV(prjns.Tensor, "testdata/pca_projection.csv", tensor.Comma) + ProjectOnMatrixColumn(vecs, data, colidx, prjns) + // tensor.SaveCSV(prjns.Tensor, "testdata/pca_projection.csv", tensor.Comma) + trgprjns := []float64{ + 2.6692308782935146, + 2.696434011868953, + 2.4811633041648684, + 2.5715124347750256, + 2.5906582247213543, + 3.0080988099460613, + 2.490941664609344, + 2.7014546083439073, + 2.4615836931965167, + 2.6716628159090594, + } + assert.Equal(t, trgprjns, tensor.AsFloat64(prjns.Tensor).Values[:10]) //////////////////////////////////////////////////////////// // SVD @@ -143,4 +156,20 @@ func TestPCAIris(t *testing.T) { assert.InDelta(t, corvals[3-i], v, errtol) // opposite order } + colidx.Tensor.SetFloat1D(0, 0) // strongest at start + ProjectOnMatrixColumn(vecs, data, colidx, prjns) + tensor.SaveCSV(prjns.Tensor, "testdata/svd_projection.csv", tensor.Comma) + trgprjns = []float64{ + -2.6692308782935172, + -2.696434011868955, + -2.48116330416487, + -2.5715124347750273, + -2.590658224721357, + -3.008098809946064, + -2.4909416646093456, + -2.70145460834391, + -2.4615836931965185, + -2.671662815909061, + } + assert.Equal(t, trgprjns, tensor.AsFloat64(prjns.Tensor).Values[:10]) } From 71be05ec35b85a5d1bfdff7b0ec5b8b091d5340a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 13 Sep 2024 15:26:01 -0700 Subject: [PATCH 030/311] keylist as more convenient but slower alternative to ordmap -- using for tensor table. --- base/keylist/README.md | 9 + base/keylist/keylist.go | 225 +++++++++++++++++++++++ base/keylist/keylist_test.go | 39 ++++ base/ordmap/ordmap.go | 1 - tensor/list.go | 1 + tensor/stats/histogram/histogram.go | 50 +---- tensor/stats/histogram/histogram_test.go | 39 +--- tensor/stats/metric/metric_test.go | 2 +- 8 files changed, 287 insertions(+), 79 deletions(-) create mode 100644 base/keylist/README.md create mode 100644 base/keylist/keylist.go create mode 100644 base/keylist/keylist_test.go create mode 100644 tensor/list.go diff --git a/base/keylist/README.md b/base/keylist/README.md new file mode 100644 index 0000000000..8bc40bcc83 --- /dev/null +++ b/base/keylist/README.md @@ -0,0 +1,9 @@ +# keylist + +keylist implements an ordered list (slice) of items, with a map from a key (e.g., names) to indexes, to support fast lookup by name. + +Compared to the [ordmap](../ordmap) package, this is not as efficient for operations such as deletion and insertion, but it has the advantage of providing a simple slice of the target items that can be used directly in many cases. + +Thus, it is more suitable for largely static lists, which are constructed by adding items to the end of the list. + + diff --git a/base/keylist/keylist.go b/base/keylist/keylist.go new file mode 100644 index 0000000000..377cb87322 --- /dev/null +++ b/base/keylist/keylist.go @@ -0,0 +1,225 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +package keylist implements an ordered list (slice) of items, +with a map from a key (e.g., names) to indexes, +to support fast lookup by name. +Compared to the [ordmap] package, this is not as efficient +for operations such as deletion and insertion, but it +has the advantage of providing a simple slice of the target +items that can be used directly in many cases. +Thus, it is more suitable for largely static lists, which +are constructed by adding items to the end of the list. +*/ +package keylist + +import ( + "fmt" + "slices" +) + +// List implements an ordered list (slice) of items, +// with a map from a key (e.g., names) to indexes, +// to support fast lookup by name. +type List[K comparable, V any] struct { //types:add + // List is the ordered slice of items. + List []V + + // indexes is the key-to-index mapping. + indexes map[K]int +} + +// New returns a new Key List. Zero value +// is usable without initialization, so this is +// just a simple standard convenience method. +func New[K comparable, V any]() *List[K, V] { + return &List[K, V]{} +} + +func (kl *List[K, V]) newIndexes() { + kl.indexes = make(map[K]int) +} + +// initIndexes ensures that the index map exists. +func (kl *List[K, V]) initIndexes() { + if kl.indexes == nil { + kl.newIndexes() + } +} + +// Reset resets the list, removing any existing elements. +func (kl *List[K, V]) Reset() { + kl.List = nil + kl.newIndexes() +} + +// Keys returns the list of keys in List sequential order. +func (kl *List[K, V]) Keys() []K { + keys := make([]K, len(kl.indexes)) + for k, i := range kl.indexes { + keys[i] = k + } + return keys +} + +// Add adds an item to the list with given key. +// An error is returned if the key is already on the list. +// See [AddReplace] for a method that automatically replaces. +func (kl *List[K, V]) Add(key K, val V) error { + kl.initIndexes() + if _, ok := kl.indexes[key]; ok { + return fmt.Errorf("keylist.Add: key %v is already on the list", key) + } + kl.indexes[key] = len(kl.List) + kl.List = append(kl.List, val) + return nil +} + +// AddReplace adds an item to the list with given key, +// replacing any existing item with the same key. +func (kl *List[K, V]) AddReplace(key K, val V) { + kl.initIndexes() + if idx, ok := kl.indexes[key]; ok { + kl.List[idx] = val + return + } + kl.indexes[key] = len(kl.List) + kl.List = append(kl.List, val) +} + +// InsertAtIndex inserts the given value with the given key at the given index. +// This is relatively slow because it needs regenerate the keys list. +// It returns an error if the key already exists because +// the behavior is undefined in that situation. +func (kl *List[K, V]) InsertAtIndex(idx int, key K, val V) error { + if _, has := kl.indexes[key]; has { + return fmt.Errorf("keylist.Add: key %v is already on the list", key) + } + + keys := kl.Keys() + keys = slices.Insert(keys, idx, key) + kl.List = slices.Insert(kl.List, idx, val) + kl.newIndexes() + for i, k := range keys { + kl.indexes[k] = i + } + return nil +} + +// ValueByKey returns the value corresponding to the given key, +// with a zero value returned for a missing key. See [List.ValueByKeyTry] +// for one that returns a bool for missing keys. +func (kl *List[K, V]) ValueByKey(key K) V { + idx, ok := kl.indexes[key] + if ok { + return kl.List[idx] + } + var zv V + return zv +} + +// ValueByKeyTry returns the value corresponding to the given key, +// with false returned for a missing key, in case the zero value +// is not diagnostic. +func (kl *List[K, V]) ValueByKeyTry(key K) (V, bool) { + idx, ok := kl.indexes[key] + if ok { + return kl.List[idx], true + } + var zv V + return zv, false +} + +// IndexIsValid returns an error if the given index is invalid +func (kl *List[K, V]) IndexIsValid(idx int) error { + if idx >= len(kl.List) || idx < 0 { + return fmt.Errorf("keylist.List: IndexIsValid: index %d is out of range of a list of length %d", idx, len(kl.List)) + } + return nil +} + +// IndexByKey returns the index of the given key, with a -1 for missing key. +func (kl *List[K, V]) IndexByKey(key K) int { + idx, ok := kl.indexes[key] + if !ok { + return -1 + } + return idx +} + +// Len returns the number of items in the list. +func (kl *List[K, V]) Len() int { + if kl == nil { + return 0 + } + return len(kl.List) +} + +// DeleteIndex deletes item(s) within the index range [i:j]. +// This is relatively slow because it needs to regenerate the +// index map. +func (kl *List[K, V]) DeleteIndex(i, j int) { + ndel := j - i + if ndel <= 0 { + panic("index range is <= 0") + } + keys := kl.Keys() + keys = slices.Delete(keys, i, j) + kl.List = slices.Delete(kl.List, i, j) + kl.newIndexes() + for i, k := range keys { + kl.indexes[k] = i + } + +} + +// DeleteKey deletes the item with the given key, +// returning false if it does not find it. +// This is relatively slow because it needs to regenerate the +// index map. +func (kl *List[K, V]) DeleteKey(key K) bool { + idx, ok := kl.indexes[key] + if !ok { + return false + } + kl.DeleteIndex(idx, idx+1) + return true +} + +// Copy copies all of the entries from the given keyed list +// into this list. It keeps existing entries in this +// list unless they also exist in the given list, in which case +// they are overwritten. Use [Reset] first to get an exact copy. +func (kl *List[K, V]) Copy(from *List[K, V]) { + keys := from.Keys() + for i, v := range from.List { + kl.AddReplace(keys[i], v) + } +} + +// String returns a string representation of the list. +func (kl *List[K, V]) String() string { + sv := "{" + keys := kl.Keys() + for i, v := range kl.List { + sv += fmt.Sprintf("%v", keys[i]) + ": " + fmt.Sprintf("%v", v) + ", " + } + sv += "}" + return sv +} + +/* +// GoString returns the list as Go code. +func (kl *List[K, V]) GoString() string { + var zk K + var zv V + res := fmt.Sprintf("ordlist.Make([]ordlist.KeyVal[%T, %T]{\n", zk, zv) + for _, kv := range kl.Order { + res += fmt.Sprintf("{%#v, %#v},\n", kv.Key, kv.Value) + } + res += "})" + return res +} +*/ diff --git a/base/keylist/keylist_test.go b/base/keylist/keylist_test.go new file mode 100644 index 0000000000..ea3fddaa30 --- /dev/null +++ b/base/keylist/keylist_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package keylist + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestKeyList(t *testing.T) { + om := New[string, int]() + om.Add("key0", 0) + om.Add("key1", 1) + om.Add("key2", 2) + + assert.Equal(t, 1, om.ValueByKey("key1")) + assert.Equal(t, 2, om.IndexByKey("key2")) + + assert.Equal(t, 1, om.List[1]) + + assert.Equal(t, 3, om.Len()) + + om.DeleteIndex(1, 2) + assert.Equal(t, 2, om.List[1]) + assert.Equal(t, 1, om.IndexByKey("key2")) + + om.InsertAtIndex(0, "new0", 3) + assert.Equal(t, 3, om.List[0]) + assert.Equal(t, 0, om.List[1]) + assert.Equal(t, 2, om.IndexByKey("key2")) + + // nm := Make([]KeyValue[string, int]{{"one", 1}, {"two", 2}, {"three", 3}}) + // assert.Equal(t, 3, nm.List[2]) + // assert.Equal(t, 2, nm.List[1]) + // assert.Equal(t, 3, nm.ValueByKey("three")) +} diff --git a/base/ordmap/ordmap.go b/base/ordmap/ordmap.go index 9c66951ed4..f459ad88af 100644 --- a/base/ordmap/ordmap.go +++ b/base/ordmap/ordmap.go @@ -35,7 +35,6 @@ type KeyValue[K comparable, V any] struct { // and the fast key lookup of a map. A map stores an index // into a slice that has the value and key associated with the value. type Map[K comparable, V any] struct { - // Order is an ordered list of values and associated keys, in the order added. Order []KeyValue[K, V] diff --git a/tensor/list.go b/tensor/list.go new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/tensor/list.go @@ -0,0 +1 @@ + diff --git a/tensor/stats/histogram/histogram.go b/tensor/stats/histogram/histogram.go index 3cf7e73c3a..e8bee794ed 100644 --- a/tensor/stats/histogram/histogram.go +++ b/tensor/stats/histogram/histogram.go @@ -8,9 +8,6 @@ package histogram import ( "cogentcore.org/core/base/slicesx" - "cogentcore.org/core/math32" - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/table" ) // F64 generates a histogram of counts of values within given @@ -36,6 +33,8 @@ func F64(hist *[]float64, vals []float64, nBins int, min, max float64) { } } +/* + // F64Table generates an table with a histogram of counts of values within given // number of bins and min / max range. The table has columns: Value, Count // if value is < min or > max it is ignored. @@ -55,47 +54,4 @@ func F64Table(dt *table.Table, vals []float64, nBins int, min, max float64) { } } -////////////////////////////////////////////////////// -// float32 - -// F32 generates a histogram of counts of values within given -// number of bins and min / max range. hist vals is sized to nBins. -// if value is < min or > max it is ignored. -func F32(hist *[]float32, vals []float32, nBins int, min, max float32) { - *hist = slicesx.SetLength(*hist, nBins) - h := *hist - // 0.1.2.3 = 3-0 = 4 bins - inc := (max - min) / float32(nBins) - for i := 0; i < nBins; i++ { - h[i] = 0 - } - for _, v := range vals { - if v < min || v > max { - continue - } - bin := int((v - min) / inc) - if bin >= nBins { - bin = nBins - 1 - } - h[bin] += 1 - } -} - -// F32Table generates an table with a histogram of counts of values within given -// number of bins and min / max range. The table has columns: Value, Count -// if value is < min or > max it is ignored. -// The Value column represents the min value for each bin, with the max being -// the value of the next bin, or the max if at the end. -func F32Table(dt *table.Table, vals []float32, nBins int, min, max float32) { - dt.DeleteAll() - dt.AddFloat32Column("Value") - dt.AddFloat32Column("Count") - dt.SetNumRows(nBins) - ct := dt.Columns[1].(*tensor.Float32) - F32(&ct.Values, vals, nBins, min, max) - inc := (max - min) / float32(nBins) - vls := dt.Columns[0].(*tensor.Float32).Values - for i := 0; i < nBins; i++ { - vls[i] = math32.Truncate(min+float32(i)*inc, 4) - } -} +*/ diff --git a/tensor/stats/histogram/histogram_test.go b/tensor/stats/histogram/histogram_test.go index e2c31cee5e..c25452b06a 100644 --- a/tensor/stats/histogram/histogram_test.go +++ b/tensor/stats/histogram/histogram_test.go @@ -7,30 +7,9 @@ package histogram import ( "testing" - "cogentcore.org/core/tensor/table" "github.com/stretchr/testify/assert" ) -func TestHistogram32(t *testing.T) { - vals := []float32{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - ex := []float32{4, 3, 4} - res := []float32{} - - F32(&res, vals, 3, 0, 1) - - assert.Equal(t, ex, res) - - exvals := []float32{0, 0.3333, 0.6667} - dt := table.NewTable() - F32Table(dt, vals, 3, 0, 1) - for ri, v := range ex { - vv := float32(dt.Float("Value", ri)) - cv := float32(dt.Float("Count", ri)) - assert.Equal(t, exvals[ri], vv) - assert.Equal(t, v, cv) - } -} - func TestHistogram64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} ex := []float64{4, 3, 4} @@ -40,13 +19,13 @@ func TestHistogram64(t *testing.T) { assert.Equal(t, ex, res) - exvals := []float64{0, 0.3333, 0.6667} - dt := table.NewTable() - F64Table(dt, vals, 3, 0, 1) - for ri, v := range ex { - vv := dt.Float("Value", ri) - cv := dt.Float("Count", ri) - assert.Equal(t, exvals[ri], vv) - assert.Equal(t, v, cv) - } + // exvals := []float64{0, 0.3333, 0.6667} + // dt := table.NewTable() + // F64Table(dt, vals, 3, 0, 1) + // for ri, v := range ex { + // vv := dt.Float("Value", ri) + // cv := dt.Float("Count", ri) + // assert.Equal(t, exvals[ri], vv) + // assert.Equal(t, v, cv) + // } } diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 20d6e74546..c77e276310 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -158,7 +158,7 @@ func TestPCAIris(t *testing.T) { colidx.Tensor.SetFloat1D(0, 0) // strongest at start ProjectOnMatrixColumn(vecs, data, colidx, prjns) - tensor.SaveCSV(prjns.Tensor, "testdata/svd_projection.csv", tensor.Comma) + // tensor.SaveCSV(prjns.Tensor, "testdata/svd_projection.csv", tensor.Comma) trgprjns = []float64{ -2.6692308782935172, -2.696434011868955, From 9a293add4f13eae1f0be413437c63778e5268089 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 14 Sep 2024 03:06:37 -0700 Subject: [PATCH 031/311] progress updating table --- base/keylist/keylist.go | 6 +- base/keylist/keylist_test.go | 2 +- tensor/README.md | 15 +- tensor/bits.go | 24 ++ tensor/indexed.go | 13 +- tensor/list.go | 1 - tensor/number.go | 24 ++ tensor/slice.go | 29 ++ tensor/stats/glm/glm.go | 8 +- tensor/stats/metric/metric_test.go | 4 +- tensor/string.go | 24 ++ tensor/table/README.md | 8 +- tensor/table/columns.go | 96 +++++++ tensor/table/indexed.go | 381 +++++++++---------------- tensor/table/io.go | 104 ++----- tensor/table/splits.go | 51 ++-- tensor/table/table.go | 433 +++++++++++------------------ tensor/table/table_test.go | 2 +- tensor/tensor.go | 18 +- 19 files changed, 598 insertions(+), 645 deletions(-) delete mode 100644 tensor/list.go create mode 100644 tensor/table/columns.go diff --git a/base/keylist/keylist.go b/base/keylist/keylist.go index 377cb87322..fb5e510aa3 100644 --- a/base/keylist/keylist.go +++ b/base/keylist/keylist.go @@ -20,6 +20,8 @@ import ( "slices" ) +// TODO: probably want to consolidate ordmap and keylist + // List implements an ordered list (slice) of items, // with a map from a key (e.g., names) to indexes, // to support fast lookup by name. @@ -89,11 +91,11 @@ func (kl *List[K, V]) AddReplace(key K, val V) { kl.List = append(kl.List, val) } -// InsertAtIndex inserts the given value with the given key at the given index. +// Insert inserts the given value with the given key at the given index. // This is relatively slow because it needs regenerate the keys list. // It returns an error if the key already exists because // the behavior is undefined in that situation. -func (kl *List[K, V]) InsertAtIndex(idx int, key K, val V) error { +func (kl *List[K, V]) Insert(idx int, key K, val V) error { if _, has := kl.indexes[key]; has { return fmt.Errorf("keylist.Add: key %v is already on the list", key) } diff --git a/base/keylist/keylist_test.go b/base/keylist/keylist_test.go index ea3fddaa30..e5c1a3e9e1 100644 --- a/base/keylist/keylist_test.go +++ b/base/keylist/keylist_test.go @@ -27,7 +27,7 @@ func TestKeyList(t *testing.T) { assert.Equal(t, 2, om.List[1]) assert.Equal(t, 1, om.IndexByKey("key2")) - om.InsertAtIndex(0, "new0", 3) + om.Insert(0, "new0", 3) assert.Equal(t, 3, om.List[0]) assert.Equal(t, 0, om.List[1]) assert.Equal(t, 2, om.IndexByKey("key2")) diff --git a/tensor/README.md b/tensor/README.md index 7f6d5ac0bf..1f6d9966b7 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -14,7 +14,7 @@ The `Vectorize` function and its variants provide a universal "apply function to All tensor package functions are registered using a single name to function map (`Funcs`). -* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. The `table` package also has an `Indexed` that provides a shared index for the column tensors. +* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including [plot/plotcore](../plot/plotcore). @@ -35,6 +35,19 @@ All tensor package functions are registered using a single name to function map - TODO: where? [pca](pca) provides principal-components-analysis (PCA) and covariance matrix computation functions. - TODO: in metric? [clust](clust) provides standard agglomerative hierarchical clustering including ability to plot results in an eplot. +# Standard shapes + +There are various standard shapes of tensor data that different functions expect: + +* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one row-wise dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data (indexes are only available for the outermost row dimension). + +* **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Indexed` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In most packages including [tmath](tmath) and [stats](stats), 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. + + A higher-dimensional tensor can also be re-shaped into this row, cell format by collapsing any number of additional outer dimensions into a longer, effective "row" index, with the remaining inner dimensions forming the cell-wise patterns. You can decide where to make the cut, and the `RowCellSplit` function makes it easy to create a new view of an existing tensor with this split made at a given dimension. + +* **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations. + + # History diff --git a/tensor/bits.go b/tensor/bits.go index 6a416b5926..43f8842803 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -249,6 +249,30 @@ func (tsr *Bits) CopyFrom(frm Tensor) { } } +// AppendFrom appends values from other tensor into this tensor, +// which must have the same cell size as this tensor. +// It uses and optimized implementation if the other tensor +// is of the same type, and otherwise it goes through +// appropriate standard type. +func (tsr *Bits) AppendFrom(frm Tensor) error { + rows, cell := tsr.RowCellSize() + frows, fcell := frm.RowCellSize() + if cell != fcell { + return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) + } + tsr.SetNumRows(rows + frows) + st := rows * cell + fsz := frows * fcell + if fsm, ok := frm.(*Bits); ok { + copy(tsr.Values[st:st+fsz], fsm.Values) + return nil + } + for i := 0; i < fsz; i++ { + tsr.Values.Set(Float64ToBool(frm.Float1D(i)), st+i) + } + return nil +} + // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). func (tsr *Bits) SetShapeFrom(frm Tensor) { diff --git a/tensor/indexed.go b/tensor/indexed.go index 60f46861a3..1d3a5be3a3 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -33,7 +33,7 @@ type Indexed struct { //types:add // Tensor that we are an indexed view onto. Tensor Tensor - // Indexes are the indexes into Tensor rows. + // Indexes are the indexes into Tensor rows, with nil = sequential. // Only set if order is different from default sequential order. // Use the Index() method for nil-aware logic. Indexes []int @@ -395,14 +395,13 @@ func (ix *Indexed) AddRows(n int) { //types:add // given index in this view func (ix *Indexed) InsertRows(at, n int) { stidx := ix.Tensor.DimSize(0) + ix.IndexesNeeded() ix.Tensor.SetNumRows(stidx + n) - if ix.Indexes != nil { - nw := make([]int, n, n+len(ix.Indexes)-at) - for i := 0; i < n; i++ { - nw[i] = stidx + i - } - ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) + nw := make([]int, n, n+len(ix.Indexes)-at) + for i := 0; i < n; i++ { + nw[i] = stidx + i } + ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) } // DeleteRows deletes n rows of indexes starting at given index in the list of indexes diff --git a/tensor/list.go b/tensor/list.go deleted file mode 100644 index 8b13789179..0000000000 --- a/tensor/list.go +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tensor/number.go b/tensor/number.go index 4911e85e94..070f0c95cd 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -276,6 +276,30 @@ func (tsr *Number[T]) CopyFrom(frm Tensor) { } } +// AppendFrom appends values from other tensor into this tensor, +// which must have the same cell size as this tensor. +// It uses and optimized implementation if the other tensor +// is of the same type, and otherwise it goes through +// appropriate standard type. +func (tsr *Number[T]) AppendFrom(frm Tensor) error { + rows, cell := tsr.RowCellSize() + frows, fcell := frm.RowCellSize() + if cell != fcell { + return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) + } + tsr.SetNumRows(rows + frows) + st := rows * cell + fsz := frows * fcell + if fsm, ok := frm.(*Number[T]); ok { + copy(tsr.Values[st:st+fsz], fsm.Values) + return nil + } + for i := 0; i < fsz; i++ { + tsr.Values[st+i] = T(frm.Float1D(i)) + } + return nil +} + // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). func (tsr *Number[T]) SetShapeFrom(frm Tensor) { diff --git a/tensor/slice.go b/tensor/slice.go index 36f2d78cde..a64e307dfb 100644 --- a/tensor/slice.go +++ b/tensor/slice.go @@ -74,6 +74,13 @@ func SliceSize(sizes []int, ranges ...Range) ([]int, error) { // efficient, relative to indexes, and it is simpler to only support one. // also, the need for direct shared access is limited. +// todo: make a version of these functions that takes +// a standard Indexed tensor with n x 3 shape, where the 3 inner values +// specify the Range Start, End, Incr values, across n ranges. +// these would convert to the current Range-based format that does the impl, +// using the Range helper functions, which are also easier and more explicit +// to use in Go code. + // Slice extracts a subset of values from the given tensor into the // output tensor, according to the provided ranges. // Dimensions beyond the ranges specified are automatically included. @@ -155,3 +162,25 @@ func SliceSet(tsr, slc *Indexed, ranges ...Range) error { } return nil } + +// RowCellSplit splits the given tensor into a standard 2D row, cell +// shape at the given split dimension index. All dimensions prior to +// split are collapsed into the row dimension, and from split onward +// form the cells dimension. The resulting tensor is a re-shaped view +// of the original tensor, sharing the same underlying data. +func RowCellSplit(tsr Tensor, split int) Tensor { + sizes := tsr.Shape().Sizes + rows := sizes[:split] + cells := sizes[split:] + nr := 1 + for _, r := range rows { + nr *= r + } + nc := 1 + for _, c := range cells { + nc *= c + } + vw := tsr.View() + vw.SetShape(nr, nc) + return vw +} diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index a625ce8599..8187c896f3 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -124,23 +124,23 @@ func (glm *GLM) init(nIv, nDv int) { // predVars and errVars (predicted values and error values) are optional. func (glm *GLM) SetTable(ix *table.Indexed, indepVars, depVars, predVars, errVars string) error { dt := ix.Table - iv, err := dt.ColumnByName(indepVars) + iv, err := dt.Column(indepVars) if err != nil { return err } - dv, err := dt.ColumnByName(depVars) + dv, err := dt.Column(depVars) if err != nil { return err } var pv, ev tensor.Tensor if predVars != "" { - pv, err = dt.ColumnByName(predVars) + pv, err = dt.Column(predVars) if err != nil { return err } } if errVars != "" { - ev, err = dt.ColumnByName(errVars) + ev, err = dt.Column(errVars) if err != nil { return err } diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index c77e276310..436b5e2886 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -91,7 +91,7 @@ func TestMatrix(t *testing.T) { err := dt.OpenCSV("../cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) // smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) - in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) + in := tensor.NewIndexed(errors.Log1(dt.Column("Input"))) out := tensor.NewFloat64Indexed() Matrix(Euclidean.FuncName(), in, out) // fmt.Println(out.Tensor) @@ -111,7 +111,7 @@ func TestPCAIris(t *testing.T) { } // pc.TableColumn(ix, "data", metric.Covariance64) // fmt.Printf("covar: %v\n", pc.Covar) - data := tensor.NewIndexed(errors.Log1(dt.ColumnByName("data"))) + data := tensor.NewIndexed(errors.Log1(dt.Column("data"))) covar := tensor.NewFloat64Indexed() CovarMatrix(Correlation.FuncName(), data, covar) // fmt.Printf("correl: %s\n", covar.Tensor.String()) diff --git a/tensor/string.go b/tensor/string.go index f66d5f520c..05b08d780c 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -212,6 +212,30 @@ func (tsr *String) CopyFrom(frm Tensor) { } } +// AppendFrom appends values from other tensor into this tensor, +// which must have the same cell size as this tensor. +// It uses and optimized implementation if the other tensor +// is of the same type, and otherwise it goes through +// appropriate standard type. +func (tsr *String) AppendFrom(frm Tensor) error { + rows, cell := tsr.RowCellSize() + frows, fcell := frm.RowCellSize() + if cell != fcell { + return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) + } + tsr.SetNumRows(rows + frows) + st := rows * cell + fsz := frows * fcell + if fsm, ok := frm.(*String); ok { + copy(tsr.Values[st:st+fsz], fsm.Values) + return nil + } + for i := 0; i < fsz; i++ { + tsr.Values[st+i] = Float64ToString(frm.Float1D(i)) + } + return nil +} + // SetShapeFrom copies just the shape from given source tensor // calling SetShape with the shape params from source (see for more docs). func (tsr *String) SetShapeFrom(frm Tensor) { diff --git a/tensor/table/README.md b/tensor/table/README.md index ef3a8e1d31..b5035f0b8d 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -12,9 +12,11 @@ As a general convention, it is safest, clearest, and quite fast to access column The table itself stores raw data `tensor.Tensor` values, and the `Column` (by index) and `ColumnByName` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). It is best to use the table-wise `Sort` and `Filter` methods (and any others that affect the indexes) to ensure the indexes are properly coordinated. Resetting the column tensor indexes to `nil` (via the `Sequential` method) will break any connection to the table indexes, so that any subsequent index-altering operations on that indexed tensor will be fine. +It is very low-cost to create a new View of a + # Cheat Sheet -`dt` is the etable pointer variable for examples below: +`dt` is the Table pointer variable for examples below: ## Table Access @@ -86,7 +88,7 @@ SortedTable := ix.NewTable() // turn an Indexed back into a new Table organized or: ```Go -nmcl := dt.ColumnByName("Name") // nmcl is an etensor of the Name column, cached +nmcl := dt.Column("Name") // nmcl is an etensor of the Name column, cached ix.Sort(func(t *Table, i, j int) bool { return nmcl.StringValue1D(i) < nmcl.StringValue1D(j) }) @@ -95,7 +97,7 @@ ix.Sort(func(t *Table, i, j int) bool { ### Filter ```Go -nmcl := dt.ColumnByName("Name") // column we're filtering on +nmcl := dt.Column("Name") // column we're filtering on ix.Filter(func(t *Table, row int) bool { // filter return value is for what to *keep* (=true), not exclude // here we keep any row with a name that contains the string "in" diff --git a/tensor/table/columns.go b/tensor/table/columns.go new file mode 100644 index 0000000000..98d57b3f92 --- /dev/null +++ b/tensor/table/columns.go @@ -0,0 +1,96 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package table + +import ( + "fmt" + + "cogentcore.org/core/tensor" +) + +// Columns is the underlying column list and number of rows for Table. +// Table is an Indexed view onto the Columns. +type Columns struct { + tensor.List + + // number of rows, which is enforced to be the size of the + // outermost row dimension of the column tensors. + Rows int `edit:"-"` +} + +// NewColumns returns a new Columns. +func NewColumns() *Columns { + return &Columns{} +} + +// IsValidRow returns error if the row is invalid +func (cl *Columns) IsValidRow(row int) error { + if row < 0 || row >= cl.Rows { + return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, cl.Rows) + } + return nil +} + +// SetNumRows sets the number of rows in the table, across all columns +// if rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0 +func (cl *Columns) SetNumRows(rows int) *Columns { //types:add + cl.Rows = rows // can be 0 + rows = max(1, rows) + for _, tsr := range cl.List.List { + tsr.SetNumRows(rows) + } + return cl +} + +// AddColumn adds the given tensor as a column, +// returning an error and not adding if the name is not unique. +// Automatically adjusts the shape to fit the current number of rows. +func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { + err := cl.Add(name, tsr) + if err != nil { + return err + } + rows := max(1, cl.Rows) + tsr.SetNumRows(rows) + return nil +} + +// InsertColumn inserts the given tensor as a column at given index, +// returning an error and not adding if the name is not unique. +// Automatically adjusts the shape to fit the current number of rows. +func (cl *Columns) InsertColumn(idx int, name string, tsr tensor.Tensor) error { + err := cl.Insert(idx, name, tsr) + if err != nil { + return err + } + rows := max(1, cl.Rows) + tsr.SetNumRows(rows) + return nil +} + +// Clone returns a complete copy of this set of columns. +func (cl *Columns) Clone() *Columns { + cp := NewColumns().SetNumRows(cl.Rows) + keys := cl.Keys() + for i, nm := range keys { + tsr := cl.List.List[i] + cp.AddColumn(nm, tsr.Clone()) + } + return cl +} + +// AppendRows appends shared columns in both tables with input table rows +func (cl *Columns) AppendRows(cl2 *Columns) { + keys := cl.Keys() + for i, nm := range keys { + c2 := cl2.ValueByKey(nm) + if c2 == nil { + continue + } + c1 := cl.List.List[i] + c1.AppendFrom(c2) + } + cl.SetNumRows(cl.Rows + cl2.Rows) +} diff --git a/tensor/table/indexed.go b/tensor/table/indexed.go index 064642ba24..4477039783 100644 --- a/tensor/table/indexed.go +++ b/tensor/table/indexed.go @@ -5,110 +5,84 @@ package table import ( - "fmt" "math/rand" "slices" "sort" "strings" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor" ) -// Indexed is an indexed wrapper around a table.Table that provides a -// specific view onto the Table defined by the set of indexes. -// This provides an efficient way of sorting and filtering a table by only -// updating the indexes while doing nothing to the Table itself. -// To produce a table that has data actually organized according to the -// indexed order, call the NewTable method. -// Indexed views on a table can also be organized together as Splits -// of the table rows, e.g., by grouping values along a given column. -type Indexed struct { //types:add - - // Table that we are an indexed view onto - Table *Table - - // current indexes into Table - Indexes []int +// Index returns the actual index into underlying tensor row based on given +// index value. If Indexes == nil, index is passed through. +func (dt *Table) Index(idx int) int { + if dt.Indexes == nil { + return idx + } + return dt.Indexes[idx] } -// NewIndexed returns a new Indexed based on given table, initialized with sequential idxes -func NewIndexed(dt *Table) *Indexed { - ix := &Indexed{} - ix.SetTable(dt) - return ix +// Rows returns the number of rows, which is the number of Indexes if present, +// else actual number of rows in Tensor data. +func (dt *Table) Rows() int { + if dt.Indexes == nil { + return dt.Columns.Rows + } + return len(dt.Indexes) } -// SetTable sets as indexes into given table with sequential initial indexes -func (ix *Indexed) SetTable(dt *Table) { - ix.Table = dt - ix.Sequential() +// Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor. +func (dt *Table) Sequential() { //types:add + dt.Indexes = nil } -// DeleteInvalid deletes all invalid indexes from the list. -// Call this if rows (could) have been deleted from table. -func (ix *Indexed) DeleteInvalid() { - if ix.Table == nil || ix.Table.Rows <= 0 { - ix.Indexes = nil +// IndexesNeeded is called prior to an operation that needs actual indexes, +// e.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise +// current indexes are left as is. Use Sequential, then IndexesNeeded to ensure +// all rows are represented. +func (dt *Table) IndexesNeeded() { //types:add + if dt.Indexes != nil { return } - ni := ix.Len() - for i := ni - 1; i >= 0; i-- { - if ix.Indexes[i] >= ix.Table.Rows { - ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) - } + dt.Indexes = make([]int, dt.Columns.Rows) + for i := range dt.Indexes { + dt.Indexes[i] = i } } -// Sequential sets indexes to sequential row-wise indexes into table -func (ix *Indexed) Sequential() { //types:add - if ix.Table == nil || ix.Table.Rows <= 0 { - ix.Indexes = nil +// DeleteInvalid deletes all invalid indexes from the list. +// Call this if rows (could) have been deleted from table. +func (dt *Table) DeleteInvalid() { + if dt.Columns.Rows <= 0 || dt.Indexes == nil { + dt.Indexes = nil return } - ix.Indexes = make([]int, ix.Table.Rows) - for i := range ix.Indexes { - ix.Indexes[i] = i + ni := dt.Rows() + for i := ni - 1; i >= 0; i-- { + if dt.Indexes[i] >= dt.Columns.Rows { + dt.Indexes = append(dt.Indexes[:i], dt.Indexes[i+1:]...) + } } } // Permuted sets indexes to a permuted order -- if indexes already exist // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated -func (ix *Indexed) Permuted() { - if ix.Table == nil || ix.Table.Rows <= 0 { - ix.Indexes = nil +func (dt *Table) Permuted() { + if dt.Columns.Rows <= 0 { + dt.Indexes = nil return } - if len(ix.Indexes) == 0 { - ix.Indexes = rand.Perm(ix.Table.Rows) + if dt.Indexes == nil { + dt.Indexes = rand.Perm(dt.Columns.Rows) } else { - rand.Shuffle(len(ix.Indexes), func(i, j int) { - ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] + rand.Shuffle(len(dt.Indexes), func(i, j int) { + dt.Indexes[i], dt.Indexes[j] = dt.Indexes[j], dt.Indexes[i] }) } } // AddIndex adds a new index to the list -func (ix *Indexed) AddIndex(idx int) { - ix.Indexes = append(ix.Indexes, idx) -} - -// IndexedColumnName returns a tensor.Indexed view of given column name, -// using our current indexes. -func (ix *Indexed) IndexedColumnName(column string) (*tensor.Indexed, error) { //types:add - ci, err := ix.Table.ColumnIndex(column) - if errors.Log(err) != nil { - return nil, err - } - return ix.IndexedColumn(ci), nil -} - -// IndexedColumnName returns a tensor.Indexed view of given column, -// using our current indexes. -func (ix *Indexed) IndexedColumn(colIndex int) *tensor.Indexed { - cl := ix.Table.Columns[colIndex] - return tensor.NewIndexed(cl, ix.Indexes) +func (dt *Table) AddIndex(idx int) { + dt.Indexes = append(dt.Indexes, idx) } // SortFunc sorts the indexes into our Table using given compare function. @@ -116,85 +90,65 @@ func (ix *Indexed) IndexedColumn(colIndex int) *tensor.Indexed { // as these row numbers have already been projected through the indexes. // cmp(a, b) should return a negative number when a < b, a positive // number when a > b and zero when a == b. -func (ix *Indexed) SortFunc(cmp func(dt *Table, i, j int) int) { - slices.SortFunc(ix.Indexes, func(a, b int) int { - return cmp(ix.Table, ix.Indexes[a], ix.Indexes[b]) +func (dt *Table) SortFunc(cmp func(dt *Table, i, j int) int) { + dt.IndexesNeeded() + slices.SortFunc(dt.Indexes, func(a, b int) int { + return cmp(dt, dt.Indexes[a], dt.Indexes[b]) }) } // SortIndexes sorts the indexes into our Table directly in // numerical order, producing the native ordering, while preserving // any filtering that might have occurred. -func (ix *Indexed) SortIndexes() { - sort.Ints(ix.Indexes) +func (dt *Table) SortIndexes() { + if dt.Indexes == nil { + return + } + sort.Ints(dt.Indexes) } -// SortColumnName sorts the indexes into our Table according to values in +// SortColumn sorts the indexes into our Table according to values in // given column name, using either ascending or descending order. // Only valid for 1-dimensional columns. -// Returns error if column name not found. -func (ix *Indexed) SortColumnName(column string, ascending bool) error { //types:add - ci, err := ix.Table.ColumnIndex(column) - if errors.Log(err) != nil { +// Returns error if column name not found or column is not 1D. +func (dt *Table) SortColumn(column string, ascending bool) error { //types:add + dt.IndexesNeeded() + cl, err := dt.ColumnTry(column) // has our indexes + if err != nil { return err } - ix.SortColumn(ci, ascending) - return nil -} - -// SortColumn sorts the indexes into our Table according to values in -// given column index, using either ascending or descending order. -// Only valid for 1-dimensional columns (returns error if not). -func (ix *Indexed) SortColumn(colIndex int, ascending bool) error { - cl := ix.Table.Columns[colIndex] - if cl.NumDims() > 1 { - return errors.New("tensor Sorting is only for 1D tensors") - } - if cl.IsString() { - ix.SortFunc(func(dt *Table, i, j int) int { - return tensor.CompareStrings(cl.String1D(i), cl.String1D(j), ascending) - }) - } else { - ix.SortFunc(func(dt *Table, i, j int) int { - return tensor.CompareNumbers(cl.Float1D(i), cl.Float1D(j), ascending) - }) - } - return nil + return cl.Sort(ascending) } -// SortColumnNames sorts the indexes into our Table according to values in +// SortColumns sorts the indexes into our Table according to values in // given column names, using either ascending or descending order, // and optionally using a stable sort. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *Indexed) SortColumnNames(columns []string, ascending, stable bool) error { +func (dt *Table) SortColumns(ascending, stable bool, columns ...string) { nc := len(columns) - if nc == 0 { - return fmt.Errorf("table.Indexed.SortColumnNames: no column names provided") - } - cis := make([]int, nc) - for i, cn := range columns { - ci, err := ix.Table.ColumnIndex(cn) - if errors.Log(err) != nil { - return err + cis := make([]int, 0, nc) + for _, cn := range columns { + ci := dt.Columns.IndexByKey(cn) + if ci >= 0 { + cis = append(cis, ci) } - cis[i] = ci } - ix.SortColumns(cis, ascending, stable) - return nil + dt.SortColumnIndexes(ascending, stable, cis...) } -// SortColumns sorts the indexes into our Table according to values in +// SortColumnIndexes sorts the indexes into our Table according to values in // given list of column indexes, using either ascending or descending order for // all of the columns. Only valid for 1-dimensional columns. -func (ix *Indexed) SortColumns(colIndexes []int, ascending, stable bool) { - sf := ix.SortFunc +func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { + dt.IndexesNeeded() + sf := dt.SortFunc if stable { - sf = ix.SortStableFunc + sf = dt.SortStableFunc } sf(func(dt *Table, i, j int) int { for _, ci := range colIndexes { - cl := ix.Table.Columns[ci] + cl := dt.Columns.List.List[ci] if cl.IsString() { if ascending { if cl.String1D(i) < cl.String1D(j) { @@ -239,39 +193,24 @@ func (ix *Indexed) SortColumns(colIndexes []int, ascending, stable bool) { // number when a > b and zero when a == b. // It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (ix *Indexed) SortStableFunc(cmp func(dt *Table, i, j int) int) { - slices.SortStableFunc(ix.Indexes, func(a, b int) int { - return cmp(ix.Table, ix.Indexes[a], ix.Indexes[b]) +func (dt *Table) SortStableFunc(cmp func(dt *Table, i, j int) int) { + dt.IndexesNeeded() + slices.SortStableFunc(dt.Indexes, func(a, b int) int { + return cmp(dt, dt.Indexes[a], dt.Indexes[b]) }) } -// SortStableColumnName sorts the indexes into our Table according to values in +// SortStableColumn sorts the indexes into our Table according to values in // given column name, using either ascending or descending order. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *Indexed) SortStableColumnName(column string, ascending bool) error { - ci, err := ix.Table.ColumnIndex(column) - if errors.Log(err) != nil { +func (dt *Table) SortStableColumn(column string, ascending bool) error { + dt.IndexesNeeded() + cl, err := dt.ColumnTry(column) // has our indexes + if err != nil { return err } - ix.SortStableColumn(ci, ascending) - return nil -} - -// SortStableColumn sorts the indexes into our Table according to values in -// given column index, using either ascending or descending order. -// Only valid for 1-dimensional columns. -func (ix *Indexed) SortStableColumn(colIndex int, ascending bool) { - cl := ix.Table.Columns[colIndex] - if cl.IsString() { - ix.SortStableFunc(func(dt *Table, i, j int) int { - return tensor.CompareStrings(cl.String1D(i), cl.String1D(j), ascending) - }) - } else { - ix.SortStableFunc(func(dt *Table, i, j int) int { - return tensor.CompareNumbers(cl.Float1D(i), cl.Float1D(j), ascending) - }) - } + return cl.SortStable(ascending) } // FilterFunc is a function used for filtering that returns @@ -282,42 +221,31 @@ type FilterFunc func(dt *Table, row int) bool // Filter filters the indexes into our Table using given Filter function. // The Filter function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. -func (ix *Indexed) Filter(filterer func(dt *Table, row int) bool) { - sz := len(ix.Indexes) +func (dt *Table) Filter(filterer func(dt *Table, row int) bool) { + dt.IndexesNeeded() + sz := len(dt.Indexes) for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering - if !filterer(ix.Table, ix.Indexes[i]) { // delete - ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + if !filterer(dt, dt.Indexes[i]) { // delete + dt.Indexes = append(dt.Indexes[:i], dt.Indexes[i+1:]...) } } } -// FilterColumnName filters the indexes into our Table according to values in +// FilterColumn filters the indexes into our Table according to values in // given column name, using string representation of column values. // Includes rows with matching values unless exclude is set. // If contains, only checks if row contains string; if ignoreCase, ignores case. // Use named args for greater clarity. // Only valid for 1-dimensional columns. // Returns error if column name not found. -func (ix *Indexed) FilterColumnName(column string, str string, exclude, contains, ignoreCase bool) error { //types:add - ci, err := ix.Table.ColumnIndex(column) - if errors.Log(err) != nil { +func (dt *Table) FilterColumn(column string, str string, exclude, contains, ignoreCase bool) error { //types:add + col, err := dt.ColumnTry(column) + if err != nil { return err } - ix.FilterColumn(ci, str, exclude, contains, ignoreCase) - return nil -} - -// FilterColumn sorts the indexes into our Table according to values in -// given column index, using string representation of column values. -// Includes rows with matching values unless exclude is set. -// If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use named args for greater clarity. -// Only valid for 1-dimensional columns. -func (ix *Indexed) FilterColumn(colIndex int, str string, exclude, contains, ignoreCase bool) { - col := ix.Table.Columns[colIndex] lowstr := strings.ToLower(str) - ix.Filter(func(dt *Table, row int) bool { - val := col.String1D(row) + dt.Filter(func(dt *Table, row int) bool { + val := col.StringRowCell(row, 0) has := false switch { case contains && ignoreCase: @@ -334,78 +262,66 @@ func (ix *Indexed) FilterColumn(colIndex int, str string, exclude, contains, ign } return has }) + return nil } // NewTable returns a new table with column data organized according to -// the indexes -func (ix *Indexed) NewTable() *Table { - rows := len(ix.Indexes) - nt := ix.Table.Clone() +// the indexes. If Indexes are nil, a clone of the current tensor is returned +// but this function is only sensible if there is an indexed view in place. +func (dt *Table) NewTable() *Table { + if dt.Indexes == nil { + return dt.Clone() + } + rows := len(dt.Indexes) + nt := dt.Clone() + nt.Indexes = nil nt.SetNumRows(rows) if rows == 0 { return nt } - for ci := range nt.Columns { - scl := ix.Table.Columns[ci] - tcl := nt.Columns[ci] - _, csz := tcl.RowCellSize() - for i, srw := range ix.Indexes { - tcl.CopyCellsFrom(scl, i*csz, srw*csz, csz) + for ci, cl := range nt.Columns.List.List { + scl := dt.Columns.List.List[ci] + _, csz := cl.RowCellSize() + for i, srw := range dt.Indexes { + cl.CopyCellsFrom(scl, i*csz, srw*csz, csz) } } return nt } -// Clone returns a copy of the current index view with its own index memory -func (ix *Indexed) Clone() *Indexed { - nix := &Indexed{} - nix.CopyFrom(ix) - return nix -} - -// CopyFrom copies from given other Indexed (we have our own unique copy of indexes) -func (ix *Indexed) CopyFrom(oix *Indexed) { - ix.Table = oix.Table - ix.Indexes = slices.Clone(oix.Indexes) -} - -// AddRows adds n rows to end of underlying Table, and to the indexes in this view -func (ix *Indexed) AddRows(n int) { //types:add - stidx := ix.Table.Rows - ix.Table.SetNumRows(stidx + n) - for i := stidx; i < stidx+n; i++ { - ix.Indexes = append(ix.Indexes, i) - } -} - -// InsertRows adds n rows to end of underlying Table, and to the indexes starting at -// given index in this view -func (ix *Indexed) InsertRows(at, n int) { - stidx := ix.Table.Rows - ix.Table.SetNumRows(stidx + n) - nw := make([]int, n, n+len(ix.Indexes)-at) - for i := 0; i < n; i++ { - nw[i] = stidx + i - } - ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) -} - // DeleteRows deletes n rows of indexes starting at given index in the list of indexes -func (ix *Indexed) DeleteRows(at, n int) { - ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) +func (dt *Table) DeleteRows(at, n int) { + dt.IndexesNeeded() + dt.Indexes = append(dt.Indexes[:at], dt.Indexes[at+n:]...) } -// RowsByStringIndex returns the list of *our indexes* whose row in the table has -// given string value in given column index (de-reference our indexes to get actual row). +// Named arg values for Contains, IgnoreCase +const ( + // Contains means the string only needs to contain the target string (see Equals) + Contains bool = true + // Equals means the string must equal the target string (see Contains) + Equals = false + // IgnoreCase means that differences in case are ignored in comparing strings + IgnoreCase = true + // UseCase means that case matters when comparing strings + UseCase = false +) + +// RowsByString returns the list of *our indexes* whose row in the table has +// given string value in given column name (de-reference our indexes to get actual row). // if contains, only checks if row contains string; if ignoreCase, ignores case. // Use named args for greater clarity. -func (ix *Indexed) RowsByStringIndex(colIndex int, str string, contains, ignoreCase bool) []int { - dt := ix.Table - col := dt.Columns[colIndex] +func (dt *Table) RowsByString(colname string, str string, contains, ignoreCase bool) []int { + col := dt.Column(colname) + if col == nil { + return nil + } lowstr := strings.ToLower(str) var indexes []int - for idx, srw := range ix.Indexes { - val := col.String1D(srw) + rows := dt.Rows() + for idx := range rows { + srw := dt.Index(idx) + val := col.Tensor.String1D(srw) has := false switch { case contains && ignoreCase: @@ -424,26 +340,7 @@ func (ix *Indexed) RowsByStringIndex(colIndex int, str string, contains, ignoreC return indexes } -// RowsByString returns the list of *our indexes* whose row in the table has -// given string value in given column name (de-reference our indexes to get actual row). -// if contains, only checks if row contains string; if ignoreCase, ignores case. -// returns error message for invalid column name. -// Use named args for greater clarity. -func (ix *Indexed) RowsByString(column string, str string, contains, ignoreCase bool) ([]int, error) { - dt := ix.Table - ci, err := dt.ColumnIndex(column) - if errors.Log(err) != nil { - return nil, err - } - return ix.RowsByStringIndex(ci, str, contains, ignoreCase), nil -} - -// Len returns the length of the index list -func (ix *Indexed) Len() int { - return len(ix.Indexes) -} - // Swap switches the indexes for i and j -func (ix *Indexed) Swap(i, j int) { - ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] +func (dt *Table) Swap(i, j int) { + dt.Indexes[i], dt.Indexes[j] = dt.Indexes[j], dt.Indexes[i] } diff --git a/tensor/table/io.go b/tensor/table/io.go index a2f30dcb8a..6ba3b0e9c9 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -18,6 +18,7 @@ import ( "strings" "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/core" "cogentcore.org/core/tensor" ) @@ -50,25 +51,6 @@ func (dt *Table) SaveCSV(filename core.Filename, delim tensor.Delims, headers bo return err } -// SaveCSV writes a table index view to a comma-separated-values (CSV) file -// (where comma = any delimiter, specified in the delim arg). -// If headers = true then generate column headers that capture the type -// and tensor cell geometry of the columns, enabling full reloading -// of exactly the same table format and data (recommended). -// Otherwise, only the data is written. -func (ix *Indexed) SaveCSV(filename core.Filename, delim tensor.Delims, headers bool) error { //types:add - fp, err := os.Create(string(filename)) - defer fp.Close() - if err != nil { - log.Println(err) - return err - } - bw := bufio.NewWriter(fp) - err = ix.WriteCSV(bw, delim, headers) - bw.Flush() - return err -} - // OpenCSV reads a table from a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg), // using the Go standard encoding/csv reader conforming to the official CSV standard. @@ -97,28 +79,6 @@ func (dt *Table) OpenFS(fsys fs.FS, filename string, delim tensor.Delims) error return dt.ReadCSV(bufio.NewReader(fp), delim) } -// OpenCSV reads a table idx view from a comma-separated-values (CSV) file -// (where comma = any delimiter, specified in the delim arg), -// using the Go standard encoding/csv reader conforming to the official CSV standard. -// If the table does not currently have any columns, the first row of the file -// is assumed to be headers, and columns are constructed therefrom. -// If the file was saved from table with headers, then these have full configuration -// information for tensor type and dimensionality. -// If the table DOES have existing columns, then those are used robustly -// for whatever information fits from each row of the file. -func (ix *Indexed) OpenCSV(filename core.Filename, delim tensor.Delims) error { //types:add - err := ix.Table.OpenCSV(filename, delim) - ix.Sequential() - return err -} - -// OpenFS is the version of [Indexed.OpenCSV] that uses an [fs.FS] filesystem. -func (ix *Indexed) OpenFS(fsys fs.FS, filename string, delim tensor.Delims) error { - err := ix.Table.OpenFS(fsys, filename, delim) - ix.Sequential() - return err -} - // ReadCSV reads a table from a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg), // using the Go standard encoding/csv reader conforming to the official CSV standard. @@ -129,6 +89,7 @@ func (ix *Indexed) OpenFS(fsys fs.FS, filename string, delim tensor.Delims) erro // If the table DOES have existing columns, then those are used robustly // for whatever information fits from each row of the file. func (dt *Table) ReadCSV(r io.Reader, delim tensor.Delims) error { + dt.Sequential() cr := csv.NewReader(r) cr.Comma = delim.Rune() rec, err := cr.ReadAll() // todo: lazy, avoid resizing @@ -157,14 +118,12 @@ func (dt *Table) ReadCSV(r io.Reader, delim tensor.Delims) error { // ReadCSVRow reads a record of CSV data into given row in table func (dt *Table) ReadCSVRow(rec []string, row int) { - tc := dt.NumColumns() ci := 0 if rec[0] == "_D:" { // data row ci++ } nan := math.NaN() - for j := 0; j < tc; j++ { - tsr := dt.Columns[j] + for _, tsr := range dt.Columns.List.List { _, csz := tsr.RowCellSize() stoff := row * csz for cc := 0; cc < csz; cc++ { @@ -227,14 +186,14 @@ func ConfigFromTableHeaders(dt *Table, hdrs []string) error { hd = hd[:lbst] csh := ShapeFromString(dims) // new tensor starting - dt.AddTensorColumnOfType(typ, hd, csh...) + dt.AddTensorColumnOfType(hd, typ, csh...) continue } dimst = strings.Index(hd, "[") if dimst > 0 { continue } - dt.AddColumnOfType(typ, hd) + dt.AddColumnOfType(hd, typ) } return nil } @@ -329,7 +288,7 @@ func ConfigFromDataValues(dt *Table, hdrs []string, rec [][]string) error { typ = ctyp } } - dt.AddColumnOfType(typ, hd) + dt.AddColumnOfType(hd, typ) } return nil } @@ -358,7 +317,7 @@ func InferDataType(str string) reflect.Kind { ////////////////////////////////////////////////////////////////////////// // WriteCSV -// WriteCSV writes a table to a comma-separated-values (CSV) file +// WriteCSV writes only rows in table idx view to a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg). // If headers = true then generate column headers that capture the type // and tensor cell geometry of the columns, enabling full reloading @@ -376,38 +335,10 @@ func (dt *Table) WriteCSV(w io.Writer, delim tensor.Delims, headers bool) error } cw := csv.NewWriter(w) cw.Comma = delim.Rune() - for ri := 0; ri < dt.Rows; ri++ { - err = dt.WriteCSVRowWriter(cw, ri, ncol) - if err != nil { - log.Println(err) - return err - } - } - cw.Flush() - return nil -} - -// WriteCSV writes only rows in table idx view to a comma-separated-values (CSV) file -// (where comma = any delimiter, specified in the delim arg). -// If headers = true then generate column headers that capture the type -// and tensor cell geometry of the columns, enabling full reloading -// of exactly the same table format and data (recommended). -// Otherwise, only the data is written. -func (ix *Indexed) WriteCSV(w io.Writer, delim tensor.Delims, headers bool) error { - ncol := 0 - var err error - if headers { - ncol, err = ix.Table.WriteCSVHeaders(w, delim) - if err != nil { - log.Println(err) - return err - } - } - cw := csv.NewWriter(w) - cw.Comma = delim.Rune() - nrow := ix.Len() - for ri := 0; ri < nrow; ri++ { - err = ix.Table.WriteCSVRowWriter(cw, ix.Indexes[ri], ncol) + nrow := dt.Rows() + for ri := range nrow { + ix := dt.Index(ri) + err = dt.WriteCSVRowWriter(cw, ix, ncol) if err != nil { log.Println(err) return err @@ -446,8 +377,8 @@ func (dt *Table) WriteCSVRow(w io.Writer, row int, delim tensor.Delims) error { // WriteCSVRowWriter uses csv.Writer to write one row func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { prec := -1 - if ps, ok := dt.MetaData["precision"]; ok { - prec, _ = strconv.Atoi(ps) + if ps, err := metadata.Get[int](dt.Meta, "precision"); err == nil { + prec = ps } var rec []string if ncol > 0 { @@ -456,8 +387,7 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { rec = make([]string, 0) } rc := 0 - for i := range dt.Columns { - tsr := dt.Columns[i] + for _, tsr := range dt.Columns.List.List { nd := tsr.NumDims() if nd == 1 { vl := "" @@ -499,9 +429,9 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { // with full information about type and tensor cell dimensionality. func (dt *Table) TableHeaders() []string { hdrs := []string{} - for i := range dt.Columns { - tsr := dt.Columns[i] - nm := dt.ColumnNames[i] + names := dt.Columns.Keys() + for i, nm := range names { + tsr := dt.Columns.List.List[i] nm = string([]byte{TableHeaderChar(tsr.DataType())}) + nm if tsr.NumDims() == 1 { hdrs = append(hdrs, nm) diff --git a/tensor/table/splits.go b/tensor/table/splits.go index 6bc5f8d406..4489b5f53b 100644 --- a/tensor/table/splits.go +++ b/tensor/table/splits.go @@ -9,8 +9,6 @@ import ( "slices" "sort" "strings" - - "cogentcore.org/core/base/errors" ) // SplitAgg contains aggregation results for splits @@ -58,7 +56,7 @@ type SplitAgg struct { type Splits struct { // the list of index views for each split - Splits []*Indexed + Splits []*Table // levels of indexes used to organize the splits -- each split contains the full outer product across these index levels. for example, if the split was generated by grouping over column values, then these are the column names in order of grouping. the splits are not automatically sorted hierarchically by these levels but e.g., the GroupBy method produces that result -- use the Sort methods to explicitly sort. Levels []string @@ -88,16 +86,16 @@ func (spl *Splits) Table() *Table { if len(spl.Splits) == 0 { return nil } - return spl.Splits[0].Table + return spl.Splits[0] } // New adds a new split to the list for given table, and with associated // values, which are copied before saving into Values list, and any number of rows // from the table associated with this split (also copied). // Any existing Aggs are deleted by this. -func (spl *Splits) New(dt *Table, values []string, rows ...int) *Indexed { +func (spl *Splits) New(dt *Table, values []string, rows ...int) *Table { spl.Aggs = nil - ix := &Indexed{Table: dt} + ix := NewTableView(dt) spl.Splits = append(spl.Splits, ix) if len(rows) > 0 { ix.Indexes = append(ix.Indexes, slices.Clone(rows)...) @@ -239,7 +237,7 @@ func (spl *Splits) ExtractLevels(levels []int) (*Splits, error) { // now just do the grouping by levels values lstValues := make([]string, nlv) curValues := make([]string, nlv) - var curIx *Indexed + var curIx *Table nsp := len(ss.Splits) for si := nsp - 1; si >= 0; si-- { diff := false @@ -295,7 +293,7 @@ func (spl *Splits) Clone() *Splits { // CopyFrom copies from other Splits -- we get our own unique copy of everything func (spl *Splits) CopyFrom(osp *Splits) { - spl.Splits = make([]*Indexed, len(osp.Splits)) + spl.Splits = make([]*Table, len(osp.Splits)) spl.Values = make([][]string, len(osp.Values)) for si := range osp.Splits { spl.Splits[si] = osp.Splits[si].Clone() @@ -347,9 +345,9 @@ func (spl *Splits) AggByColumnName(name string) (*SplitAgg, error) { return nil, fmt.Errorf("table.Splits AggByColumnName: table nil") } nmsp := strings.Split(name, ":") - colIndex, err := dt.ColumnIndex(nmsp[0]) - if err != nil { - return nil, err + colIndex := dt.Columns.IndexByKey(nmsp[0]) + if colIndex < 0 { + return nil, nil } for _, ag := range spl.Aggs { if ag.ColumnIndex != colIndex { @@ -388,19 +386,19 @@ func (spl *Splits) AggsToTable(colName bool) *Table { if nsp == 0 { return nil } - dt := spl.Splits[0].Table + dt := spl.Splits[0] st := NewTable().SetNumRows(nsp) for _, cn := range spl.Levels { - oc, _ := dt.ColumnByName(cn) + oc := dt.Column(cn) if oc != nil { - st.AddColumnOfType(oc.DataType(), cn) + st.AddColumnOfType(cn, oc.Tensor.DataType()) } else { st.AddStringColumn(cn) } } for _, ag := range spl.Aggs { - col := dt.Columns[ag.ColumnIndex] - an := dt.ColumnNames[ag.ColumnIndex] + col := dt.Columns.List.List[ag.ColumnIndex] + an := "" // dt.ColumnNames[ag.ColumnIndex] // todo: need fast access! if colName == AddAggName { an += ":" + ag.Name } @@ -409,12 +407,12 @@ func (spl *Splits) AggsToTable(colName bool) *Table { for si := range spl.Splits { cidx := 0 for ci := range spl.Levels { - col := st.Columns[cidx] + col := st.Columns.List.List[cidx] col.SetString1D(spl.Values[si][ci], si) cidx++ } for _, ag := range spl.Aggs { - col := st.Columns[cidx] + col := st.Columns.List.List[cidx] _, csz := col.RowCellSize() sti := si * csz av := ag.Aggs[si] @@ -439,7 +437,7 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { if nsp == 0 { return nil } - dt := spl.Splits[0].Table + dt := spl.Splits[0] st := NewTable().SetNumRows(nsp) exmap := make(map[string]struct{}) for _, cn := range spl.Levels { @@ -447,8 +445,8 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { exmap[cn] = struct{}{} } for _, ag := range spl.Aggs { - col := dt.Columns[ag.ColumnIndex] - an := dt.ColumnNames[ag.ColumnIndex] + col := dt.Columns.List.List[ag.ColumnIndex] + an := "" // dt.ColumnNames[ag.ColumnIndex] exmap[an] = struct{}{} if colName == AddAggName { an += ":" + ag.Name @@ -456,22 +454,23 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:]...) } var cpcol []string - for _, cn := range dt.ColumnNames { + names := dt.Columns.Keys() + for _, cn := range names { if _, ok := exmap[cn]; !ok { cpcol = append(cpcol, cn) - col := errors.Log1(dt.ColumnByName(cn)) - st.AddColumn(col.Clone(), cn) + col := dt.Column(cn) + st.AddColumn(cn, col.Tensor.Clone()) } } for si, sidx := range spl.Splits { cidx := 0 for ci := range spl.Levels { - col := st.Columns[cidx] + col := st.Columns.List.List[cidx] col.SetString1D(spl.Values[si][ci], si) cidx++ } for _, ag := range spl.Aggs { - col := st.Columns[cidx] + col := st.Columns.List.List[cidx] _, csz := col.RowCellSize() sti := si * csz av := ag.Aggs[si] diff --git a/tensor/table/table.go b/tensor/table/table.go index 449619d079..a47e4809f3 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -7,82 +7,109 @@ package table //go:generate core generate import ( - "errors" "fmt" - "log/slog" "math" "reflect" "slices" - "strings" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/tensor" ) -// Table is a table of data, with columns of tensors, -// each with the same number of Rows (outermost dimension). +// Table is a table of Tensor columns aligned by a common outermost row dimension. +// Use the Column (by name) and ColumnIndex methods to obtain a `tensor.Indexed` view of +// the column, using the shared Indexes of the Table. Thus, a coordinated +// sorting and filtered view of the column data is automatically available for +// any of the tensor package functions that use `tensor.Indexed` as the one +// common data representation for all operations. type Table struct { //types:add - - // columns of data, as tensor.Tensor tensors - Columns []tensor.Tensor `display:"no-inline"` - - // the names of the columns - ColumnNames []string - - // number of rows, which is enforced to be the size of the outermost dimension of the column tensors - Rows int `edit:"-"` - - // the map of column names to column numbers - ColumnNameMap map[string]int `display:"-"` - - // misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column - MetaData map[string]string -} - + // Columns has the list of column tensor data for this table. + // Different tables can have + Columns *Columns + + // Indexes are the indexes into Tensor rows, with nil = sequential. + // Only set if order is different from default sequential order. + // These indexes are shared into the `tensor.Indexed` Column values + // to provide a coordinated indexed view into the underlying data. + Indexes []int + + // Meta is misc metadata for the table. Use lower-case key names + // following the struct tag convention: + // - name string = name of table + // - doc string = documentation, description + // - read-only bool = gui is read-only + // - precision int = n for precision to write out floats in csv. + Meta metadata.Data +} + +// NewTable returnes a new Table with its own (empty) set of Columns. +// Can pass an optional name. func NewTable(name ...string) *Table { dt := &Table{} + dt.Columns = NewColumns() if len(name) > 0 { - dt.SetMetaData("name", name[0]) + dt.Meta.Set("name", name[0]) } return dt } +// NewTableView returns a new Table with its own Indexed view into the +// same underlying set of Column tensor data as the source table. +// Indexes are nil in the new Table, resulting in default full sequential view. +func NewTableView(src *Table) *Table { + dt := &Table{Columns: src.Columns} + dt.Meta.Copy(src.Meta) + return dt +} + // IsValidRow returns error if the row is invalid func (dt *Table) IsValidRow(row int) error { - if row < 0 || row >= dt.Rows { - return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, dt.Rows) + if row < 0 || row >= dt.Rows() { + return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, dt.Rows()) } return nil } -// NumRows returns the number of rows -func (dt *Table) NumRows() int { return dt.Rows } +// NumRows returns the number of rows. +func (dt *Table) NumRows() int { return dt.Columns.Rows } -// NumColumns returns the number of columns -func (dt *Table) NumColumns() int { return len(dt.Columns) } +// NumColumns returns the number of columns. +func (dt *Table) NumColumns() int { return dt.Columns.Len() } -// Column returns the tensor at given column index -func (dt *Table) Column(i int) tensor.Tensor { return dt.Columns[i] } - -// ColumnByName returns the tensor at given column name, with error message if not found. -// Returns nil if not found -func (dt *Table) ColumnByName(name string) (tensor.Tensor, error) { - i, ok := dt.ColumnNameMap[name] - if !ok { - return nil, fmt.Errorf("table.Table ColumnByNameTry: column named: %v not found", name) +// Column returns the tensor with given column name, as a [tensor.Indexed] +// with the shared [Indexes] from this table. It is best practice to +// access columns by name, and direct access through Columns does not +// provide the shared table-wide Indexes. +// Returns nil if not found. +func (dt *Table) Column(name string) *tensor.Indexed { + cl := dt.Columns.ValueByKey(name) + if cl == nil { + return nil } - return dt.Columns[i], nil + return tensor.NewIndexed(cl, dt.Indexes) } -// ColumnIndex returns the index of the given column name, -// along with an error if not found. -func (dt *Table) ColumnIndex(name string) (int, error) { - i, ok := dt.ColumnNameMap[name] - if !ok { - return 0, fmt.Errorf("table.Table ColumnIndex: column named: %v not found", name) +// ColumnTry is a version of [Column] that also returns an error +// if the column name is not found, for cases when error is needed. +func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { + cl := dt.Column(name) + if cl != nil { + return cl, nil } - return i, nil + return nil, fmt.Errorf("table.Table: Column named %q not found", name) } +// ColumnIndex returns the tensor at the given index, as a [tensor.Indexed] +// with the shared [Indexes] from this table. It is best practice to +// access columns by name using [Column] method instead. +// Direct access through Columns does not provide the shared table-wide Indexes. +// Returns nil if not found. +func (dt *Table) ColumnIndex(idx int) *tensor.Indexed { + cl := dt.Columns.List.List[idx] + return tensor.NewIndexed(cl, dt.Indexes) +} + +/* // ColumnIndexesByNames returns the indexes of the given column names. // idxs have -1 if name not found. func (dt *Table) ColumnIndexesByNames(names ...string) ([]int, error) { @@ -101,52 +128,32 @@ func (dt *Table) ColumnIndexesByNames(names ...string) ([]int, error) { } return cidx, errors.Join(errs...) } +*/ // ColumnName returns the name of given column func (dt *Table) ColumnName(i int) string { return dt.ColumnNames[i] } -// UpdateColumnNameMap updates the column name map, returning an error -// if any of the column names are duplicates. -func (dt *Table) UpdateColumnNameMap() error { - nc := dt.NumColumns() - dt.ColumnNameMap = make(map[string]int, nc) - var errs []error - for i, nm := range dt.ColumnNames { - if _, has := dt.ColumnNameMap[nm]; has { - err := fmt.Errorf("table.Table duplicate column name: %s", nm) - slog.Warn(err.Error()) - errs = append(errs, err) - } else { - dt.ColumnNameMap[nm] = i - } - } - if len(errs) > 0 { - return errors.Join(errs...) - } - return nil -} - // AddColumn adds a new column to the table, of given type and column name // (which must be unique). The cells of this column hold a single scalar value: // see AddColumnTensor for n-dimensional cells. -func AddColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string) tensor.Tensor { - rows := max(1, dt.Rows) +func AddColumn[T tensor.DataTypes](dt *Table, name string) tensor.Tensor { + rows := max(1, dt.Columns.Rows) tsr := tensor.New[T](rows) tsr.SetNames("Row") - dt.AddColumn(tsr, name) + dt.AddColumn(name, tsr) return tsr } // InsertColumn inserts a new column to the table, of given type and column name // (which must be unique), at given index. // The cells of this column hold a single scalar value. -func InsertColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string, idx int) tensor.Tensor { - rows := max(1, dt.Rows) +func InsertColumn[T tensor.DataTypes](dt *Table, name string, idx int) tensor.Tensor { + rows := max(1, dt.Columns.Rows) tsr := tensor.New[T](rows) tsr.SetNames("Row") - dt.InsertColumn(tsr, name, idx) + dt.InsertColumn(idx, name, tsr) return tsr } @@ -154,57 +161,38 @@ func InsertColumn[T string | bool | float32 | float64 | int | int32 | byte](dt * // (which must be unique), and dimensionality of each _cell_. // An outermost Row dimension will be added to this dimensionality to create // the tensor column. -func AddTensorColumn[T string | bool | float32 | float64 | int | int32 | byte](dt *Table, name string, cellSizes ...int) tensor.Tensor { - rows := max(1, dt.Rows) +func AddTensorColumn[T tensor.DataTypes](dt *Table, name string, cellSizes ...int) tensor.Tensor { + rows := max(1, dt.Columns.Rows) sz := append([]int{rows}, cellSizes...) tsr := tensor.New[T](sz...) tsr.SetNames("Row") - dt.AddColumn(tsr, name) + dt.AddColumn(name, tsr) return tsr } // AddColumn adds the given tensor as a column to the table, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows. -func (dt *Table) AddColumn(tsr tensor.Tensor, name string) error { - dt.ColumnNames = append(dt.ColumnNames, name) - err := dt.UpdateColumnNameMap() - if err != nil { - dt.ColumnNames = dt.ColumnNames[:len(dt.ColumnNames)-1] - return err - } - dt.Columns = append(dt.Columns, tsr) - rows := max(1, dt.Rows) - tsr.SetNumRows(rows) - return nil +func (dt *Table) AddColumn(name string, tsr tensor.Tensor) error { + return dt.Columns.AddColumn(name, tsr) } // InsertColumn inserts the given tensor as a column to the table at given index, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows. -func (dt *Table) InsertColumn(tsr tensor.Tensor, name string, idx int) error { - if _, has := dt.ColumnNameMap[name]; has { - err := fmt.Errorf("table.Table duplicate column name: %s", name) - slog.Warn(err.Error()) - return err - } - dt.ColumnNames = slices.Insert(dt.ColumnNames, idx, name) - dt.UpdateColumnNameMap() - dt.Columns = slices.Insert(dt.Columns, idx, tsr) - rows := max(1, dt.Rows) - tsr.SetNumRows(rows) - return nil +func (dt *Table) InsertColumn(idx int, name string, tsr tensor.Tensor) error { + return dt.Columns.InsertColumn(idx, name, tsr) } // AddColumnOfType adds a new scalar column to the table, of given reflect type, // column name (which must be unique), // The cells of this column hold a single (scalar) value of given type. // Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. -func (dt *Table) AddColumnOfType(typ reflect.Kind, name string) tensor.Tensor { - rows := max(1, dt.Rows) +func (dt *Table) AddColumnOfType(name string, typ reflect.Kind) tensor.Tensor { + rows := max(1, dt.Columns.Rows) tsr := tensor.NewOfType(typ, rows) tsr.SetNames("Row") - dt.AddColumn(tsr, name) + dt.AddColumn(name, tsr) return tsr } @@ -213,12 +201,12 @@ func (dt *Table) AddColumnOfType(typ reflect.Kind, name string) tensor.Tensor { // An outermost Row dimension will be added to this dimensionality to create // the tensor column. // Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. -func (dt *Table) AddTensorColumnOfType(typ reflect.Kind, name string, cellSizes ...int) tensor.Tensor { - rows := max(1, dt.Rows) +func (dt *Table) AddTensorColumnOfType(name string, typ reflect.Kind, cellSizes ...int) tensor.Tensor { + rows := max(1, dt.Columns.Rows) sz := append([]int{rows}, cellSizes...) tsr := tensor.NewOfType(typ, sz...) tsr.SetNames("Row") - dt.AddColumn(tsr, name) + dt.AddColumn(name, tsr) return tsr } @@ -271,161 +259,72 @@ func (dt *Table) AddIntTensorColumn(name string, cellSizes ...int) *tensor.Int { } // DeleteColumnName deletes column of given name. -// returns error if not found. -func (dt *Table) DeleteColumnName(name string) error { - ci, err := dt.ColumnIndex(name) - if err != nil { - return err - } - dt.DeleteColumnIndex(ci) - return nil +// returns false if not found. +func (dt *Table) DeleteColumnName(name string) bool { + return dt.Columns.DeleteKey(name) } -// DeleteColumnIndex deletes column of given index -func (dt *Table) DeleteColumnIndex(idx int) { - dt.Columns = append(dt.Columns[:idx], dt.Columns[idx+1:]...) - dt.ColumnNames = append(dt.ColumnNames[:idx], dt.ColumnNames[idx+1:]...) - dt.UpdateColumnNameMap() +// DeleteColumnIndex deletes column within the index range [i:j]. +func (dt *Table) DeleteColumnIndex(i, j int) { + dt.Columns.DeleteIndex(i, j) } -// DeleteAll deletes all columns -- full reset +// DeleteAll deletes all columns, does full reset. func (dt *Table) DeleteAll() { - dt.Columns = nil - dt.ColumnNames = nil - dt.Rows = 0 - dt.ColumnNameMap = nil + dt.Columns.Reset() } -// AddRows adds n rows to each of the columns +// AddRows adds n rows to end of underlying Table, and to the indexes in this view func (dt *Table) AddRows(n int) { //types:add - dt.SetNumRows(dt.Rows + n) + stidx := dt.Columns.Rows + dt.Columns.SetNumRows(stidx + n) + for i := stidx; i < stidx+n; i++ { + dt.Indexes = append(dt.Indexes, i) + } +} + +// InsertRows adds n rows to end of underlying Table, and to the indexes starting at +// given index in this view +func (dt *Table) InsertRows(at, n int) { + stidx := dt.Columns.Rows + dt.Columns.SetNumRows(stidx + n) + if dt.Indexes != nil { + nw := make([]int, n, n+len(dt.Indexes)-at) + for i := 0; i < n; i++ { + nw[i] = stidx + i + } + dt.Indexes = append(dt.Indexes[:at], append(nw, dt.Indexes[at:]...)...) + } } // SetNumRows sets the number of rows in the table, across all columns // if rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0 func (dt *Table) SetNumRows(rows int) *Table { //types:add - dt.Rows = rows // can be 0 - rows = max(1, rows) - for _, tsr := range dt.Columns { - tsr.SetNumRows(rows) - } + dt.Columns.SetNumRows(rows) return dt } // note: no really clean definition of CopyFrom -- no point of re-using existing // table -- just clone it. -// Clone returns a complete copy of this table +// Clone returns a complete copy of this table, +// including cloning the underlying Columns tensors. func (dt *Table) Clone() *Table { - cp := NewTable().SetNumRows(dt.Rows) - cp.CopyMetaDataFrom(dt) - for i, cl := range dt.Columns { - cp.AddColumn(cl.Clone(), dt.ColumnNames[i]) + cp := &Table{} + cp.Columns = dt.Columns.Clone() + cp.Meta.Copy(dt.Meta) + if dt.Indexes != nil { + cp.Indexes = slices.Clone(dt.Indexes) } return cp } // AppendRows appends shared columns in both tables with input table rows func (dt *Table) AppendRows(dt2 *Table) { - shared := false - strow := dt.Rows - for iCol := range dt.Columns { - colName := dt.ColumnName(iCol) - _, err := dt2.ColumnIndex(colName) - if err != nil { - continue - } - if !shared { - shared = true - dt.AddRows(dt2.Rows) - } - for iRow := 0; iRow < dt2.Rows; iRow++ { - dt.CopyCell(colName, iRow+strow, dt2, colName, iRow) - } - } -} - -// SetMetaData sets given meta-data key to given value, safely creating the -// map if not yet initialized. Standard Keys are: -// * name -- name of table -// * desc -- description of table -// * read-only -- makes gui read-only (inactive edits) for tensorcore.Table -// * ColumnName:* -- prefix for all column-specific meta-data -// - desc -- description of column -func (dt *Table) SetMetaData(key, val string) { - if dt.MetaData == nil { - dt.MetaData = make(map[string]string) - } - dt.MetaData[key] = val -} - -// CopyMetaDataFrom copies meta data from other table -func (dt *Table) CopyMetaDataFrom(cp *Table) { - nm := len(cp.MetaData) - if nm == 0 { - return - } - if dt.MetaData == nil { - dt.MetaData = make(map[string]string, nm) - } - for k, v := range cp.MetaData { - dt.MetaData[k] = v - } -} - -// Named arg values for Contains, IgnoreCase -const ( - // Contains means the string only needs to contain the target string (see Equals) - Contains bool = true - // Equals means the string must equal the target string (see Contains) - Equals = false - // IgnoreCase means that differences in case are ignored in comparing strings - IgnoreCase = true - // UseCase means that case matters when comparing strings - UseCase = false -) - -// RowsByStringIndex returns the list of rows that have given -// string value in given column index. -// if contains, only checks if row contains string; if ignoreCase, ignores case. -// Use named args for greater clarity. -func (dt *Table) RowsByStringIndex(column int, str string, contains, ignoreCase bool) []int { - col := dt.Columns[column] - lowstr := strings.ToLower(str) - var idxs []int - for i := 0; i < dt.Rows; i++ { - val := col.String1D(i) - has := false - switch { - case contains && ignoreCase: - has = strings.Contains(strings.ToLower(val), lowstr) - case contains: - has = strings.Contains(val, str) - case ignoreCase: - has = strings.EqualFold(val, str) - default: - has = (val == str) - } - if has { - idxs = append(idxs, i) - } - } - return idxs -} - -// RowsByString returns the list of rows that have given -// string value in given column name. returns nil & error if name invalid. -// if contains, only checks if row contains string; if ignoreCase, ignores case. -// Use named args for greater clarity. -func (dt *Table) RowsByString(column string, str string, contains, ignoreCase bool) ([]int, error) { - ci, err := dt.ColumnIndex(column) - if err != nil { - return nil, err - } - return dt.RowsByStringIndex(ci, str, contains, ignoreCase), nil + dt.Columns.AppendRows(dt2.Columns) } -////////////////////////////////////////////////////////////////////////////////////// +/////////////////////////////////////////////////////////////////////////// // Cell convenience access methods // FloatIndex returns the float64 value of cell at given column, row index @@ -435,7 +334,7 @@ func (dt *Table) FloatIndex(column, row int) float64 { if dt.IsValidRow(row) != nil { return math.NaN() } - ct := dt.Columns[column] + ct := dt.Columns.List.List[column] if ct.NumDims() != 1 { return math.NaN() } @@ -450,14 +349,14 @@ func (dt *Table) Float(column string, row int) float64 { if dt.IsValidRow(row) != nil { return math.NaN() } - ct, err := dt.ColumnByName(column) - if err != nil { + ct := dt.Column(column) + if ct == nil { return math.NaN() } - if ct.NumDims() != 1 { + if ct.Tensor.NumDims() != 1 { return math.NaN() } - return ct.Float1D(row) + return ct.Tensor.Float1D(row) } // StringIndex returns the string value of cell at given column, row index @@ -467,7 +366,7 @@ func (dt *Table) StringIndex(column, row int) string { if dt.IsValidRow(row) != nil { return "" } - ct := dt.Columns[column] + ct := dt.Columns.List.List[column] if ct.NumDims() != 1 { return "" } @@ -483,14 +382,14 @@ func (dt *Table) StringValue(column string, row int) string { if dt.IsValidRow(row) != nil { return "" } - ct, err := dt.ColumnByName(column) - if err != nil { + ct := dt.Column(column) + if ct == nil { return "" } - if ct.NumDims() != 1 { + if ct.Tensor.NumDims() != 1 { return "" } - return ct.String1D(row) + return ct.Tensor.String1D(row) } // TensorIndex returns the tensor SubSpace for given column, row index @@ -502,7 +401,7 @@ func (dt *Table) TensorIndex(column, row int) tensor.Tensor { if dt.IsValidRow(row) != nil { return nil } - ct := dt.Columns[column] + ct := dt.Columns.List.List[column] if ct.NumDims() == 1 { return nil } @@ -517,11 +416,11 @@ func (dt *Table) Tensor(column string, row int) tensor.Tensor { if dt.IsValidRow(row) != nil { return nil } - ct, err := dt.ColumnByName(column) - if err != nil { + ct := dt.Column(column) + if ct == nil { return nil } - if ct.NumDims() == 1 { + if ct.Tensor.NumDims() == 1 { return nil } return ct.SubSpace(row) @@ -536,11 +435,11 @@ func (dt *Table) TensorFloat1D(column string, row int, idx int) float64 { if dt.IsValidRow(row) != nil { return math.NaN() } - ct, err := dt.ColumnByName(column) - if err != nil { + ct := dt.Column(column) + if ct == nil { return math.NaN() } - if ct.NumDims() == 1 { + if ct.Tensor.NumDims() == 1 { return math.NaN() } _, sz := ct.RowCellSize() @@ -548,7 +447,7 @@ func (dt *Table) TensorFloat1D(column string, row int, idx int) float64 { return math.NaN() } off := row*sz + idx - return ct.Float1D(off) + return ct.Tensor.Float1D(off) } ///////////////////////////////////////////////////////////////////////////////////// @@ -560,7 +459,7 @@ func (dt *Table) SetFloatIndex(column, row int, val float64) error { if err := dt.IsValidRow(row); err != nil { return err } - ct := dt.Columns[column] + ct := dt.Columns.List.List[column] if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetFloatIndex: Column %d is a tensor, must use SetTensorFloat1D", column) } @@ -574,14 +473,14 @@ func (dt *Table) SetFloat(column string, row int, val float64) error { if err := dt.IsValidRow(row); err != nil { return err } - ct, err := dt.ColumnByName(column) + ct, err := dt.ColumnTry(column) if err != nil { return err } - if ct.NumDims() != 1 { + if ct.Tensor.NumDims() != 1 { return fmt.Errorf("table.Table SetFloat: Column %s is a tensor, must use SetTensorFloat1D", column) } - ct.SetFloat1D(val, row) + ct.Tensor.SetFloat1D(val, row) return nil } @@ -591,7 +490,7 @@ func (dt *Table) SetStringIndex(column, row int, val string) error { if err := dt.IsValidRow(row); err != nil { return err } - ct := dt.Columns[column] + ct := dt.Columns.List.List[column] if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetStringIndex: Column %d is a tensor, must use SetTensorFloat1D", column) } @@ -605,14 +504,14 @@ func (dt *Table) SetString(column string, row int, val string) error { if err := dt.IsValidRow(row); err != nil { return err } - ct, err := dt.ColumnByName(column) + ct, err := dt.ColumnTry(column) if err != nil { return err } - if ct.NumDims() != 1 { + if ct.Tensor.NumDims() != 1 { return fmt.Errorf("table.Table SetString: Column %s is a tensor, must use SetTensorFloat1D", column) } - ct.SetString1D(val, row) + ct.Tensor.SetString1D(val, row) return nil } @@ -622,7 +521,7 @@ func (dt *Table) SetTensorIndex(column, row int, val tensor.Tensor) error { if err := dt.IsValidRow(row); err != nil { return err } - ct := dt.Columns[column] + ct := dt.Columns.List.List[column] _, csz := ct.RowCellSize() st := row * csz sz := min(csz, val.Len()) @@ -644,8 +543,8 @@ func (dt *Table) SetTensor(column string, row int, val tensor.Tensor) error { if err := dt.IsValidRow(row); err != nil { return err } - ci, err := dt.ColumnIndex(column) - if err != nil { + ci := dt.Columns.IndexByKey(column) + if ci == nil { return err } return dt.SetTensorIndex(ci, row, val) @@ -658,7 +557,7 @@ func (dt *Table) SetTensorFloat1D(column string, row int, idx int, val float64) if err := dt.IsValidRow(row); err != nil { return err } - ct, err := dt.ColumnByName(column) + ct, err := dt.ColumnTry(column) if err != nil { return err } @@ -667,7 +566,7 @@ func (dt *Table) SetTensorFloat1D(column string, row int, idx int, val float64) return fmt.Errorf("table.Table IsValidRow: index %d is out of valid range [0..%d]", idx, sz) } off := row*sz + idx - ct.SetFloat1D(val, off) + ct.Tensor.SetFloat1D(val, off) return nil } @@ -678,21 +577,21 @@ func (dt *Table) SetTensorFloat1D(column string, row int, idx int, val float64) // It is robust to differences in type; uses destination cell type. // Returns error if column names are invalid. func (dt *Table) CopyCell(column string, row int, cpt *Table, cpColNm string, cpRow int) error { - ct, err := dt.ColumnByName(column) + ct, err := dt.ColumnTry(column) if err != nil { return err } - cpct, err := cpt.ColumnByName(cpColNm) + cpct, err := cpt.ColumnTry(cpColNm) if err != nil { return err } _, sz := ct.RowCellSize() if sz == 1 { - if ct.IsString() { + if ct.Tensor.IsString() { ct.SetString1D(cpct.String1D(cpRow), row) return nil } - ct.SetFloat1D(cpct.Float1D(cpRow), row) + ct.Tensor.SetFloat1D(cpct.Float1D(cpRow), row) return nil } _, cpsz := cpct.RowCellSize() diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index dbf34cdc43..d818e0dc12 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -15,7 +15,7 @@ func TestAdd3DCol(t *testing.T) { dt := NewTable() dt.AddFloat32TensorColumn("Values", []int{11, 1, 16}) - col, err := dt.ColumnByName("Values") + col, err := dt.Column("Values") if err != nil { t.Error(err) } diff --git a/tensor/tensor.go b/tensor/tensor.go index 8d7a1e4957..680e993445 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -10,10 +10,18 @@ import ( "fmt" "reflect" + "cogentcore.org/core/base/keylist" "cogentcore.org/core/base/metadata" "gonum.org/v1/gonum/mat" ) +// DataTypes are the primary tensor data types with specific support. +// Any numerical type can also be used. bool is represented using an +// efficient bit slice. +type DataTypes interface { + string | bool | float32 | float64 | int | int32 | byte +} + // todo: add a conversion function to copy data from Column-Major to a tensor: // It is also possible to use Column-Major order, which is used in R, Julia, and MATLAB // where the inner-most index is first and outermost last. @@ -155,6 +163,11 @@ type Tensor interface { // otherwise it goes through the appropriate standard type (Float or String). CopyFrom(from Tensor) + // AppendFrom appends all values from other tensor into this tensor, with an + // optimized implementation if the other tensor is of the same type, and + // otherwise it goes through the appropriate standard type (Float or String). + AppendFrom(from Tensor) error + // SetShapeFrom sets our shape from given source tensor, calling // [Tensor.SetShape] with the shape params from source. SetShapeFrom(from Tensor) @@ -176,7 +189,7 @@ type Tensor interface { // New returns a new n-dimensional tensor of given value type // with the given sizes per dimension (shape). -func New[T string | bool | float32 | float64 | int | int32 | byte](sizes ...int) Tensor { +func New[T DataTypes](sizes ...int) Tensor { var v T switch any(v).(type) { case string: @@ -246,3 +259,6 @@ func CopyDense(to Tensor, dm *mat.Dense) { } } } + +// List is an ordered list of Tensors with name lookup. +type List = keylist.List[string, Tensor] From 6bdeda59fe134ddfcbcdc274ab9474b4375f5810 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 14 Sep 2024 12:14:41 -0700 Subject: [PATCH 032/311] store Keys in keylist -- needed for random access; add RowTensor api for tensors, to replace table sub-tensor api. --- base/keylist/README.md | 7 +- base/keylist/keylist.go | 77 +++++++++----------- base/keylist/keylist_test.go | 34 ++++----- tensor/README.md | 4 +- tensor/bits.go | 17 ++++- tensor/indexed.go | 12 ++++ tensor/number.go | 19 ++++- tensor/string.go | 15 ++++ tensor/table/columns.go | 14 ++-- tensor/table/{indexed.go => indexes.go} | 6 +- tensor/table/io.go | 9 ++- tensor/table/splits.go | 15 ++-- tensor/table/table.go | 96 ++++++++++++++----------- tensor/tensor.go | 15 +++- 14 files changed, 202 insertions(+), 138 deletions(-) rename tensor/table/{indexed.go => indexes.go} (98%) diff --git a/base/keylist/README.md b/base/keylist/README.md index 8bc40bcc83..c89691916a 100644 --- a/base/keylist/README.md +++ b/base/keylist/README.md @@ -1,9 +1,6 @@ # keylist -keylist implements an ordered list (slice) of items, with a map from a key (e.g., names) to indexes, to support fast lookup by name. - -Compared to the [ordmap](../ordmap) package, this is not as efficient for operations such as deletion and insertion, but it has the advantage of providing a simple slice of the target items that can be used directly in many cases. - -Thus, it is more suitable for largely static lists, which are constructed by adding items to the end of the list. +keylist implements an ordered list (slice) of items (Values), with a map from a Key (e.g., names) to indexes, to support fast lookup by name. There is also a Keys slice. +This is a different implementation of the [ordmap](../ordmap) package, and has the advantage of direct slice access to the values, instead of having to go through the KeyValue tuple struct in ordmap. diff --git a/base/keylist/keylist.go b/base/keylist/keylist.go index fb5e510aa3..4f0b0b1a0b 100644 --- a/base/keylist/keylist.go +++ b/base/keylist/keylist.go @@ -6,12 +6,10 @@ package keylist implements an ordered list (slice) of items, with a map from a key (e.g., names) to indexes, to support fast lookup by name. -Compared to the [ordmap] package, this is not as efficient -for operations such as deletion and insertion, but it -has the advantage of providing a simple slice of the target -items that can be used directly in many cases. -Thus, it is more suitable for largely static lists, which -are constructed by adding items to the end of the list. +This is a different implementation of the [ordmap] package, +that has separate slices for Values and Keys, instead of +using a tuple list of both. The awkwardness of value access +through the tuple is the major problem with ordmap. */ package keylist @@ -22,12 +20,14 @@ import ( // TODO: probably want to consolidate ordmap and keylist -// List implements an ordered list (slice) of items, +// List implements an ordered list (slice) of Values, // with a map from a key (e.g., names) to indexes, // to support fast lookup by name. type List[K comparable, V any] struct { //types:add // List is the ordered slice of items. - List []V + Values []V + + Keys []K // indexes is the key-to-index mapping. indexes map[K]int @@ -53,19 +53,11 @@ func (kl *List[K, V]) initIndexes() { // Reset resets the list, removing any existing elements. func (kl *List[K, V]) Reset() { - kl.List = nil + kl.Values = nil + kl.Keys = nil kl.newIndexes() } -// Keys returns the list of keys in List sequential order. -func (kl *List[K, V]) Keys() []K { - keys := make([]K, len(kl.indexes)) - for k, i := range kl.indexes { - keys[i] = k - } - return keys -} - // Add adds an item to the list with given key. // An error is returned if the key is already on the list. // See [AddReplace] for a method that automatically replaces. @@ -74,8 +66,9 @@ func (kl *List[K, V]) Add(key K, val V) error { if _, ok := kl.indexes[key]; ok { return fmt.Errorf("keylist.Add: key %v is already on the list", key) } - kl.indexes[key] = len(kl.List) - kl.List = append(kl.List, val) + kl.indexes[key] = len(kl.Values) + kl.Values = append(kl.Values, val) + kl.Keys = append(kl.Keys, key) return nil } @@ -84,11 +77,13 @@ func (kl *List[K, V]) Add(key K, val V) error { func (kl *List[K, V]) AddReplace(key K, val V) { kl.initIndexes() if idx, ok := kl.indexes[key]; ok { - kl.List[idx] = val + kl.Values[idx] = val + kl.Keys[idx] = key return } - kl.indexes[key] = len(kl.List) - kl.List = append(kl.List, val) + kl.indexes[key] = len(kl.Values) + kl.Values = append(kl.Values, val) + kl.Keys = append(kl.Keys, key) } // Insert inserts the given value with the given key at the given index. @@ -100,11 +95,10 @@ func (kl *List[K, V]) Insert(idx int, key K, val V) error { return fmt.Errorf("keylist.Add: key %v is already on the list", key) } - keys := kl.Keys() - keys = slices.Insert(keys, idx, key) - kl.List = slices.Insert(kl.List, idx, val) + kl.Keys = slices.Insert(kl.Keys, idx, key) + kl.Values = slices.Insert(kl.Values, idx, val) kl.newIndexes() - for i, k := range keys { + for i, k := range kl.Keys { kl.indexes[k] = i } return nil @@ -116,7 +110,7 @@ func (kl *List[K, V]) Insert(idx int, key K, val V) error { func (kl *List[K, V]) ValueByKey(key K) V { idx, ok := kl.indexes[key] if ok { - return kl.List[idx] + return kl.Values[idx] } var zv V return zv @@ -128,16 +122,16 @@ func (kl *List[K, V]) ValueByKey(key K) V { func (kl *List[K, V]) ValueByKeyTry(key K) (V, bool) { idx, ok := kl.indexes[key] if ok { - return kl.List[idx], true + return kl.Values[idx], true } var zv V return zv, false } -// IndexIsValid returns an error if the given index is invalid +// IndexIsValid returns an error if the given index is invalid. func (kl *List[K, V]) IndexIsValid(idx int) error { - if idx >= len(kl.List) || idx < 0 { - return fmt.Errorf("keylist.List: IndexIsValid: index %d is out of range of a list of length %d", idx, len(kl.List)) + if idx >= len(kl.Values) || idx < 0 { + return fmt.Errorf("keylist.List: IndexIsValid: index %d is out of range of a list of length %d", idx, len(kl.Values)) } return nil } @@ -156,7 +150,7 @@ func (kl *List[K, V]) Len() int { if kl == nil { return 0 } - return len(kl.List) + return len(kl.Values) } // DeleteIndex deletes item(s) within the index range [i:j]. @@ -167,11 +161,10 @@ func (kl *List[K, V]) DeleteIndex(i, j int) { if ndel <= 0 { panic("index range is <= 0") } - keys := kl.Keys() - keys = slices.Delete(keys, i, j) - kl.List = slices.Delete(kl.List, i, j) + kl.Keys = slices.Delete(kl.Keys, i, j) + kl.Values = slices.Delete(kl.Values, i, j) kl.newIndexes() - for i, k := range keys { + for i, k := range kl.Keys { kl.indexes[k] = i } @@ -195,18 +188,16 @@ func (kl *List[K, V]) DeleteKey(key K) bool { // list unless they also exist in the given list, in which case // they are overwritten. Use [Reset] first to get an exact copy. func (kl *List[K, V]) Copy(from *List[K, V]) { - keys := from.Keys() - for i, v := range from.List { - kl.AddReplace(keys[i], v) + for i, v := range from.Values { + kl.AddReplace(kl.Keys[i], v) } } // String returns a string representation of the list. func (kl *List[K, V]) String() string { sv := "{" - keys := kl.Keys() - for i, v := range kl.List { - sv += fmt.Sprintf("%v", keys[i]) + ": " + fmt.Sprintf("%v", v) + ", " + for i, v := range kl.Values { + sv += fmt.Sprintf("%v", kl.Keys[i]) + ": " + fmt.Sprintf("%v", v) + ", " } sv += "}" return sv diff --git a/base/keylist/keylist_test.go b/base/keylist/keylist_test.go index e5c1a3e9e1..e880a899e7 100644 --- a/base/keylist/keylist_test.go +++ b/base/keylist/keylist_test.go @@ -11,29 +11,29 @@ import ( ) func TestKeyList(t *testing.T) { - om := New[string, int]() - om.Add("key0", 0) - om.Add("key1", 1) - om.Add("key2", 2) + kl := New[string, int]() + kl.Add("key0", 0) + kl.Add("key1", 1) + kl.Add("key2", 2) - assert.Equal(t, 1, om.ValueByKey("key1")) - assert.Equal(t, 2, om.IndexByKey("key2")) + assert.Equal(t, 1, kl.ValueByKey("key1")) + assert.Equal(t, 2, kl.IndexByKey("key2")) - assert.Equal(t, 1, om.List[1]) + assert.Equal(t, 1, kl.Values[1]) - assert.Equal(t, 3, om.Len()) + assert.Equal(t, 3, kl.Len()) - om.DeleteIndex(1, 2) - assert.Equal(t, 2, om.List[1]) - assert.Equal(t, 1, om.IndexByKey("key2")) + kl.DeleteIndex(1, 2) + assert.Equal(t, 2, kl.Values[1]) + assert.Equal(t, 1, kl.IndexByKey("key2")) - om.Insert(0, "new0", 3) - assert.Equal(t, 3, om.List[0]) - assert.Equal(t, 0, om.List[1]) - assert.Equal(t, 2, om.IndexByKey("key2")) + kl.Insert(0, "new0", 3) + assert.Equal(t, 3, kl.Values[0]) + assert.Equal(t, 0, kl.Values[1]) + assert.Equal(t, 2, kl.IndexByKey("key2")) // nm := Make([]KeyValue[string, int]{{"one", 1}, {"two", 2}, {"three", 3}}) - // assert.Equal(t, 3, nm.List[2]) - // assert.Equal(t, 2, nm.List[1]) + // assert.Equal(t, 3, nm.Values[2]) + // assert.Equal(t, 2, nm.Values[1]) // assert.Equal(t, 3, nm.ValueByKey("three")) } diff --git a/tensor/README.md b/tensor/README.md index 1f6d9966b7..1a080a1050 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -35,7 +35,9 @@ All tensor package functions are registered using a single name to function map - TODO: where? [pca](pca) provides principal-components-analysis (PCA) and covariance matrix computation functions. - TODO: in metric? [clust](clust) provides standard agglomerative hierarchical clustering including ability to plot results in an eplot. -# Standard shapes +# Standard shapes and dimensional terminology + +In general, **1D** refers to a flat, 1-dimensional list, and **ND** is used by contrast to refer to the _2+_ dimensional case. There are various standard shapes of tensor data that different functions expect: diff --git a/tensor/bits.go b/tensor/bits.go index 43f8842803..c144db1b77 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -17,8 +17,9 @@ import ( "gonum.org/v1/gonum/mat" ) -// Bits is a tensor of bits backed by a bitslice.Slice for efficient storage -// of binary data +// Bits is a tensor of bits backed by a [bitslice.Slice] for efficient storage +// of binary data. Bits does not support [Tensor.SubSpace] access and related +// methods due to the nature of the underlying data representation. type Bits struct { shape Shape Values bitslice.Slice @@ -135,11 +136,21 @@ func (tsr *Bits) SetNumRows(rows int) { tsr.Values.SetLen(nln) } -// SubSpace is not possible with Bits +// SubSpace is not possible with Bits. func (tsr *Bits) SubSpace(offs ...int) Tensor { return nil } +// RowTensor not possible with Bits. +func (tsr *Bits) RowTensor(row int) Tensor { + return nil +} + +// SetRowTensor not possible with Bits. +func (tsr *Bits) SetRowTensor(val Tensor, row int) { + +} + func (tsr *Bits) Float(i ...int) float64 { return BoolToFloat64(tsr.Values.Index(tsr.shape.Offset(i...))) } diff --git a/tensor/indexed.go b/tensor/indexed.go index 1d3a5be3a3..4e5e91ec79 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -520,3 +520,15 @@ func (ix *Indexed) SubSpace(offs ...int) Tensor { func (ix *Indexed) Cells1D(row int) *Indexed { return NewIndexed(New1DViewOf(ix.SubSpace(row))) } + +// RowTensor is a convenience version of [Indexed.SubSpace] to return the +// SubSpace for the outermost row dimension, indirected through the indexes. +func (ix *Indexed) RowTensor(row int) Tensor { + return ix.Tensor.RowTensor(ix.Index(row)) +} + +// SetRowTensor sets the values of the SubSpace at given row to given values, +// with row indirected through the indexes. +func (ix *Indexed) SetRowTensor(val Tensor, row int) { + ix.Tensor.SetRowTensor(val, ix.Index(row)) +} diff --git a/tensor/number.go b/tensor/number.go index 070f0c95cd..f31c61b741 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -315,12 +315,12 @@ func (tsr *Number[T]) SetShapeFrom(frm Tensor) { // of the same type, and otherwise it goes through appropriate standard type. func (tsr *Number[T]) CopyCellsFrom(frm Tensor, to, start, n int) { if fsm, ok := frm.(*Number[T]); ok { - for i := 0; i < n; i++ { + for i := range n { tsr.Values[to+i] = fsm.Values[start+i] } return } - for i := 0; i < n; i++ { + for i := range n { tsr.Values[to+i] = T(frm.Float1D(start + i)) } } @@ -336,3 +336,18 @@ func (tsr *Number[T]) SubSpace(offs ...int) Tensor { rt := &Number[T]{Base: *b} return rt } + +// RowTensor is a convenience version of [Tensor.SubSpace] to return the +// SubSpace for the outermost row dimension. [Indexed] defines a version +// of this that indirects through the row indexes. +func (tsr *Number[T]) RowTensor(row int) Tensor { + return tsr.SubSpace(row) +} + +// SetRowTensor sets the values of the SubSpace at given row to given values. +func (tsr *Number[T]) SetRowTensor(val Tensor, row int) { + _, cells := tsr.RowCellSize() + st := row * cells + mx := min(val.Len(), cells) + tsr.CopyCellsFrom(val, st, 0, mx) +} diff --git a/tensor/string.go b/tensor/string.go index 05b08d780c..da5666be0e 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -272,3 +272,18 @@ func (tsr *String) SubSpace(offs ...int) Tensor { rt := &String{Base: *b} return rt } + +// RowTensor is a convenience version of [Tensor.SubSpace] to return the +// SubSpace for the outermost row dimension. [Indexed] defines a version +// of this that indirects through the row indexes. +func (tsr *String) RowTensor(row int) Tensor { + return tsr.SubSpace(row) +} + +// SetRowTensor sets the values of the SubSpace at given row to given values. +func (tsr *String) SetRowTensor(val Tensor, row int) { + _, cells := tsr.RowCellSize() + st := row * cells + mx := min(val.Len(), cells) + tsr.CopyCellsFrom(val, st, 0, mx) +} diff --git a/tensor/table/columns.go b/tensor/table/columns.go index 98d57b3f92..cda3b500d1 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -38,7 +38,7 @@ func (cl *Columns) IsValidRow(row int) error { func (cl *Columns) SetNumRows(rows int) *Columns { //types:add cl.Rows = rows // can be 0 rows = max(1, rows) - for _, tsr := range cl.List.List { + for _, tsr := range cl.Values { tsr.SetNumRows(rows) } return cl @@ -73,23 +73,21 @@ func (cl *Columns) InsertColumn(idx int, name string, tsr tensor.Tensor) error { // Clone returns a complete copy of this set of columns. func (cl *Columns) Clone() *Columns { cp := NewColumns().SetNumRows(cl.Rows) - keys := cl.Keys() - for i, nm := range keys { - tsr := cl.List.List[i] + for i, nm := range cl.Keys { + tsr := cl.Values[i] cp.AddColumn(nm, tsr.Clone()) } return cl } -// AppendRows appends shared columns in both tables with input table rows +// AppendRows appends shared columns in both tables with input table rows. func (cl *Columns) AppendRows(cl2 *Columns) { - keys := cl.Keys() - for i, nm := range keys { + for i, nm := range cl.Keys { c2 := cl2.ValueByKey(nm) if c2 == nil { continue } - c1 := cl.List.List[i] + c1 := cl.Values[i] c1.AppendFrom(c2) } cl.SetNumRows(cl.Rows + cl2.Rows) diff --git a/tensor/table/indexed.go b/tensor/table/indexes.go similarity index 98% rename from tensor/table/indexed.go rename to tensor/table/indexes.go index 4477039783..b8b8b1cc05 100644 --- a/tensor/table/indexed.go +++ b/tensor/table/indexes.go @@ -148,7 +148,7 @@ func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { } sf(func(dt *Table, i, j int) int { for _, ci := range colIndexes { - cl := dt.Columns.List.List[ci] + cl := dt.Columns.Values[ci] if cl.IsString() { if ascending { if cl.String1D(i) < cl.String1D(j) { @@ -279,8 +279,8 @@ func (dt *Table) NewTable() *Table { if rows == 0 { return nt } - for ci, cl := range nt.Columns.List.List { - scl := dt.Columns.List.List[ci] + for ci, cl := range nt.Columns.Values { + scl := dt.Columns.Values[ci] _, csz := cl.RowCellSize() for i, srw := range dt.Indexes { cl.CopyCellsFrom(scl, i*csz, srw*csz, csz) diff --git a/tensor/table/io.go b/tensor/table/io.go index 6ba3b0e9c9..0f59bb06b8 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -123,7 +123,7 @@ func (dt *Table) ReadCSVRow(rec []string, row int) { ci++ } nan := math.NaN() - for _, tsr := range dt.Columns.List.List { + for _, tsr := range dt.Columns.Values { _, csz := tsr.RowCellSize() stoff := row * csz for cc := 0; cc < csz; cc++ { @@ -387,7 +387,7 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { rec = make([]string, 0) } rc := 0 - for _, tsr := range dt.Columns.List.List { + for _, tsr := range dt.Columns.Values { nd := tsr.NumDims() if nd == 1 { vl := "" @@ -429,9 +429,8 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { // with full information about type and tensor cell dimensionality. func (dt *Table) TableHeaders() []string { hdrs := []string{} - names := dt.Columns.Keys() - for i, nm := range names { - tsr := dt.Columns.List.List[i] + for i, nm := range dt.Columns.Keys { + tsr := dt.Columns.Values[i] nm = string([]byte{TableHeaderChar(tsr.DataType())}) + nm if tsr.NumDims() == 1 { hdrs = append(hdrs, nm) diff --git a/tensor/table/splits.go b/tensor/table/splits.go index 4489b5f53b..99aca74d45 100644 --- a/tensor/table/splits.go +++ b/tensor/table/splits.go @@ -397,7 +397,7 @@ func (spl *Splits) AggsToTable(colName bool) *Table { } } for _, ag := range spl.Aggs { - col := dt.Columns.List.List[ag.ColumnIndex] + col := dt.Columns.Values[ag.ColumnIndex] an := "" // dt.ColumnNames[ag.ColumnIndex] // todo: need fast access! if colName == AddAggName { an += ":" + ag.Name @@ -407,12 +407,12 @@ func (spl *Splits) AggsToTable(colName bool) *Table { for si := range spl.Splits { cidx := 0 for ci := range spl.Levels { - col := st.Columns.List.List[cidx] + col := st.Columns.Values[cidx] col.SetString1D(spl.Values[si][ci], si) cidx++ } for _, ag := range spl.Aggs { - col := st.Columns.List.List[cidx] + col := st.Columns.Values[cidx] _, csz := col.RowCellSize() sti := si * csz av := ag.Aggs[si] @@ -445,7 +445,7 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { exmap[cn] = struct{}{} } for _, ag := range spl.Aggs { - col := dt.Columns.List.List[ag.ColumnIndex] + col := dt.Columns.Values[ag.ColumnIndex] an := "" // dt.ColumnNames[ag.ColumnIndex] exmap[an] = struct{}{} if colName == AddAggName { @@ -454,8 +454,7 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:]...) } var cpcol []string - names := dt.Columns.Keys() - for _, cn := range names { + for _, cn := range dt.Columns.Keys { if _, ok := exmap[cn]; !ok { cpcol = append(cpcol, cn) col := dt.Column(cn) @@ -465,12 +464,12 @@ func (spl *Splits) AggsToTableCopy(colName bool) *Table { for si, sidx := range spl.Splits { cidx := 0 for ci := range spl.Levels { - col := st.Columns.List.List[cidx] + col := st.Columns.Values[cidx] col.SetString1D(spl.Values[si][ci], si) cidx++ } for _, ag := range spl.Aggs { - col := st.Columns.List.List[cidx] + col := st.Columns.Values[cidx] _, csz := col.RowCellSize() sti := si * csz av := ag.Aggs[si] diff --git a/tensor/table/table.go b/tensor/table/table.go index a47e4809f3..a8acae39c6 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -17,14 +17,14 @@ import ( ) // Table is a table of Tensor columns aligned by a common outermost row dimension. -// Use the Column (by name) and ColumnIndex methods to obtain a `tensor.Indexed` view of -// the column, using the shared Indexes of the Table. Thus, a coordinated -// sorting and filtered view of the column data is automatically available for -// any of the tensor package functions that use `tensor.Indexed` as the one +// Use the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a +// [tensor.Indexed] view of the column, using the shared [Table.Indexes] of the Table. +// Thus, a coordinated sorting and filtered view of the column data is automatically +// available for any of the tensor package functions that use [tensor.Indexed] as the one // common data representation for all operations. type Table struct { //types:add // Columns has the list of column tensor data for this table. - // Different tables can have + // Different tables can provide different indexed views onto the same Columns. Columns *Columns // Indexes are the indexes into Tensor rows, with nil = sequential. @@ -42,8 +42,8 @@ type Table struct { //types:add Meta metadata.Data } -// NewTable returnes a new Table with its own (empty) set of Columns. -// Can pass an optional name. +// NewTable returns a new Table with its own (empty) set of Columns. +// Can pass an optional name which sets metadata. func NewTable(name ...string) *Table { dt := &Table{} dt.Columns = NewColumns() @@ -62,7 +62,7 @@ func NewTableView(src *Table) *Table { return dt } -// IsValidRow returns error if the row is invalid +// IsValidRow returns error if the row is invalid, if error checking is needed. func (dt *Table) IsValidRow(row int) error { if row < 0 || row >= dt.Rows() { return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, dt.Rows()) @@ -77,7 +77,7 @@ func (dt *Table) NumRows() int { return dt.Columns.Rows } func (dt *Table) NumColumns() int { return dt.Columns.Len() } // Column returns the tensor with given column name, as a [tensor.Indexed] -// with the shared [Indexes] from this table. It is best practice to +// with the shared [Table.Indexes] from this table. It is best practice to // access columns by name, and direct access through Columns does not // provide the shared table-wide Indexes. // Returns nil if not found. @@ -89,7 +89,7 @@ func (dt *Table) Column(name string) *tensor.Indexed { return tensor.NewIndexed(cl, dt.Indexes) } -// ColumnTry is a version of [Column] that also returns an error +// ColumnTry is a version of [Table.Column] that also returns an error // if the column name is not found, for cases when error is needed. func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { cl := dt.Column(name) @@ -100,12 +100,12 @@ func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { } // ColumnIndex returns the tensor at the given index, as a [tensor.Indexed] -// with the shared [Indexes] from this table. It is best practice to -// access columns by name using [Column] method instead. +// with the shared [Table.Indexes] from this table. It is best practice to +// access columns by name using [Table.Column] method instead. // Direct access through Columns does not provide the shared table-wide Indexes. // Returns nil if not found. func (dt *Table) ColumnIndex(idx int) *tensor.Indexed { - cl := dt.Columns.List.List[idx] + cl := dt.Columns.Values[idx] return tensor.NewIndexed(cl, dt.Indexes) } @@ -132,12 +132,12 @@ func (dt *Table) ColumnIndexesByNames(names ...string) ([]int, error) { // ColumnName returns the name of given column func (dt *Table) ColumnName(i int) string { - return dt.ColumnNames[i] + return dt.Columns.Keys[i] } // AddColumn adds a new column to the table, of given type and column name // (which must be unique). The cells of this column hold a single scalar value: -// see AddColumnTensor for n-dimensional cells. +// see [Table.AddColumnTensor] for n-dimensional cells. func AddColumn[T tensor.DataTypes](dt *Table, name string) tensor.Tensor { rows := max(1, dt.Columns.Rows) tsr := tensor.New[T](rows) @@ -275,40 +275,47 @@ func (dt *Table) DeleteAll() { } // AddRows adds n rows to end of underlying Table, and to the indexes in this view -func (dt *Table) AddRows(n int) { //types:add - stidx := dt.Columns.Rows - dt.Columns.SetNumRows(stidx + n) - for i := stidx; i < stidx+n; i++ { - dt.Indexes = append(dt.Indexes, i) - } +func (dt *Table) AddRows(n int) *Table { //types:add + dt.Columns.SetNumRows(dt.Columns.Rows + n) + return dt } // InsertRows adds n rows to end of underlying Table, and to the indexes starting at -// given index in this view -func (dt *Table) InsertRows(at, n int) { - stidx := dt.Columns.Rows - dt.Columns.SetNumRows(stidx + n) - if dt.Indexes != nil { - nw := make([]int, n, n+len(dt.Indexes)-at) - for i := 0; i < n; i++ { - nw[i] = stidx + i - } - dt.Indexes = append(dt.Indexes[:at], append(nw, dt.Indexes[at:]...)...) - } +// given index in this view, providing an efficient insertion operation that only +// exists in the indexed view. To create an in-memory ordering, use [Table.NewTable]. +func (dt *Table) InsertRows(at, n int) *Table { + dt.IndexesNeeded() + strow := dt.Columns.Rows + stidx := len(dt.Indexes) + dt.Columns.SetNumRows(strow + n) // adds n indexes to end of list + // move those indexes to at:at+n in index list + dt.Indexes = append(dt.Indexes[:at], append(dt.Indexes[stidx:], dt.Indexes[at:]...)...) } // SetNumRows sets the number of rows in the table, across all columns // if rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0 func (dt *Table) SetNumRows(rows int) *Table { //types:add + strow := dt.Columns.Rows dt.Columns.SetNumRows(rows) + if dt.Indexes == nil { + return dt + } + if rows > strow { + for i := range rows - strow { + dt.Indexes = append(dt.Indexes, strow+i) + } + } else { + dt.DeleteInvalid() + } return dt } // note: no really clean definition of CopyFrom -- no point of re-using existing // table -- just clone it. -// Clone returns a complete copy of this table, -// including cloning the underlying Columns tensors. +// Clone returns a complete copy of this table, including cloning +// the underlying Columns tensors, and the current [Table.Indexes]. +// See also [Table.NewTable] to flatten the current indexes. func (dt *Table) Clone() *Table { cp := &Table{} cp.Columns = dt.Columns.Clone() @@ -321,7 +328,16 @@ func (dt *Table) Clone() *Table { // AppendRows appends shared columns in both tables with input table rows func (dt *Table) AppendRows(dt2 *Table) { + strow := dt.Columns.Rows + n := dt2.Columns.Rows dt.Columns.AppendRows(dt2.Columns) + if dt.Indexes == nil { + return + } + if rows > strow { + for i := range rows - strow { + dt.Indexes = append(dt.Indexes, strow+i) + } } /////////////////////////////////////////////////////////////////////////// @@ -334,7 +350,7 @@ func (dt *Table) FloatIndex(column, row int) float64 { if dt.IsValidRow(row) != nil { return math.NaN() } - ct := dt.Columns.List.List[column] + ct := dt.Columns.Values[column] if ct.NumDims() != 1 { return math.NaN() } @@ -366,7 +382,7 @@ func (dt *Table) StringIndex(column, row int) string { if dt.IsValidRow(row) != nil { return "" } - ct := dt.Columns.List.List[column] + ct := dt.Columns.Values[column] if ct.NumDims() != 1 { return "" } @@ -401,7 +417,7 @@ func (dt *Table) TensorIndex(column, row int) tensor.Tensor { if dt.IsValidRow(row) != nil { return nil } - ct := dt.Columns.List.List[column] + ct := dt.Columns.Values[column] if ct.NumDims() == 1 { return nil } @@ -459,7 +475,7 @@ func (dt *Table) SetFloatIndex(column, row int, val float64) error { if err := dt.IsValidRow(row); err != nil { return err } - ct := dt.Columns.List.List[column] + ct := dt.Columns.Values[column] if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetFloatIndex: Column %d is a tensor, must use SetTensorFloat1D", column) } @@ -490,7 +506,7 @@ func (dt *Table) SetStringIndex(column, row int, val string) error { if err := dt.IsValidRow(row); err != nil { return err } - ct := dt.Columns.List.List[column] + ct := dt.Columns.Values[column] if ct.NumDims() != 1 { return fmt.Errorf("table.Table SetStringIndex: Column %d is a tensor, must use SetTensorFloat1D", column) } @@ -521,7 +537,7 @@ func (dt *Table) SetTensorIndex(column, row int, val tensor.Tensor) error { if err := dt.IsValidRow(row); err != nil { return err } - ct := dt.Columns.List.List[column] + ct := dt.Columns.Values[column] _, csz := ct.RowCellSize() st := row * csz sz := min(csz, val.Len()) diff --git a/tensor/tensor.go b/tensor/tensor.go index 680e993445..111d3cb08b 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -87,8 +87,6 @@ type Tensor interface { // SetFloat sets the value of given n-dimensional index (matching Shape) as a float64. SetFloat(val float64, i ...int) - // NOTE: String conflicts with [fmt.Stringer], so we have to use StringValue - // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. Float1D(i int) float64 @@ -107,7 +105,10 @@ type Tensor interface { // and use this interface extensively. SetFloatRowCell(val float64, row, cell int) + ///////////////////// Strings + // StringValue returns the value of given n-dimensional index (matching Shape) as a string. + // 'String' conflicts with [fmt.Stringer], so we have to use StringValue here. StringValue(i ...int) string // SetString sets the value of given n-dimensional index (matching Shape) as a string. @@ -138,6 +139,14 @@ type Tensor interface { // extract arbitrary subspaces along ranges of each dimension. SubSpace(offs ...int) Tensor + // RowTensor is a convenience version of [Tensor.SubSpace] to return the + // SubSpace for the outermost row dimension. [Indexed] defines a version + // of this that indirects through the row indexes. + RowTensor(row int) Tensor + + // SetRowTensor sets the values of the [Tensor.SubSpace] at given row to given values. + SetRowTensor(val Tensor, row int) + // Range returns the min, max (and associated indexes, -1 = no values) for the tensor. // This is needed for display and is thus in the core api in optimized form. // See the [stats] package for many more statistics functions. @@ -153,7 +162,7 @@ type Tensor interface { Clone() Tensor // View clones this tensor, *keeping the same underlying Values slice*, - // instead of making a copy like Clone() does. A major use of this + // instead of making a copy like [Tensor.Clone] does. A major use of this // is to then change the shape of the view to provide a different way // of accessing the same data. See [New1DViewOf] for example. View() Tensor From 095565bd2b33c2f061a1c98dfd5dde1bcfe25f3b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 14 Sep 2024 15:10:58 -0700 Subject: [PATCH 033/311] new table api in place without any redundant tensor accessors, building, tests working, etc --- base/randx/dists_test.go | 10 +- tensor/cmd/tablecat/tablecat.go | 2 +- tensor/funcs_test.go | 4 +- tensor/indexed.go | 42 ++- tensor/slice.go | 4 +- tensor/stats/cluster/clust_test.go | 13 +- tensor/stats/cluster/cluster.go | 4 +- tensor/stats/cluster/plot.go | 29 +- tensor/stats/metric/funcs.go | 62 ++-- tensor/stats/metric/matrix.go | 10 +- tensor/stats/metric/metric_test.go | 12 +- tensor/stats/metric/misc.go | 6 +- tensor/stats/metric/vec.go | 62 ++-- tensor/{table => stats/split}/splits.go | 0 tensor/stats/stats/funcs.go | 56 ++-- tensor/stats/stats/quantiles.go | 4 +- tensor/stats/stats/table.go | 8 +- tensor/stats/stats/vec.go | 22 +- tensor/table/io.go | 3 +- tensor/table/io_test.go | 33 +- tensor/table/slicetable.go | 46 +-- tensor/table/slicetable_test.go | 2 +- tensor/table/table.go | 420 ++---------------------- tensor/table/table_test.go | 31 +- tensor/table/util.go | 12 +- tensor/tmath/math.go | 74 ++--- tensor/tmath/math_test.go | 6 +- tensor/tmath/norm.go | 12 +- tensor/tmath/ops.go | 28 +- tensor/tmath/ops_test.go | 52 +-- 30 files changed, 362 insertions(+), 707 deletions(-) rename tensor/{table => stats/split}/splits.go (100%) diff --git a/base/randx/dists_test.go b/base/randx/dists_test.go index f2c8764873..63fedb24df 100644 --- a/base/randx/dists_test.go +++ b/base/randx/dists_test.go @@ -15,7 +15,7 @@ import ( func TestGaussianGen(t *testing.T) { nsamp := int(1e6) - dt := &table.Table{} + dt := table.NewTable() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -51,7 +51,7 @@ func TestGaussianGen(t *testing.T) { func TestBinomialGen(t *testing.T) { nsamp := int(1e6) - dt := &table.Table{} + dt := table.NewTable() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -97,7 +97,7 @@ func TestBinomialGen(t *testing.T) { func TestPoissonGen(t *testing.T) { nsamp := int(1e6) - dt := &table.Table{} + dt := table.NewTable() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -142,7 +142,7 @@ func TestPoissonGen(t *testing.T) { func TestGammaGen(t *testing.T) { nsamp := int(1e6) - dt := &table.Table{} + dt := table.NewTable() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -178,7 +178,7 @@ func TestGammaGen(t *testing.T) { func TestBetaGen(t *testing.T) { nsamp := int(1e6) - dt := &table.Table{} + dt := table.NewTable() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index 77092ef86f..96eb32fcb8 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -110,7 +110,7 @@ func RawCat(files []string) { func AvgCat(files []string) { dts := make([]*table.Table, 0, len(files)) for _, fn := range files { - dt := &table.Table{} + dt := table.NewTable() err := dt.OpenCSV(core.Filename(fn), table.Tab) if err != nil { fmt.Println("Error opening file: ", err) diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index decdf5732c..1f97e43161 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -15,7 +15,7 @@ func abs(in, out *Indexed) { out.SetShapeFrom(in) VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...*Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Abs(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(i)), i) }, in, out) } @@ -37,7 +37,7 @@ func TestFuncs(t *testing.T) { err = Call("Abs", oned, oneout) assert.NoError(t, err) - assert.Equal(t, 1.507556722888818, oneout.Tensor.Float1D(0)) + assert.Equal(t, 1.507556722888818, oneout.Float1D(0)) err = Call("Abs", oned) assert.Error(t, err) diff --git a/tensor/indexed.go b/tensor/indexed.go index 4e5e91ec79..d1d820482f 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -182,7 +182,7 @@ func (ix *Indexed) ExcludeMissing1D() { //types:add ix.IndexesNeeded() ni := ix.Rows() for i := ni - 1; i >= 0; i-- { - if math.IsNaN(ix.Tensor.Float1D(ix.Indexes[i])) { + if math.IsNaN(ix.Float1D(ix.Indexes[i])) { ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) } } @@ -433,7 +433,7 @@ func (ix *Indexed) Float(i ...int) float64 { } // SetFloat sets the value of given index as a float64 -// The first index value is indirected through the [Indexes]. +// The first index value is indirected through the [Indexed.Indexes]. func (ix *Indexed) SetFloat(val float64, i ...int) { if ix.Indexes == nil { ix.Tensor.SetFloat(val, i...) @@ -446,7 +446,7 @@ func (ix *Indexed) SetFloat(val float64, i ...int) { // FloatRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexes]. +// Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) FloatRowCell(row, cell int) float64 { return ix.Tensor.FloatRowCell(ix.Index(row), cell) @@ -454,12 +454,26 @@ func (ix *Indexed) FloatRowCell(row, cell int) float64 { // SetFloatRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexes]. +// Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { ix.Tensor.SetFloatRowCell(val, ix.Index(row), cell) } +// Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. +// This is just a convenience pass-through to the Tensor, and does _not_ use +// the [Indexed.Indexes]. +func (ix *Indexed) Float1D(i int) float64 { + return ix.Tensor.Float1D(i) +} + +// SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. +// This is just a convenience pass-through to the Tensor, and does _not_ use +// the [Indexed.Indexes]. +func (ix *Indexed) SetFloat1D(val float64, i int) { + ix.Tensor.SetFloat1D(val, i) +} + // StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. func (ix *Indexed) StringValue(i ...int) string { @@ -472,7 +486,7 @@ func (ix *Indexed) StringValue(i ...int) string { } // SetString sets the value of given index as a string -// The first index value is indirected through the [Indexes]. +// The first index value is indirected through the [Indexed.Indexes]. func (ix *Indexed) SetString(val string, i ...int) { if ix.Indexes == nil { ix.Tensor.SetString(val, i...) @@ -484,7 +498,7 @@ func (ix *Indexed) SetString(val string, i ...int) { // StringRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexes]. +// Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) StringRowCell(row, cell int) string { return ix.Tensor.StringRowCell(ix.Index(row), cell) @@ -492,12 +506,26 @@ func (ix *Indexed) StringRowCell(row, cell int) string { // SetStringRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexes]. +// Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetStringRowCell(val string, row, cell int) { ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) } +// String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. +// This is just a convenience pass-through to the Tensor, and does _not_ use +// the [Indexed.Indexes]. +func (ix *Indexed) String1D(i int) string { + return ix.Tensor.String1D(i) +} + +// SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. +// This is just a convenience pass-through to the Tensor, and does _not_ use +// the [Indexed.Indexes]. +func (ix *Indexed) SetString1D(val string, i int) { + ix.Tensor.SetString1D(val, i) +} + // SubSpace returns a new tensor with innermost subspace at given // offset(s) in outermost dimension(s) (len(offs) < NumDims). // The new tensor points to the values of the this tensor (i.e., modifications diff --git a/tensor/slice.go b/tensor/slice.go index a64e307dfb..669f5cf4da 100644 --- a/tensor/slice.go +++ b/tensor/slice.go @@ -117,7 +117,7 @@ func Slice(tsr, out *Indexed, ranges ...Range) error { if out.Tensor.IsString() { out.Tensor.SetString1D(tsr.Tensor.String1D(oi), ni) } else { - out.Tensor.SetFloat1D(tsr.Tensor.Float1D(oi), ni) + out.SetFloat1D(tsr.Float1D(oi), ni) } } return nil @@ -157,7 +157,7 @@ func SliceSet(tsr, slc *Indexed, ranges ...Range) error { if slc.Tensor.IsString() { tsr.Tensor.SetString1D(slc.Tensor.String1D(ni), oi) } else { - tsr.Tensor.SetFloat1D(slc.Tensor.Float1D(ni), oi) + tsr.SetFloat1D(slc.Float1D(ni), oi) } } return nil diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 90f4b70138..1bc7b63655 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -7,7 +7,6 @@ package cluster import ( "testing" - "cogentcore.org/core/base/errors" "cogentcore.org/core/base/tolassert" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/stats/metric" @@ -29,16 +28,16 @@ var clustres = ` 3.605551275463989: Wendy_sad Wendy_happy ` func TestClust(t *testing.T) { - dt := &table.Table{} - err := dt.OpenCSV("testdata/faces.dat", table.Tab) + dt := table.NewTable() + err := dt.OpenCSV("testdata/faces.dat", tensor.Tab) if err != nil { t.Error(err) } - in := tensor.NewIndexed(errors.Log1(dt.ColumnByName("Input"))) - out := tensor.NewIndexed(tensor.NewFloat64()) - metric.Matrix(metric.Euclidean.String(), in, out) + in := dt.Column("Input") + out := tensor.NewFloat64Indexed() + metric.Matrix(metric.Euclidean.FuncName(), in, out) - cl := Cluster(Avg.String(), out, tensor.NewIndexed(errors.Log1(dt.ColumnByName("Name")))) + cl := Cluster(Avg.String(), out, dt.Column("Name")) var dists []float64 diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index 73dd570b7b..d504ce02de 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -113,9 +113,9 @@ func InitAllLeaves(ntot int) *Node { // to start with predefined clusters. func Glom(root *Node, funcName string, dmat *tensor.Indexed) *Node { ntot := dmat.Tensor.DimSize(0) // number of leaves - mout := tensor.NewFloatScalar(0) + mout := tensor.NewFloat64Scalar(0) stats.MaxFunc(tensor.NewIndexed(tensor.New1DViewOf(dmat.Tensor)), mout) - maxd := mout.Tensor.Float1D(0) + maxd := mout.Float1D(0) // indexes in each group aidx := make([]int, ntot) bidx := make([]int, ntot) diff --git a/tensor/stats/cluster/plot.go b/tensor/stats/cluster/plot.go index 384f796c70..48a535ebf1 100644 --- a/tensor/stats/cluster/plot.go +++ b/tensor/stats/cluster/plot.go @@ -27,32 +27,35 @@ func Plot(pt *table.Table, root *Node, dmat, labels *tensor.Indexed) { // will render this node in a cluster plot when plotted with a standard plotting package. // The lines double-back on themselves to form a continuous line to be plotted. func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { - row := pt.Rows + row := pt.Rows() + xc := pt.ColumnIndex(0) + yc := pt.ColumnIndex(1) + lbl := pt.ColumnIndex(2) if nn.IsLeaf() { pt.SetNumRows(row + 1) - pt.SetFloatIndex(0, row, nn.ParDist) - pt.SetFloatIndex(1, row, nn.Y) + xc.SetFloatRowCell(nn.ParDist, row, 0) + yc.SetFloatRowCell(nn.Y, row, 0) if labels.Len() > nn.Index { - pt.SetStringIndex(2, row, labels.StringValue(nn.Index)) + lbl.SetStringRowCell(labels.StringValue(nn.Index), row, 0) } } else { for _, kn := range nn.Kids { pt.SetNumRows(row + 2) - pt.SetFloatIndex(0, row, nn.ParDist) - pt.SetFloatIndex(1, row, kn.Y) + xc.SetFloatRowCell(nn.ParDist, row, 0) + yc.SetFloatRowCell(kn.Y, row, 0) row++ - pt.SetFloatIndex(0, row, nn.ParDist+nn.Dist) - pt.SetFloatIndex(1, row, kn.Y) + xc.SetFloatRowCell(nn.ParDist+nn.Dist, row, 0) + yc.SetFloatRowCell(kn.Y, row, 0) kn.Plot(pt, dmat, labels) - row = pt.Rows + row = pt.Rows() pt.SetNumRows(row + 1) - pt.SetFloatIndex(0, row, nn.ParDist) - pt.SetFloatIndex(1, row, kn.Y) + xc.SetFloatRowCell(nn.ParDist, row, 0) + yc.SetFloatRowCell(kn.Y, row, 0) row++ } pt.SetNumRows(row + 1) - pt.SetFloatIndex(0, row, nn.ParDist) - pt.SetFloatIndex(1, row, nn.Y) + xc.SetFloatRowCell(nn.ParDist, row, 0) + yc.SetFloatRowCell(nn.Y, row, 0) } } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index f8f5bddd18..4043c46865 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -47,16 +47,16 @@ func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { - scale := scale64.Tensor.Float1D(i) - ss := ss64.Tensor.Float1D(i) + scale := scale64.Float1D(i) + ss := ss64.Float1D(i) v := 0.0 if math.IsInf(scale, 1) { v = math.Inf(1) } else { v = scale * scale * ss } - scale64.Tensor.SetFloat1D(v, i) - out.Tensor.SetFloat1D(v, i) + scale64.SetFloat1D(v, i) + out.SetFloat1D(v, i) } return scale64 } @@ -73,16 +73,16 @@ func EuclideanFunc(a, b, out *tensor.Indexed) { scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { - scale := scale64.Tensor.Float1D(i) - ss := ss64.Tensor.Float1D(i) + scale := scale64.Float1D(i) + ss := ss64.Float1D(i) v := 0.0 if math.IsInf(scale, 1) { v = math.Inf(1) } else { v = scale * math.Sqrt(ss) } - scale64.Tensor.SetFloat1D(v, i) - out.Tensor.SetFloat1D(v, i) + scale64.SetFloat1D(v, i) + out.SetFloat1D(v, i) } } @@ -134,16 +134,16 @@ func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { - scale := scale64.Tensor.Float1D(i) - ss := ss64.Tensor.Float1D(i) + scale := scale64.Float1D(i) + ss := ss64.Float1D(i) v := 0.0 if math.IsInf(scale, 1) { v = math.Inf(1) } else { v = scale * math.Sqrt(ss) } - scale64.Tensor.SetFloat1D(v, i) - out.Tensor.SetFloat1D(v, i) + scale64.SetFloat1D(v, i) + out.SetFloat1D(v, i) } } @@ -153,16 +153,16 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { - scale := scale64.Tensor.Float1D(i) - ss := ss64.Tensor.Float1D(i) + scale := scale64.Float1D(i) + ss := ss64.Float1D(i) v := 0.0 if math.IsInf(scale, 1) { v = math.Inf(1) } else { v = scale * scale * ss } - scale64.Tensor.SetFloat1D(v, i) - out.Tensor.SetFloat1D(v, i) + scale64.SetFloat1D(v, i) + out.SetFloat1D(v, i) } } @@ -212,12 +212,12 @@ func CovarianceFunc(a, b, out *tensor.Indexed) { }, a, b, amean, bmean, out) nsub := out.Tensor.Len() for i := range nsub { - c := acount.Tensor.Float1D(i) + c := acount.Float1D(i) if c == 0 { continue } - cov64.Tensor.SetFloat1D(cov64.Tensor.Float1D(i)/c, i) - out.Tensor.SetFloat1D(cov64.Tensor.Float1D(i), i) + cov64.SetFloat1D(cov64.Float1D(i)/c, i) + out.SetFloat1D(cov64.Float1D(i), i) } } @@ -244,13 +244,13 @@ func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { nsub := out.Tensor.Len() for i := range nsub { - ss := ss64.Tensor.Float1D(i) - vp := math.Sqrt(avar64.Tensor.Float1D(i) * bvar64.Tensor.Float1D(i)) + ss := ss64.Float1D(i) + vp := math.Sqrt(avar64.Float1D(i) * bvar64.Float1D(i)) if vp > 0 { ss /= vp } - ss64.Tensor.SetFloat1D(ss, i) - out.Tensor.SetFloat1D(ss, i) + ss64.SetFloat1D(ss, i) + out.SetFloat1D(ss, i) } return ss64 } @@ -277,8 +277,8 @@ func InvCorrelationFunc(a, b, out *tensor.Indexed) { cor64 := CorrelationFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { - cor := cor64.Tensor.Float1D(i) - out.Tensor.SetFloat1D(1-cor, i) + cor := cor64.Float1D(i) + out.SetFloat1D(1-cor, i) } } @@ -296,13 +296,13 @@ func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { }, a, b, out) nsub := out.Tensor.Len() for i := range nsub { - ss := ss64.Tensor.Float1D(i) - vp := math.Sqrt(avar64.Tensor.Float1D(i) * bvar64.Tensor.Float1D(i)) + ss := ss64.Float1D(i) + vp := math.Sqrt(avar64.Float1D(i) * bvar64.Float1D(i)) if vp > 0 { ss /= vp } - ss64.Tensor.SetFloat1D(ss, i) - out.Tensor.SetFloat1D(ss, i) + ss64.SetFloat1D(ss, i) + out.SetFloat1D(ss, i) } return ss64 } @@ -323,7 +323,7 @@ func InvCosineFunc(a, b, out *tensor.Indexed) { cos64 := CosineFuncOut64(a, b, out) nsub := out.Tensor.Len() for i := range nsub { - cos := cos64.Tensor.Float1D(i) - out.Tensor.SetFloat1D(1-cos, i) + cos := cos64.Float1D(i) + out.SetFloat1D(1-cos, i) } } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 1e786dc058..3064ad77cc 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -44,7 +44,7 @@ func Matrix(funcName string, in, out *tensor.Indexed) { sa := tsr[0].Cells1D(c.X) sb := tsr[0].Cells1D(c.Y) tensor.Call(funcName, sa, sb, mout) - tsr[1].SetFloat(mout.Tensor.Float1D(0), c.X, c.Y) + tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, in, out) for _, c := range coords { // copy to upper if c.X == c.Y { // exclude diag @@ -84,7 +84,7 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { sa := tsr[0].Cells1D(ar) sb := tsr[1].Cells1D(br) tensor.Call(funcName, sa, sb, mout) - tsr[2].SetFloat(mout.Tensor.Float1D(0), ar, br) + tsr[2].SetFloat(mout.Float1D(0), ar, br) }, a, b, out) } @@ -134,7 +134,7 @@ func CovarMatrix(funcName string, in, out *tensor.Indexed) { curCoords.Y = c.Y } tensor.Call(funcName, av, bv, mout) - tsr[1].SetFloat(mout.Tensor.Float1D(0), c.X, c.Y) + tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, flatix, out) for _, c := range coords { // copy to upper if c.X == c.Y { // exclude diag @@ -212,7 +212,7 @@ func SVD(covar, eigenvecs, vals *tensor.Indexed) { // 1D output with the number of rows. Otherwise a single number is produced. // This is typically done with results from SVD or PCA. func ProjectOnMatrixColumn(mtx, vec, colindex, out *tensor.Indexed) { - ci := int(colindex.Tensor.Float1D(0)) + ci := int(colindex.Float1D(0)) col := tensor.NewFloat64Indexed() tensor.Slice(mtx, col, tensor.Range{}, tensor.Range{Start: ci, End: ci + 1}) // fmt.Println(mtx.Tensor.String(), col.Tensor.String()) @@ -226,7 +226,7 @@ func ProjectOnMatrixColumn(mtx, vec, colindex, out *tensor.Indexed) { tmath.Mul(vec.Cells1D(i), col, mout) stats.SumFunc(mout, msum) // fmt.Println(vec.Cells1D(i).Tensor.String(), mout.Tensor.String(), msum.Tensor.String()) - out.Tensor.SetFloat1D(msum.Tensor.Float1D(0), i) + out.SetFloat1D(msum.Float1D(0), i) } } else { tmath.Mul(vec, col, mout) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 436b5e2886..e20e811f59 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -8,7 +8,6 @@ import ( "math" "testing" - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" "github.com/stretchr/testify/assert" @@ -87,11 +86,10 @@ func TestMatrix(t *testing.T) { [10]: 5.291502622129181 6.324555320336759 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 4.242640687119285 5.477225575051661 9.16515138991168 9.539392014169456 0 3.4641016151377544 [11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 ` - dt := &table.Table{} + dt := table.NewTable() err := dt.OpenCSV("../cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) - // smat.TableColumn(ix, "Input", "Name", false, metric.Euclidean64) - in := tensor.NewIndexed(errors.Log1(dt.Column("Input"))) + in := dt.Column("Input") out := tensor.NewFloat64Indexed() Matrix(Euclidean.FuncName(), in, out) // fmt.Println(out.Tensor) @@ -109,9 +107,7 @@ func TestPCAIris(t *testing.T) { if err != nil { t.Error(err) } - // pc.TableColumn(ix, "data", metric.Covariance64) - // fmt.Printf("covar: %v\n", pc.Covar) - data := tensor.NewIndexed(errors.Log1(dt.Column("data"))) + data := dt.Column("data") covar := tensor.NewFloat64Indexed() CovarMatrix(Correlation.FuncName(), data, covar) // fmt.Printf("correl: %s\n", covar.Tensor.String()) @@ -156,7 +152,7 @@ func TestPCAIris(t *testing.T) { assert.InDelta(t, corvals[3-i], v, errtol) // opposite order } - colidx.Tensor.SetFloat1D(0, 0) // strongest at start + colidx.SetFloat1D(0, 0) // strongest at start ProjectOnMatrixColumn(vecs, data, colidx, prjns) // tensor.SaveCSV(prjns.Tensor, "testdata/svd_projection.csv", tensor.Comma) trgprjns = []float64{ diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index f95fe26da0..365fecd71c 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -31,7 +31,7 @@ func ClosestRow(funcName string, probe, vocab, out *tensor.Indexed) { for ri := range rows { sub := vocab.Cells1D(ri) tensor.Call(funcName, pr1d, sub, mout) - d := mout.Tensor.Float1D(0) + d := mout.Float1D(0) if d < mind { mi = ri mind = d @@ -39,6 +39,6 @@ func ClosestRow(funcName string, probe, vocab, out *tensor.Indexed) { } out.Tensor.SetShape(2) out.Sequential() - out.Tensor.SetFloat1D(float64(mi), 0) - out.Tensor.SetFloat1D(mind, 1) + out.SetFloat1D(float64(mi), 0) + out.SetFloat1D(mind, 1) } diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 9fdf032402..ccaa6f9257 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -66,7 +66,7 @@ func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(ini, i) + out.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -76,7 +76,7 @@ func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg if math.IsNaN(bv) { continue } - out.Tensor.SetFloat1D(fun(av, bv, out.Tensor.Float1D(i)), i) + out.SetFloat1D(fun(av, bv, out.Float1D(i)), i) } } @@ -88,8 +88,8 @@ func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu nsub := out2.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(ini1, i) - out2.Tensor.SetFloat1D(ini2, i) + out1.SetFloat1D(ini1, i) + out2.SetFloat1D(ini2, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -99,7 +99,7 @@ func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu if math.IsNaN(bv) { continue } - scale, ss := out1.Tensor.Float1D(i), out2.Tensor.Float1D(i) + scale, ss := out1.Float1D(i), out2.Float1D(i) d := fun(av, bv) if d == 0 { continue @@ -111,8 +111,8 @@ func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu } else { ss = ss + (absxi/scale)*(absxi/scale) } - out1.Tensor.SetFloat1D(scale, i) - out2.Tensor.SetFloat1D(ss, i) + out1.SetFloat1D(scale, i) + out2.SetFloat1D(ss, i) } } @@ -124,7 +124,7 @@ func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun fun nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(ini, i) + out.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -134,9 +134,9 @@ func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun fun if math.IsNaN(bv) { continue } - av2 := a2.Tensor.Float1D(i) - bv2 := b2.Tensor.Float1D(i) - out.Tensor.SetFloat1D(fun(av, bv, av2, bv2, out.Tensor.Float1D(i)), i) + av2 := a2.Float1D(i) + bv2 := b2.Float1D(i) + out.SetFloat1D(fun(av, bv, av2, bv2, out.Float1D(i)), i) } } @@ -148,9 +148,9 @@ func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini nsub := out1.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(ini, i) - out2.Tensor.SetFloat1D(ini, i) - out3.Tensor.SetFloat1D(ini, i) + out1.SetFloat1D(ini, i) + out2.SetFloat1D(ini, i) + out3.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -160,15 +160,15 @@ func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini if math.IsNaN(bv) { continue } - av2 := a2.Tensor.Float1D(i) - bv2 := b2.Tensor.Float1D(i) - o1 := out1.Tensor.Float1D(i) - o2 := out2.Tensor.Float1D(i) - o3 := out3.Tensor.Float1D(i) + av2 := a2.Float1D(i) + bv2 := b2.Float1D(i) + o1 := out1.Float1D(i) + o2 := out2.Float1D(i) + o3 := out3.Float1D(i) o1, o2, o3 = fun(av, bv, av2, bv2, o1, o2, o3) - out1.Tensor.SetFloat1D(o1, i) - out2.Tensor.SetFloat1D(o2, i) - out3.Tensor.SetFloat1D(o3, i) + out1.SetFloat1D(o1, i) + out2.SetFloat1D(o2, i) + out3.SetFloat1D(o3, i) } } @@ -179,9 +179,9 @@ func Vec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, f nsub := out1.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(ini, i) - out2.Tensor.SetFloat1D(ini, i) - out3.Tensor.SetFloat1D(ini, i) + out1.SetFloat1D(ini, i) + out2.SetFloat1D(ini, i) + out3.SetFloat1D(ini, i) } av := a.FloatRowCell(idx, i) if math.IsNaN(av) { @@ -191,12 +191,12 @@ func Vec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, f if math.IsNaN(bv) { continue } - o1 := out1.Tensor.Float1D(i) - o2 := out2.Tensor.Float1D(i) - o3 := out3.Tensor.Float1D(i) + o1 := out1.Float1D(i) + o2 := out2.Float1D(i) + o3 := out3.Float1D(i) o1, o2, o3 = fun(av, bv, o1, o2, o3) - out1.Tensor.SetFloat1D(o1, i) - out2.Tensor.SetFloat1D(o2, i) - out3.Tensor.SetFloat1D(o3, i) + out1.SetFloat1D(o1, i) + out2.SetFloat1D(o2, i) + out3.SetFloat1D(o3, i) } } diff --git a/tensor/table/splits.go b/tensor/stats/split/splits.go similarity index 100% rename from tensor/table/splits.go rename to tensor/stats/split/splits.go diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index e4b58ae89a..c21da113e9 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -128,12 +128,12 @@ func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { count64 = CountFuncOut64(in, count) nsub := out.Tensor.Len() for i := range nsub { - c := count64.Tensor.Float1D(i) + c := count64.Float1D(i) if c == 0 { continue } - sum64.Tensor.SetFloat1D(sum64.Tensor.Float1D(i)/c, i) - out.Tensor.SetFloat1D(sum64.Tensor.Float1D(i), i) + sum64.SetFloat1D(sum64.Float1D(i)/c, i) + out.SetFloat1D(sum64.Float1D(i), i) } return sum64, count64 } @@ -163,13 +163,13 @@ func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Index var64, mean64, count64 = SumSqDevFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - c := count64.Tensor.Float1D(i) + c := count64.Float1D(i) if c < 2 { continue } - vr := var64.Tensor.Float1D(i) / (c - 1) - var64.Tensor.SetFloat1D(vr, i) - out.Tensor.SetFloat1D(vr, i) + vr := var64.Float1D(i) / (c - 1) + var64.SetFloat1D(vr, i) + out.SetFloat1D(vr, i) } return } @@ -187,9 +187,9 @@ func StdFuncOut64(in, out *tensor.Indexed) (std64, mean64, count64 *tensor.Index std64, mean64, count64 = VarFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - std := math.Sqrt(std64.Tensor.Float1D(i)) - std64.Tensor.SetFloat1D(std, i) - out.Tensor.SetFloat1D(std, i) + std := math.Sqrt(std64.Float1D(i)) + std64.SetFloat1D(std, i) + out.SetFloat1D(std, i) } return } @@ -208,11 +208,11 @@ func SemFunc(in, out *tensor.Indexed) { var64, _, count64 := VarFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - c := count64.Tensor.Float1D(i) + c := count64.Float1D(i) if c < 2 { - out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i)), i) + out.SetFloat1D(math.Sqrt(var64.Float1D(i)), i) } else { - out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c), i) + out.SetFloat1D(math.Sqrt(var64.Float1D(i))/math.Sqrt(c), i) } } } @@ -223,12 +223,12 @@ func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.In var64, mean64, count64 = SumSqDevFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - c := count64.Tensor.Float1D(i) + c := count64.Float1D(i) if c == 0 { continue } - var64.Tensor.SetFloat1D(var64.Tensor.Float1D(i)/c, i) - out.Tensor.SetFloat1D(var64.Tensor.Float1D(i), i) + var64.SetFloat1D(var64.Float1D(i)/c, i) + out.SetFloat1D(var64.Float1D(i), i) } return } @@ -247,7 +247,7 @@ func StdPopFunc(in, out *tensor.Indexed) { var64, _, _ := VarPopFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i)), i) + out.SetFloat1D(math.Sqrt(var64.Float1D(i)), i) } } @@ -258,11 +258,11 @@ func SemPopFunc(in, out *tensor.Indexed) { var64, _, count64 := VarPopFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - c := count64.Tensor.Float1D(i) + c := count64.Float1D(i) if c < 2 { - out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i)), i) + out.SetFloat1D(math.Sqrt(var64.Float1D(i)), i) } else { - out.Tensor.SetFloat1D(math.Sqrt(var64.Tensor.Float1D(i))/math.Sqrt(c), i) + out.SetFloat1D(math.Sqrt(var64.Float1D(i))/math.Sqrt(c), i) } } } @@ -295,16 +295,16 @@ func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { scale64, ss64 := SumSqScaleFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - scale := scale64.Tensor.Float1D(i) - ss := ss64.Tensor.Float1D(i) + scale := scale64.Float1D(i) + ss := ss64.Float1D(i) v := 0.0 if math.IsInf(scale, 1) { v = math.Inf(1) } else { v = scale * scale * ss } - scale64.Tensor.SetFloat1D(v, i) - out.Tensor.SetFloat1D(v, i) + scale64.SetFloat1D(v, i) + out.SetFloat1D(v, i) } return scale64 } @@ -322,16 +322,16 @@ func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { scale64, ss64 := SumSqScaleFuncOut64(in, out) nsub := out.Tensor.Len() for i := range nsub { - scale := scale64.Tensor.Float1D(i) - ss := ss64.Tensor.Float1D(i) + scale := scale64.Float1D(i) + ss := ss64.Float1D(i) v := 0.0 if math.IsInf(scale, 1) { v = math.Inf(1) } else { v = scale * math.Sqrt(ss) } - scale64.Tensor.SetFloat1D(v, i) - out.Tensor.SetFloat1D(v, i) + scale64.SetFloat1D(v, i) + out.SetFloat1D(v, i) } return scale64 } diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 70f0f67dd1..8f7f75fd58 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -34,7 +34,7 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { out.Tensor.SetShapeFrom(qs.Tensor) nq := qs.Tensor.Len() for i := range nq { - q := qs.Tensor.Float1D(i) + q := qs.Float1D(i) val := 0.0 qi := q * fsz lwi := math.Floor(qi) @@ -49,7 +49,7 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { hiv := sin.FloatRowCell(lwii+1, 0) val = (1-phi)*lwv + phi*hiv } - out.Tensor.SetFloat1D(val, i) + out.SetFloat1D(val, i) } return nil } diff --git a/tensor/stats/stats/table.go b/tensor/stats/stats/table.go index e7ee6fea57..694ca6621a 100644 --- a/tensor/stats/stats/table.go +++ b/tensor/stats/stats/table.go @@ -4,12 +4,7 @@ package stats -import ( - "reflect" - - "cogentcore.org/core/tensor/table" -) - +/* // MeanTables returns an table.Table with the mean values across all float // columns of the input tables, which must have the same columns but not // necessarily the same number of rows. @@ -76,3 +71,4 @@ func MeanTables(dts []*table.Table) *table.Table { } return ot } +*/ diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index e8da3096aa..775ebe151c 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -35,7 +35,7 @@ func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, ts } nsub := out.Tensor.Len() for i := range nsub { - out.Tensor.SetFloat1D(o64.Tensor.Float1D(i), i) + out.SetFloat1D(o64.Float1D(i), i) } return o64 } @@ -94,13 +94,13 @@ func VecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg fl nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(ini, i) + out.SetFloat1D(ini, i) } val := in.FloatRowCell(idx, i) if math.IsNaN(val) { continue } - out.Tensor.SetFloat1D(fun(val, out.Tensor.Float1D(i)), i) + out.SetFloat1D(fun(val, out.Float1D(i)), i) } } @@ -113,14 +113,14 @@ func Vec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(va nsub := out.Tensor.Len() for i := range nsub { if idx == 0 { - out.Tensor.SetFloat1D(ini, i) + out.SetFloat1D(ini, i) } val1 := in1.FloatRowCell(idx, i) if math.IsNaN(val1) { continue } - val2 := in2.Tensor.Float1D(i) // output = not nan - out.Tensor.SetFloat1D(fun(val1, val2, out.Tensor.Float1D(i)), i) + val2 := in2.Float1D(i) // output = not nan + out.SetFloat1D(fun(val1, val2, out.Float1D(i)), i) } } @@ -132,16 +132,16 @@ func Vec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu nsub := out2.Tensor.Len() for i := range nsub { if idx == 0 { - out1.Tensor.SetFloat1D(ini1, i) - out2.Tensor.SetFloat1D(ini2, i) + out1.SetFloat1D(ini1, i) + out2.SetFloat1D(ini2, i) } val := in.FloatRowCell(idx, i) if math.IsNaN(val) { continue } - ag1, ag2 := out1.Tensor.Float1D(i), out2.Tensor.Float1D(i) + ag1, ag2 := out1.Float1D(i), out2.Float1D(i) ag1, ag2 = fun(val, ag1, ag2) - out1.Tensor.SetFloat1D(ag1, i) - out2.Tensor.SetFloat1D(ag2, i) + out1.SetFloat1D(ag1, i) + out2.SetFloat1D(ag2, i) } } diff --git a/tensor/table/io.go b/tensor/table/io.go index 0f59bb06b8..2374e5d62f 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -97,7 +97,6 @@ func (dt *Table) ReadCSV(r io.Reader, delim tensor.Delims) error { return err } rows := len(rec) - // cols := len(rec[0]) strow := 0 if dt.NumColumns() == 0 || DetectTableHeaders(rec[0]) { dt.DeleteAll() @@ -186,7 +185,7 @@ func ConfigFromTableHeaders(dt *Table, hdrs []string) error { hd = hd[:lbst] csh := ShapeFromString(dims) // new tensor starting - dt.AddTensorColumnOfType(hd, typ, csh...) + dt.AddColumnOfType(hd, typ, csh...) continue } dimst = strings.Index(hd, "[") diff --git a/tensor/table/io_test.go b/tensor/table/io_test.go index 925fc9a2e7..4cfea9fc66 100644 --- a/tensor/table/io_test.go +++ b/tensor/table/io_test.go @@ -24,26 +24,27 @@ func TestTableHeaders(t *testing.T) { if dt.NumColumns() != 3 { t.Errorf("TableHeaders: len != 3\n") } - if dt.Columns[0].DataType() != reflect.String { - t.Errorf("TableHeaders: dt.Columns[0] != STRING\n") + cols := dt.Columns.Values + if cols[0].DataType() != reflect.String { + t.Errorf("TableHeaders: cols[0] != STRING\n") } - if dt.Columns[1].DataType() != reflect.Float32 { - t.Errorf("TableHeaders: dt.Columns[1] != FLOAT32\n") + if cols[1].DataType() != reflect.Float32 { + t.Errorf("TableHeaders: cols[1] != FLOAT32\n") } - if dt.Columns[2].DataType() != reflect.Float32 { - t.Errorf("TableHeaders: dt.Columns[2] != FLOAT32\n") + if cols[2].DataType() != reflect.Float32 { + t.Errorf("TableHeaders: cols[2] != FLOAT32\n") } - if dt.Columns[1].Shape().Sizes[1] != 5 { - t.Errorf("TableHeaders: dt.Columns[1].Shape().Sizes[1] != 5\n") + if cols[1].Shape().Sizes[1] != 5 { + t.Errorf("TableHeaders: cols[1].Shape().Sizes[1] != 5\n") } - if dt.Columns[1].Shape().Sizes[2] != 5 { - t.Errorf("TableHeaders: dt.Columns[1].Shape().Sizes[2] != 5\n") + if cols[1].Shape().Sizes[2] != 5 { + t.Errorf("TableHeaders: cols[1].Shape().Sizes[2] != 5\n") } - if dt.Columns[2].Shape().Sizes[1] != 5 { - t.Errorf("TableHeaders: dt.Columns[2].Shape().Sizes[1] != 5\n") + if cols[2].Shape().Sizes[1] != 5 { + t.Errorf("TableHeaders: cols[2].Shape().Sizes[1] != 5\n") } - if dt.Columns[2].Shape().Sizes[2] != 5 { - t.Errorf("TableHeaders: dt.Columns[2].Shape().Sizes[2] != 5\n") + if cols[2].Shape().Sizes[2] != 5 { + t.Errorf("TableHeaders: cols[2].Shape().Sizes[2] != 5\n") } outh := dt.TableHeaders() // fmt.Printf("headers out:\n%v\n", outh) @@ -66,12 +67,12 @@ func TestReadTableDat(t *testing.T) { if err != nil { t.Error(err) } - dt := &Table{} + dt := NewTable() err = dt.ReadCSV(fp, '\t') // tsv if err != nil { t.Error(err) } - sc := dt.Columns + sc := dt.Columns.Values if len(sc) != 3 { t.Errorf("TableHeaders: len != 3\n") } diff --git a/tensor/table/slicetable.go b/tensor/table/slicetable.go index 7587ab6d23..e25827817c 100644 --- a/tensor/table/slicetable.go +++ b/tensor/table/slicetable.go @@ -26,31 +26,13 @@ func NewSliceTable(st any) (*Table, error) { for i := 0; i < eltyp.NumField(); i++ { f := eltyp.Field(i) - switch f.Type.Kind() { - case reflect.Float32: - dt.AddFloat32Column(f.Name) - case reflect.Float64: - dt.AddFloat64Column(f.Name) - case reflect.String: - dt.AddStringColumn(f.Name) - } - } - - nr := npv.Len() - dt.SetNumRows(nr) - for ri := 0; ri < nr; ri++ { - for i := 0; i < eltyp.NumField(); i++ { - f := eltyp.Field(i) - switch f.Type.Kind() { - case reflect.Float32: - dt.SetFloat(f.Name, ri, float64(npv.Index(ri).Field(i).Interface().(float32))) - case reflect.Float64: - dt.SetFloat(f.Name, ri, float64(npv.Index(ri).Field(i).Interface().(float64))) - case reflect.String: - dt.SetString(f.Name, ri, npv.Index(ri).Field(i).Interface().(string)) - } + kind := f.Type.Kind() + if !reflectx.KindIsBasic(kind) { + continue } + dt.AddColumnOfType(f.Name, kind) } + UpdateSliceTable(st, dt) return dt, nil } @@ -65,13 +47,17 @@ func UpdateSliceTable(st any, dt *Table) { for ri := 0; ri < nr; ri++ { for i := 0; i < eltyp.NumField(); i++ { f := eltyp.Field(i) - switch f.Type.Kind() { - case reflect.Float32: - dt.SetFloat(f.Name, ri, float64(npv.Index(ri).Field(i).Interface().(float32))) - case reflect.Float64: - dt.SetFloat(f.Name, ri, float64(npv.Index(ri).Field(i).Interface().(float64))) - case reflect.String: - dt.SetString(f.Name, ri, npv.Index(ri).Field(i).Interface().(string)) + kind := f.Type.Kind() + if !reflectx.KindIsBasic(kind) { + continue + } + val := npv.Index(ri).Field(i).Interface() + cl := dt.Column(f.Name) + if kind == reflect.String { + cl.SetStringRowCell(val.(string), ri, 0) + } else { + fv, _ := reflectx.ToFloat(val) + cl.SetFloatRowCell(fv, ri, 0) } } } diff --git a/tensor/table/slicetable_test.go b/tensor/table/slicetable_test.go index d74948342f..4d23a9fdab 100644 --- a/tensor/table/slicetable_test.go +++ b/tensor/table/slicetable_test.go @@ -26,5 +26,5 @@ func TestSliceTable(t *testing.T) { if err != nil { t.Error(err.Error()) } - assert.Equal(t, 2, dt.Rows) + assert.Equal(t, 2, dt.Rows()) } diff --git a/tensor/table/table.go b/tensor/table/table.go index a8acae39c6..09bc6bf60e 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -8,7 +8,6 @@ package table import ( "fmt" - "math" "reflect" "slices" @@ -17,9 +16,9 @@ import ( ) // Table is a table of Tensor columns aligned by a common outermost row dimension. -// Use the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a +// Use the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a // [tensor.Indexed] view of the column, using the shared [Table.Indexes] of the Table. -// Thus, a coordinated sorting and filtered view of the column data is automatically +// Thus, a coordinated sorting and filtered view of the column data is automatically // available for any of the tensor package functions that use [tensor.Indexed] as the one // common data representation for all operations. type Table struct { //types:add @@ -109,38 +108,18 @@ func (dt *Table) ColumnIndex(idx int) *tensor.Indexed { return tensor.NewIndexed(cl, dt.Indexes) } -/* -// ColumnIndexesByNames returns the indexes of the given column names. -// idxs have -1 if name not found. -func (dt *Table) ColumnIndexesByNames(names ...string) ([]int, error) { - nc := len(names) - if nc == 0 { - return nil, nil - } - var errs []error - cidx := make([]int, nc) - for i, cn := range names { - var err error - cidx[i], err = dt.ColumnIndex(cn) - if err != nil { - errs = append(errs, err) - } - } - return cidx, errors.Join(errs...) -} -*/ - // ColumnName returns the name of given column func (dt *Table) ColumnName(i int) string { return dt.Columns.Keys[i] } // AddColumn adds a new column to the table, of given type and column name -// (which must be unique). The cells of this column hold a single scalar value: -// see [Table.AddColumnTensor] for n-dimensional cells. -func AddColumn[T tensor.DataTypes](dt *Table, name string) tensor.Tensor { +// (which must be unique). If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +func AddColumn[T tensor.DataTypes](dt *Table, name string, cellSizes ...int) tensor.Tensor { rows := max(1, dt.Columns.Rows) - tsr := tensor.New[T](rows) + sz := append([]int{rows}, cellSizes...) + tsr := tensor.New[T](sz...) tsr.SetNames("Row") dt.AddColumn(name, tsr) return tsr @@ -148,25 +127,14 @@ func AddColumn[T tensor.DataTypes](dt *Table, name string) tensor.Tensor { // InsertColumn inserts a new column to the table, of given type and column name // (which must be unique), at given index. -// The cells of this column hold a single scalar value. -func InsertColumn[T tensor.DataTypes](dt *Table, name string, idx int) tensor.Tensor { - rows := max(1, dt.Columns.Rows) - tsr := tensor.New[T](rows) - tsr.SetNames("Row") - dt.InsertColumn(idx, name, tsr) - return tsr -} - -// AddTensorColumn adds a new n-dimensional column to the table, of given type, column name -// (which must be unique), and dimensionality of each _cell_. -// An outermost Row dimension will be added to this dimensionality to create -// the tensor column. -func AddTensorColumn[T tensor.DataTypes](dt *Table, name string, cellSizes ...int) tensor.Tensor { +// If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +func InsertColumn[T tensor.DataTypes](dt *Table, name string, idx int, cellSizes ...int) tensor.Tensor { rows := max(1, dt.Columns.Rows) sz := append([]int{rows}, cellSizes...) tsr := tensor.New[T](sz...) tsr.SetNames("Row") - dt.AddColumn(name, tsr) + dt.InsertColumn(idx, name, tsr) return tsr } @@ -186,22 +154,10 @@ func (dt *Table) InsertColumn(idx int, name string, tsr tensor.Tensor) error { // AddColumnOfType adds a new scalar column to the table, of given reflect type, // column name (which must be unique), -// The cells of this column hold a single (scalar) value of given type. -// Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. -func (dt *Table) AddColumnOfType(name string, typ reflect.Kind) tensor.Tensor { - rows := max(1, dt.Columns.Rows) - tsr := tensor.NewOfType(typ, rows) - tsr.SetNames("Row") - dt.AddColumn(name, tsr) - return tsr -} - -// AddTensorColumnOfType adds a new n-dimensional column to the table, of given reflect type, -// column name (which must be unique), and dimensionality of each _cell_. -// An outermost Row dimension will be added to this dimensionality to create -// the tensor column. -// Supported types are string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. -func (dt *Table) AddTensorColumnOfType(name string, typ reflect.Kind, cellSizes ...int) tensor.Tensor { +// If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +// Supported types include string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. +func (dt *Table) AddColumnOfType(name string, typ reflect.Kind, cellSizes ...int) tensor.Tensor { rows := max(1, dt.Columns.Rows) sz := append([]int{rows}, cellSizes...) tsr := tensor.NewOfType(typ, sz...) @@ -211,51 +167,31 @@ func (dt *Table) AddTensorColumnOfType(name string, typ reflect.Kind, cellSizes } // AddStringColumn adds a new String column with given name. -// The cells of this column hold a single string value. -func (dt *Table) AddStringColumn(name string) *tensor.String { - return AddColumn[string](dt, name).(*tensor.String) +// If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +func (dt *Table) AddStringColumn(name string, cellSizes ...int) *tensor.String { + return AddColumn[string](dt, name, cellSizes...).(*tensor.String) } // AddFloat64Column adds a new float64 column with given name. -// The cells of this column hold a single scalar value. -func (dt *Table) AddFloat64Column(name string) *tensor.Float64 { - return AddColumn[float64](dt, name).(*tensor.Float64) -} - -// AddFloat64TensorColumn adds a new n-dimensional float64 column with given name -// and dimensionality of each _cell_. -// An outermost Row dimension will be added to this dimensionality to create -// the tensor column. -func (dt *Table) AddFloat64TensorColumn(name string, cellSizes ...int) *tensor.Float64 { - return AddTensorColumn[float64](dt, name, cellSizes...).(*tensor.Float64) +// If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +func (dt *Table) AddFloat64Column(name string, cellSizes ...int) *tensor.Float64 { + return AddColumn[float64](dt, name, cellSizes...).(*tensor.Float64) } // AddFloat32Column adds a new float32 column with given name. -// The cells of this column hold a single scalar value. -func (dt *Table) AddFloat32Column(name string) *tensor.Float32 { - return AddColumn[float32](dt, name).(*tensor.Float32) -} - -// AddFloat32TensorColumn adds a new n-dimensional float32 column with given name -// and dimensionality of each _cell_. -// An outermost Row dimension will be added to this dimensionality to create -// the tensor column. -func (dt *Table) AddFloat32TensorColumn(name string, cellSizes ...int) *tensor.Float32 { - return AddTensorColumn[float32](dt, name, cellSizes...).(*tensor.Float32) +// If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +func (dt *Table) AddFloat32Column(name string, cellSizes ...int) *tensor.Float32 { + return AddColumn[float32](dt, name, cellSizes...).(*tensor.Float32) } // AddIntColumn adds a new int column with given name. -// The cells of this column hold a single scalar value. -func (dt *Table) AddIntColumn(name string) *tensor.Int { - return AddColumn[int](dt, name).(*tensor.Int) -} - -// AddIntTensorColumn adds a new n-dimensional int column with given name -// and dimensionality of each _cell_. -// An outermost Row dimension will be added to this dimensionality to create -// the tensor column. -func (dt *Table) AddIntTensorColumn(name string, cellSizes ...int) *tensor.Int { - return AddTensorColumn[int](dt, name, cellSizes...).(*tensor.Int) +// If no cellSizes are specified, it holds scalar values, +// otherwise the cells are n-dimensional tensors of given size. +func (dt *Table) AddIntColumn(name string, cellSizes ...int) *tensor.Int { + return AddColumn[int](dt, name, cellSizes...).(*tensor.Int) } // DeleteColumnName deletes column of given name. @@ -290,6 +226,7 @@ func (dt *Table) InsertRows(at, n int) *Table { dt.Columns.SetNumRows(strow + n) // adds n indexes to end of list // move those indexes to at:at+n in index list dt.Indexes = append(dt.Indexes[:at], append(dt.Indexes[stidx:], dt.Indexes[at:]...)...) + return dt } // SetNumRows sets the number of rows in the table, across all columns @@ -326,7 +263,7 @@ func (dt *Table) Clone() *Table { return cp } -// AppendRows appends shared columns in both tables with input table rows +// AppendRows appends shared columns in both tables with input table rows. func (dt *Table) AppendRows(dt2 *Table) { strow := dt.Columns.Rows n := dt2.Columns.Rows @@ -334,294 +271,7 @@ func (dt *Table) AppendRows(dt2 *Table) { if dt.Indexes == nil { return } - if rows > strow { - for i := range rows - strow { - dt.Indexes = append(dt.Indexes, strow+i) - } -} - -/////////////////////////////////////////////////////////////////////////// -// Cell convenience access methods - -// FloatIndex returns the float64 value of cell at given column, row index -// for columns that have 1-dimensional tensors. -// Returns NaN if column is not a 1-dimensional tensor or row not valid. -func (dt *Table) FloatIndex(column, row int) float64 { - if dt.IsValidRow(row) != nil { - return math.NaN() - } - ct := dt.Columns.Values[column] - if ct.NumDims() != 1 { - return math.NaN() - } - return ct.Float1D(row) -} - -// Float returns the float64 value of cell at given column (by name), -// row index for columns that have 1-dimensional tensors. -// Returns NaN if column is not a 1-dimensional tensor -// or col name not found, or row not valid. -func (dt *Table) Float(column string, row int) float64 { - if dt.IsValidRow(row) != nil { - return math.NaN() - } - ct := dt.Column(column) - if ct == nil { - return math.NaN() - } - if ct.Tensor.NumDims() != 1 { - return math.NaN() - } - return ct.Tensor.Float1D(row) -} - -// StringIndex returns the string value of cell at given column, row index -// for columns that have 1-dimensional tensors. -// Returns "" if column is not a 1-dimensional tensor or row not valid. -func (dt *Table) StringIndex(column, row int) string { - if dt.IsValidRow(row) != nil { - return "" - } - ct := dt.Columns.Values[column] - if ct.NumDims() != 1 { - return "" - } - return ct.String1D(row) -} - -// NOTE: String conflicts with [fmt.Stringer], so we have to use StringValue - -// StringValue returns the string value of cell at given column (by name), row index -// for columns that have 1-dimensional tensors. -// Returns "" if column is not a 1-dimensional tensor or row not valid. -func (dt *Table) StringValue(column string, row int) string { - if dt.IsValidRow(row) != nil { - return "" - } - ct := dt.Column(column) - if ct == nil { - return "" - } - if ct.Tensor.NumDims() != 1 { - return "" - } - return ct.Tensor.String1D(row) -} - -// TensorIndex returns the tensor SubSpace for given column, row index -// for columns that have higher-dimensional tensors so each row is -// represented by an n-1 dimensional tensor, with the outer dimension -// being the row number. Returns nil if column is a 1-dimensional -// tensor or there is any error from the tensor.Tensor.SubSpace call. -func (dt *Table) TensorIndex(column, row int) tensor.Tensor { - if dt.IsValidRow(row) != nil { - return nil - } - ct := dt.Columns.Values[column] - if ct.NumDims() == 1 { - return nil - } - return ct.SubSpace(row) -} - -// Tensor returns the tensor SubSpace for given column (by name), row index -// for columns that have higher-dimensional tensors so each row is -// represented by an n-1 dimensional tensor, with the outer dimension -// being the row number. Returns nil on any error. -func (dt *Table) Tensor(column string, row int) tensor.Tensor { - if dt.IsValidRow(row) != nil { - return nil - } - ct := dt.Column(column) - if ct == nil { - return nil + for i := range n { + dt.Indexes = append(dt.Indexes, strow+i) } - if ct.Tensor.NumDims() == 1 { - return nil - } - return ct.SubSpace(row) -} - -// TensorFloat1D returns the float value of a Tensor cell's cell at given -// 1D offset within cell, for given column (by name), row index -// for columns that have higher-dimensional tensors so each row is -// represented by an n-1 dimensional tensor, with the outer dimension -// being the row number. Returns 0 on any error. -func (dt *Table) TensorFloat1D(column string, row int, idx int) float64 { - if dt.IsValidRow(row) != nil { - return math.NaN() - } - ct := dt.Column(column) - if ct == nil { - return math.NaN() - } - if ct.Tensor.NumDims() == 1 { - return math.NaN() - } - _, sz := ct.RowCellSize() - if idx >= sz || idx < 0 { - return math.NaN() - } - off := row*sz + idx - return ct.Tensor.Float1D(off) -} - -///////////////////////////////////////////////////////////////////////////////////// -// Set - -// SetFloatIndex sets the float64 value of cell at given column, row index -// for columns that have 1-dimensional tensors. -func (dt *Table) SetFloatIndex(column, row int, val float64) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ct := dt.Columns.Values[column] - if ct.NumDims() != 1 { - return fmt.Errorf("table.Table SetFloatIndex: Column %d is a tensor, must use SetTensorFloat1D", column) - } - ct.SetFloat1D(val, row) - return nil -} - -// SetFloat sets the float64 value of cell at given column (by name), row index -// for columns that have 1-dimensional tensors. -func (dt *Table) SetFloat(column string, row int, val float64) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ct, err := dt.ColumnTry(column) - if err != nil { - return err - } - if ct.Tensor.NumDims() != 1 { - return fmt.Errorf("table.Table SetFloat: Column %s is a tensor, must use SetTensorFloat1D", column) - } - ct.Tensor.SetFloat1D(val, row) - return nil -} - -// SetStringIndex sets the string value of cell at given column, row index -// for columns that have 1-dimensional tensors. Returns true if set. -func (dt *Table) SetStringIndex(column, row int, val string) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ct := dt.Columns.Values[column] - if ct.NumDims() != 1 { - return fmt.Errorf("table.Table SetStringIndex: Column %d is a tensor, must use SetTensorFloat1D", column) - } - ct.SetString1D(val, row) - return nil -} - -// SetString sets the string value of cell at given column (by name), row index -// for columns that have 1-dimensional tensors. Returns true if set. -func (dt *Table) SetString(column string, row int, val string) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ct, err := dt.ColumnTry(column) - if err != nil { - return err - } - if ct.Tensor.NumDims() != 1 { - return fmt.Errorf("table.Table SetString: Column %s is a tensor, must use SetTensorFloat1D", column) - } - ct.Tensor.SetString1D(val, row) - return nil -} - -// SetTensorIndex sets the tensor value of cell at given column, row index -// for columns that have n-dimensional tensors. Returns true if set. -func (dt *Table) SetTensorIndex(column, row int, val tensor.Tensor) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ct := dt.Columns.Values[column] - _, csz := ct.RowCellSize() - st := row * csz - sz := min(csz, val.Len()) - if ct.IsString() { - for j := 0; j < sz; j++ { - ct.SetString1D(val.String1D(j), st+j) - } - } else { - for j := 0; j < sz; j++ { - ct.SetFloat1D(val.Float1D(j), st+j) - } - } - return nil -} - -// SetTensor sets the tensor value of cell at given column (by name), row index -// for columns that have n-dimensional tensors. Returns true if set. -func (dt *Table) SetTensor(column string, row int, val tensor.Tensor) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ci := dt.Columns.IndexByKey(column) - if ci == nil { - return err - } - return dt.SetTensorIndex(ci, row, val) -} - -// SetTensorFloat1D sets the tensor cell's float cell value at given 1D index within cell, -// at given column (by name), row index for columns that have n-dimensional tensors. -// Returns true if set. -func (dt *Table) SetTensorFloat1D(column string, row int, idx int, val float64) error { - if err := dt.IsValidRow(row); err != nil { - return err - } - ct, err := dt.ColumnTry(column) - if err != nil { - return err - } - _, sz := ct.RowCellSize() - if idx >= sz || idx < 0 { - return fmt.Errorf("table.Table IsValidRow: index %d is out of valid range [0..%d]", idx, sz) - } - off := row*sz + idx - ct.Tensor.SetFloat1D(val, off) - return nil -} - -////////////////////////////////////////////////////////////////////////////////////// -// Copy Cell - -// CopyCell copies into cell at given column, row from cell in other table. -// It is robust to differences in type; uses destination cell type. -// Returns error if column names are invalid. -func (dt *Table) CopyCell(column string, row int, cpt *Table, cpColNm string, cpRow int) error { - ct, err := dt.ColumnTry(column) - if err != nil { - return err - } - cpct, err := cpt.ColumnTry(cpColNm) - if err != nil { - return err - } - _, sz := ct.RowCellSize() - if sz == 1 { - if ct.Tensor.IsString() { - ct.SetString1D(cpct.String1D(cpRow), row) - return nil - } - ct.Tensor.SetFloat1D(cpct.Float1D(cpRow), row) - return nil - } - _, cpsz := cpct.RowCellSize() - st := row * sz - cst := cpRow * cpsz - msz := min(sz, cpsz) - if ct.IsString() { - for j := 0; j < msz; j++ { - ct.SetString1D(cpct.String1D(cst+j), st+j) - } - } else { - for j := 0; j < msz; j++ { - ct.SetFloat1D(cpct.Float1D(cst+j), st+j) - } - } - return nil } diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index d818e0dc12..bdf9214a03 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -13,12 +13,9 @@ import ( func TestAdd3DCol(t *testing.T) { dt := NewTable() - dt.AddFloat32TensorColumn("Values", []int{11, 1, 16}) + dt.AddFloat32Column("Values", 11, 1, 16) - col, err := dt.Column("Values") - if err != nil { - t.Error(err) - } + col := dt.Column("Values").Tensor if col.NumDims() != 4 { t.Errorf("Add4DCol: # of dims != 4\n") } @@ -46,10 +43,10 @@ func NewTestTable() *Table { dt.AddFloat64Column("Flt64") dt.AddIntColumn("Int") dt.SetNumRows(3) - for i := 0; i < dt.Rows; i++ { - dt.SetString("Str", i, strconv.Itoa(i)) - dt.SetFloat("Flt64", i, float64(i)) - dt.SetFloat("Int", i, float64(i)) + for i := range dt.Rows() { + dt.Column("Str").SetStringRowCell(strconv.Itoa(i), i, 0) + dt.Column("Flt64").SetFloatRowCell(float64(i), i, 0) + dt.Column("Int").SetFloatRowCell(float64(i), i, 0) } return dt } @@ -60,19 +57,19 @@ func TestAppendRows(t *testing.T) { dt.AppendRows(st) dt.AppendRows(st) dt.AppendRows(st) - for j := 0; j < 3; j++ { - for i := 0; i < st.Rows; i++ { + for j := range 3 { + for i := range st.Rows() { sr := j*3 + i - ss := st.StringValue("Str", i) - ds := dt.StringValue("Str", sr) + ss := st.Column("Str").StringRowCell(i, 0) + ds := dt.Column("Str").StringRowCell(sr, 0) assert.Equal(t, ss, ds) - sf := st.Float("Flt64", i) - df := dt.Float("Flt64", sr) + sf := st.Column("Flt64").FloatRowCell(i, 0) + df := dt.Column("Flt64").FloatRowCell(sr, 0) assert.Equal(t, sf, df) - sf = st.Float("Int", i) - df = dt.Float("Int", sr) + sf = st.Column("Int").FloatRowCell(i, 0) + df = dt.Column("Int").FloatRowCell(sr, 0) assert.Equal(t, sf, df) } } diff --git a/tensor/table/util.go b/tensor/table/util.go index 1248521af9..697c230be7 100644 --- a/tensor/table/util.go +++ b/tensor/table/util.go @@ -27,8 +27,8 @@ func (dt *Table) InsertKeyColumns(args ...string) *Table { for j := range nc { colNm := args[2*j] val := args[2*j+1] - col := tensor.NewString(c.Rows) - c.InsertColumn(col, colNm, 0) + col := tensor.NewString(c.Columns.Rows) + c.InsertColumn(0, colNm, col) for i := range col.Values { col.Values[i] = val } @@ -40,10 +40,10 @@ func (dt *Table) InsertKeyColumns(args ...string) *Table { // values in the first two columns of given format table, conventionally named // Name, Type (but names are not used), which must be of the string type. func (dt *Table) ConfigFromTable(ft *Table) error { - nmcol := ft.Columns[0] - tycol := ft.Columns[1] + nmcol := ft.ColumnIndex(0) + tycol := ft.ColumnIndex(1) var errs []error - for i := range ft.Rows { + for i := range ft.Rows() { name := nmcol.String1D(i) typ := strings.ToLower(tycol.String1D(i)) kind := reflect.Float64 @@ -66,7 +66,7 @@ func (dt *Table) ConfigFromTable(ft *Table) error { err := fmt.Errorf("ConfigFromTable: type string %q not recognized", typ) errs = append(errs, err) } - dt.AddColumnOfType(kind, name) + dt.AddColumnOfType(name, kind) } return errors.Join(errs...) } diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index 7562ebac0a..a9ad9a9eba 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -54,7 +54,7 @@ func Abs(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Abs(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(i)), i) }, in, out) } @@ -62,7 +62,7 @@ func Acos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Acos(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(i)), i) }, in, out) } @@ -70,7 +70,7 @@ func Acosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Acosh(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(i)), i) }, in, out) } @@ -78,7 +78,7 @@ func Asin(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Asin(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(i)), i) }, in, out) } @@ -86,7 +86,7 @@ func Asinh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Asinh(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(i)), i) }, in, out) } @@ -94,7 +94,7 @@ func Atan(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Atan(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(i)), i) }, in, out) } @@ -102,7 +102,7 @@ func Atanh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Atanh(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(i)), i) }, in, out) } @@ -110,7 +110,7 @@ func Cbrt(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Cbrt(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(i)), i) }, in, out) } @@ -118,7 +118,7 @@ func Ceil(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Ceil(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(i)), i) }, in, out) } @@ -126,7 +126,7 @@ func Cos(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Cos(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(i)), i) }, in, out) } @@ -134,7 +134,7 @@ func Cosh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Cosh(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(i)), i) }, in, out) } @@ -142,7 +142,7 @@ func Erf(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Erf(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(i)), i) }, in, out) } @@ -150,7 +150,7 @@ func Erfc(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Erfc(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(i)), i) }, in, out) } @@ -158,7 +158,7 @@ func Erfcinv(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Erfcinv(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(i)), i) }, in, out) } @@ -166,7 +166,7 @@ func Erfinv(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Erfinv(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(i)), i) }, in, out) } @@ -174,7 +174,7 @@ func Exp(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Exp(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(i)), i) }, in, out) } @@ -182,7 +182,7 @@ func Exp2(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Exp2(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(i)), i) }, in, out) } @@ -190,7 +190,7 @@ func Expm1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Expm1(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(i)), i) }, in, out) } @@ -198,7 +198,7 @@ func Floor(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Floor(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(i)), i) }, in, out) } @@ -206,7 +206,7 @@ func Gamma(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Gamma(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(i)), i) }, in, out) } @@ -214,7 +214,7 @@ func J0(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.J0(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(i)), i) }, in, out) } @@ -222,7 +222,7 @@ func J1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.J1(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(i)), i) }, in, out) } @@ -230,7 +230,7 @@ func Log(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Log(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(i)), i) }, in, out) } @@ -238,7 +238,7 @@ func Log10(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Log10(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(i)), i) }, in, out) } @@ -246,7 +246,7 @@ func Log1p(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Log1p(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(i)), i) }, in, out) } @@ -254,7 +254,7 @@ func Log2(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Log2(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(i)), i) }, in, out) } @@ -262,7 +262,7 @@ func Logb(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Logb(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(i)), i) }, in, out) } @@ -270,7 +270,7 @@ func Round(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Round(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(i)), i) }, in, out) } @@ -278,7 +278,7 @@ func RoundToEven(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.RoundToEven(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(i)), i) }, in, out) } @@ -286,7 +286,7 @@ func Sin(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Sin(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(i)), i) }, in, out) } @@ -294,7 +294,7 @@ func Sinh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Sinh(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(i)), i) }, in, out) } @@ -302,7 +302,7 @@ func Sqrt(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Sqrt(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(i)), i) }, in, out) } @@ -310,7 +310,7 @@ func Tan(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Tan(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(i)), i) }, in, out) } @@ -318,7 +318,7 @@ func Tanh(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Tanh(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(i)), i) }, in, out) } @@ -326,7 +326,7 @@ func Trunc(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Trunc(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(i)), i) }, in, out) } @@ -334,7 +334,7 @@ func Y0(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Y0(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(i)), i) }, in, out) } @@ -342,7 +342,7 @@ func Y1(in, out *tensor.Indexed) { out.SetShapeFrom(in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math.Y1(tsr[0].Tensor.Float1D(i)), i) + tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(i)), i) }, in, out) } diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index ac9b0029db..198b6e3f2c 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -38,7 +38,7 @@ func TestMath(t *testing.T) { cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, ci := cell2d.RowCellIndex(idx) - cell2d.Tensor.SetFloat1D(oned.Tensor.Float1D(ci), i) + cell2d.SetFloat1D(oned.Float1D(ci), i) }, cell2d) // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() @@ -52,9 +52,9 @@ func TestMath(t *testing.T) { tf(oned, oneout) tf(cell2d, cellout) - Equal(t, fun(scalar.Tensor.Float1D(0)), scout.Tensor.Float1D(0)) + Equal(t, fun(scalar.Float1D(0)), scout.Float1D(0)) for i, v := range vals { - Equal(t, fun(v), oneout.Tensor.Float1D(i)) + Equal(t, fun(v), oneout.Float1D(i)) } lv := len(vals) for r := range 5 { diff --git a/tensor/tmath/norm.go b/tensor/tmath/norm.go index 0f910aeaa3..87c4acd98a 100644 --- a/tensor/tmath/norm.go +++ b/tensor/tmath/norm.go @@ -34,11 +34,11 @@ func UnitNorm(a, out *tensor.Indexed) { // treated as scalars (first value used). func Clamp(in, minv, maxv, out *tensor.Indexed) { out.SetShapeFrom(in) - mn := minv.Tensor.Float1D(0) - mx := maxv.Tensor.Float1D(0) + mn := minv.Float1D(0) + mx := maxv.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].Tensor.SetFloat1D(math32.Clamp64(tsr[0].Tensor.Float1D(i), mn, mx), i) + tsr[1].SetFloat1D(math32.Clamp64(tsr[0].Float1D(i), mn, mx), i) }, in, out) } @@ -47,15 +47,15 @@ func Clamp(in, minv, maxv, out *tensor.Indexed) { // treated as a scalar (first value used). func Binarize(in, threshold, out *tensor.Indexed) { out.SetShapeFrom(in) - thr := threshold.Tensor.Float1D(0) + thr := threshold.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - v := tsr[0].Tensor.Float1D(i) + v := tsr[0].Float1D(i) if v >= thr { v = 1 } else { v = 0 } - tsr[1].Tensor.SetFloat1D(v, i) + tsr[1].SetFloat1D(v, i) }, in, out) } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 11b404fbbd..212b81d519 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -41,7 +41,7 @@ func Add(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)+tsr[1].Tensor.Float1D(ib), ia) + out.SetFloat1D(tsr[0].Float1D(ia)+tsr[1].Float1D(ib), ia) }, a, b, out) } @@ -50,7 +50,7 @@ func AddScalar(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)+scalar, ia) + out.SetFloat1D(a.Float1D(ia)+scalar, ia) }, a, out) } @@ -61,7 +61,7 @@ func AddSubSpace(a, sub, out *tensor.Indexed) { tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)+sub.Tensor.Float1D(si), ia) + out.SetFloat1D(a.Float1D(ia)+sub.Float1D(si), ia) }, a, sub, out) } @@ -101,7 +101,7 @@ func Sub(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)-tsr[1].Tensor.Float1D(ib), ia) + out.SetFloat1D(tsr[0].Float1D(ia)-tsr[1].Float1D(ib), ia) }, a, b, out) } @@ -111,7 +111,7 @@ func SubScalar(sign float64, scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(sign*(a.Tensor.Float1D(ia)-scalar), ia) + out.SetFloat1D(sign*(a.Float1D(ia)-scalar), ia) }, a, out) } @@ -123,7 +123,7 @@ func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(sign*(a.Tensor.Float1D(ia)-sub.Tensor.Float1D(si)), ia) + out.SetFloat1D(sign*(a.Float1D(ia)-sub.Float1D(si)), ia) }, a, sub, out) } @@ -163,7 +163,7 @@ func Mul(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)*tsr[1].Tensor.Float1D(ib), ia) + out.SetFloat1D(tsr[0].Float1D(ia)*tsr[1].Float1D(ib), ia) }, a, b, out) } @@ -172,7 +172,7 @@ func MulScalar(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)*scalar, ia) + out.SetFloat1D(a.Float1D(ia)*scalar, ia) }, a, out) } @@ -183,7 +183,7 @@ func MulSubSpace(a, sub, out *tensor.Indexed) { tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)*sub.Tensor.Float1D(si), ia) + out.SetFloat1D(a.Float1D(ia)*sub.Float1D(si), ia) }, a, sub, out) } @@ -223,7 +223,7 @@ func Div(a, b, out *tensor.Indexed) { func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := tsr[0].RowCellIndex(idx) ib, _, _ := tsr[1].RowCellIndex(idx) - out.Tensor.SetFloat1D(tsr[0].Tensor.Float1D(ia)/tsr[1].Tensor.Float1D(ib), ia) + out.SetFloat1D(tsr[0].Float1D(ia)/tsr[1].Float1D(ib), ia) }, a, b, out) } @@ -232,7 +232,7 @@ func DivScalar(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)/scalar, ia) + out.SetFloat1D(a.Float1D(ia)/scalar, ia) }, a, out) } @@ -242,7 +242,7 @@ func DivScalarInv(scalar float64, a, out *tensor.Indexed) { out.SetShapeFrom(a) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, _ := a.RowCellIndex(idx) - out.Tensor.SetFloat1D(scalar/a.Tensor.Float1D(ia), ia) + out.SetFloat1D(scalar/a.Float1D(ia), ia) }, a, out) } @@ -253,7 +253,7 @@ func DivSubSpace(a, sub, out *tensor.Indexed) { tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(a.Tensor.Float1D(ia)/sub.Tensor.Float1D(si), ia) + out.SetFloat1D(a.Float1D(ia)/sub.Float1D(si), ia) }, a, sub, out) } @@ -264,6 +264,6 @@ func DivSubSpaceInv(a, sub, out *tensor.Indexed) { tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { ia, _, ci := a.RowCellIndex(idx) si, _, _ := sub.RowCellIndex(ci) - out.Tensor.SetFloat1D(sub.Tensor.Float1D(si)/a.Tensor.Float1D(ia), ia) + out.SetFloat1D(sub.Float1D(si)/a.Float1D(ia), ia) }, a, sub, out) } diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 2f0c76f945..166e289fb6 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -15,7 +15,7 @@ import ( func TestAdd(t *testing.T) { scalar := tensor.NewFloat64Scalar(-5.5) scb := scalar.Clone() - scb.Tensor.SetFloat1D(-4.0, 0) + scb.SetFloat1D(-4.0, 0) scout := scalar.Clone() vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} @@ -26,22 +26,22 @@ func TestAdd(t *testing.T) { cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, ci := cell2d.RowCellIndex(idx) - cell2d.Tensor.SetFloat1D(oned.Tensor.Float1D(ci), i) + cell2d.SetFloat1D(oned.Float1D(ci), i) }, cell2d) // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() Add(scalar, scb, scout) - assert.Equal(t, -5.5+-4, scout.Tensor.Float1D(0)) + assert.Equal(t, -5.5+-4, scout.Float1D(0)) Add(scalar, oned, oneout) for i, v := range vals { - assert.Equal(t, v+-5.5, oneout.Tensor.Float1D(i)) + assert.Equal(t, v+-5.5, oneout.Float1D(i)) } Add(oned, oned, oneout) for i, v := range vals { - assert.Equal(t, v+v, oneout.Tensor.Float1D(i)) + assert.Equal(t, v+v, oneout.Float1D(i)) } Add(cell2d, oned, cellout) @@ -52,24 +52,24 @@ func TestAdd(t *testing.T) { } Sub(scalar, scb, scout) - assert.Equal(t, -5.5 - -4, scout.Tensor.Float1D(0)) + assert.Equal(t, -5.5 - -4, scout.Float1D(0)) Sub(scb, scalar, scout) - assert.Equal(t, -4 - -5.5, scout.Tensor.Float1D(0)) + assert.Equal(t, -4 - -5.5, scout.Float1D(0)) Sub(scalar, oned, oneout) for i, v := range vals { - assert.Equal(t, -5.5-v, oneout.Tensor.Float1D(i)) + assert.Equal(t, -5.5-v, oneout.Float1D(i)) } Sub(oned, scalar, oneout) for i, v := range vals { - assert.Equal(t, v - -5.5, oneout.Tensor.Float1D(i)) + assert.Equal(t, v - -5.5, oneout.Float1D(i)) } Sub(oned, oned, oneout) for i, v := range vals { - assert.Equal(t, v-v, oneout.Tensor.Float1D(i)) + assert.Equal(t, v-v, oneout.Float1D(i)) } Sub(cell2d, oned, cellout) @@ -80,16 +80,16 @@ func TestAdd(t *testing.T) { } Mul(scalar, scb, scout) - assert.Equal(t, -5.5*-4, scout.Tensor.Float1D(0)) + assert.Equal(t, -5.5*-4, scout.Float1D(0)) Mul(scalar, oned, oneout) for i, v := range vals { - assert.Equal(t, v*-5.5, oneout.Tensor.Float1D(i)) + assert.Equal(t, v*-5.5, oneout.Float1D(i)) } Mul(oned, oned, oneout) for i, v := range vals { - assert.Equal(t, v*v, oneout.Tensor.Float1D(i)) + assert.Equal(t, v*v, oneout.Float1D(i)) } Mul(cell2d, oned, cellout) @@ -100,24 +100,24 @@ func TestAdd(t *testing.T) { } Div(scalar, scb, scout) - assert.Equal(t, -5.5/-4, scout.Tensor.Float1D(0)) + assert.Equal(t, -5.5/-4, scout.Float1D(0)) Div(scb, scalar, scout) - assert.Equal(t, -4/-5.5, scout.Tensor.Float1D(0)) + assert.Equal(t, -4/-5.5, scout.Float1D(0)) Div(scalar, oned, oneout) for i, v := range vals { - assert.Equal(t, -5.5/v, oneout.Tensor.Float1D(i)) + assert.Equal(t, -5.5/v, oneout.Float1D(i)) } Div(oned, scalar, oneout) for i, v := range vals { - assert.Equal(t, v/-5.5, oneout.Tensor.Float1D(i)) + assert.Equal(t, v/-5.5, oneout.Float1D(i)) } Div(oned, oned, oneout) for i, v := range vals { - assert.Equal(t, v/v, oneout.Tensor.Float1D(i)) + assert.Equal(t, v/v, oneout.Float1D(i)) } Div(cell2d, oned, cellout) @@ -130,30 +130,30 @@ func TestAdd(t *testing.T) { ZScore(oned, oneout) mout := tensor.NewIndexed(tensor.NewFloat64()) std, mean, _ := stats.StdFuncOut64(oneout, mout) - assert.InDelta(t, 1.0, std.Tensor.Float1D(0), 1.0e-6) - assert.InDelta(t, 0.0, mean.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 1.0, std.Float1D(0), 1.0e-6) + assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) UnitNorm(oned, oneout) stats.MinFunc(oneout, mout) - assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) stats.MaxFunc(oneout, mout) - assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout.Tensor) minv := tensor.NewFloat64Scalar(0) maxv := tensor.NewFloat64Scalar(1) Clamp(oned, minv, maxv, oneout) stats.MinFunc(oneout, mout) - assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) stats.MaxFunc(oneout, mout) - assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout.Tensor) thr := tensor.NewFloat64Scalar(0.5) Binarize(oned, thr, oneout) stats.MinFunc(oneout, mout) - assert.InDelta(t, 0.0, mout.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) stats.MaxFunc(oneout, mout) - assert.InDelta(t, 1.0, mout.Tensor.Float1D(0), 1.0e-6) + assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout.Tensor) } From ae27867511357176b494acd1d33a912da802adcc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 14 Sep 2024 17:08:18 -0700 Subject: [PATCH 034/311] fixed sorting: slices.Sort already translates indexes into actual values.. sheesh. --- tensor/base.go | 46 ++++++++++++++++- tensor/bits.go | 30 ++--------- tensor/indexed.go | 60 ++++++++++------------ tensor/number.go | 30 ++--------- tensor/slice.go | 4 +- tensor/stats/cluster/plot.go | 4 +- tensor/stats/metric/metric_test.go | 2 +- tensor/stats/metric/vec.go | 2 +- tensor/stats/stats/vec.go | 2 +- tensor/string.go | 31 ++---------- tensor/table/columns.go | 17 ++----- tensor/table/indexes.go | 81 +++++++++--------------------- tensor/table/io.go | 2 +- tensor/table/slicetable_test.go | 2 +- tensor/table/table.go | 35 +++++++------ tensor/table/table_test.go | 33 ++++++++++-- tensor/table/util.go | 2 +- tensor/tensor.go | 3 ++ tensor/tensor_test.go | 33 +++++++----- tensor/vectorize.go | 2 +- 20 files changed, 197 insertions(+), 224 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index 022bfa781d..0f2b02958b 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -8,6 +8,7 @@ import ( "fmt" "log" "reflect" + "strings" "unsafe" "cogentcore.org/core/base/metadata" @@ -139,9 +140,15 @@ func (tsr *Base[T]) StringRowCell(row, cell int) string { return reflectx.ToString(tsr.Values[row*sz+cell]) } -// Label satisfies the core.Labeler interface for a summary description of the tensor +// Label satisfies the core.Labeler interface for a summary description of the tensor. func (tsr *Base[T]) Label() string { - return fmt.Sprintf("Tensor: %s", tsr.shape.String()) + nm, _ := metadata.Get[string](tsr.Meta, "name") + if nm != "" { + nm += " " + tsr.shape.String() + } else { + nm = tsr.shape.String() + } + return fmt.Sprintf("Tensor: %s", nm) } // Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the @@ -183,3 +190,38 @@ func (tsr *Base[T]) SymmetricDim() int { } return tsr.shape.DimSize(nd - 1) } + +// stringIndexed is the underlying impl of String that works for indexed +// data as well. +func stringIndexed(tsr Tensor, idxs []int) string { + str := tsr.Label() + sz := tsr.Len() + if sz > 1000 { + return str + } + var b strings.Builder + b.WriteString(str) + b.WriteString("\n") + oddRow := true + rows, cols, _, _ := Projection2DShape(tsr.Shape(), oddRow) + if idxs != nil { + rows = min(rows, len(idxs)) + } + for r := range rows { + rc, _ := Projection2DCoords(tsr.Shape(), oddRow, r, 0) + b.WriteString(fmt.Sprintf("%v: ", rc)) + ri := r + if idxs != nil { + ri = idxs[r] + } + for c := 0; c < cols; c++ { + if tsr.IsString() { + b.WriteString(fmt.Sprintf("%s ", Projection2DString(tsr, oddRow, ri, c))) + } else { + b.WriteString(fmt.Sprintf("%7g ", Projection2DValue(tsr, oddRow, ri, c))) + } + } + b.WriteString("\n") + } + return b.String() +} diff --git a/tensor/bits.go b/tensor/bits.go index c144db1b77..978a118a53 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -8,7 +8,6 @@ import ( "fmt" "log/slog" "reflect" - "strings" "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" @@ -60,6 +59,11 @@ func BoolToFloat64(bv bool) float64 { } } +// String satisfies the fmt.Stringer interface for string of tensor data. +func (tsr *Bits) String() string { + return stringIndexed(tsr, nil) +} + func (tsr *Bits) IsString() bool { return false } @@ -329,27 +333,3 @@ func (tsr *Bits) T() mat.Matrix { slog.Error("tensor T gonum Matrix call made on Bits Tensor; not supported") return mat.Transpose{tsr} } - -// String satisfies the fmt.Stringer interface for string of tensor data -func (tsr *Bits) String() string { - str := tsr.Label() - sz := tsr.Len() - if sz > 1000 { - return str - } - var b strings.Builder - b.WriteString(str) - b.WriteString("\n") - oddRow := true - rows, cols, _, _ := Projection2DShape(&tsr.shape, oddRow) - for r := 0; r < rows; r++ { - rc, _ := Projection2DCoords(&tsr.shape, oddRow, r, 0) - b.WriteString(fmt.Sprintf("%v: ", rc)) - for c := 0; c < cols; c++ { - vl := Projection2DValue(tsr, oddRow, r, c) - b.WriteString(fmt.Sprintf("%g ", vl)) - } - b.WriteString("\n") - } - return b.String() -} diff --git a/tensor/indexed.go b/tensor/indexed.go index d1d820482f..11ff632cfb 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -11,7 +11,6 @@ import ( "math/rand" "slices" "sort" - "strings" ) // Indexed is an indexed wrapper around a tensor.Tensor that provides a @@ -93,12 +92,22 @@ func (ix *Indexed) Index(idx int) int { return ix.Indexes[idx] } +// String satisfies the fmt.Stringer interface for string of tensor data. +func (ix *Indexed) String() string { + return stringIndexed(ix.Tensor, ix.Indexes) +} + +// Label satisfies the core.Labeler interface for a summary description of the tensor. +func (ix *Indexed) Label() string { + return ix.Tensor.Label() +} + // RowCellSize returns the size of the outermost Row shape dimension // (via [Indexed.Rows] method), and the size of all the remaining // inner dimensions (the "cell" size). func (ix *Indexed) RowCellSize() (rows, cells int) { _, cells = ix.Tensor.RowCellSize() - rows = ix.Rows() + rows = ix.NumRows() return } @@ -112,10 +121,10 @@ func (ix *Indexed) RowCellIndex(idx int) (i1d, ri, ci int) { return } -// Rows returns the effective number of rows in this Indexed view, +// NumRows returns the effective number of rows in this Indexed view, // which is the length of the index list or number of outer -// rows dimension of tensor if no indexes. -func (ix *Indexed) Rows() int { +// rows dimension of tensor if no indexes (full sequential view). +func (ix *Indexed) NumRows() int { if ix.Indexes == nil { return ix.Tensor.DimSize(0) } @@ -126,7 +135,7 @@ func (ix *Indexed) Rows() int { // taking into account the Indexes via [Rows], // as Rows() * cell size. func (ix *Indexed) Len() int { - rows := ix.Rows() + rows := ix.NumRows() _, cells := ix.Tensor.RowCellSize() return cells * rows } @@ -138,7 +147,7 @@ func (ix *Indexed) DeleteInvalid() { ix.Indexes = nil return } - ni := ix.Rows() + ni := ix.NumRows() for i := ni - 1; i >= 0; i-- { if ix.Indexes[i] >= ix.Tensor.DimSize(0) { ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) @@ -180,7 +189,7 @@ func (ix *Indexed) ExcludeMissing1D() { //types:add return } ix.IndexesNeeded() - ni := ix.Rows() + ni := ix.NumRows() for i := ni - 1; i >= 0; i-- { if math.IsNaN(ix.Float1D(ix.Indexes[i])) { ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) @@ -205,11 +214,6 @@ func (ix *Indexed) Permuted() { } } -// AddIndex adds a new index to the list -func (ix *Indexed) AddIndex(idx int) { - ix.Indexes = append(ix.Indexes, idx) -} - const ( // Ascending specifies an ascending sort direction for tensor Sort routines Ascending = true @@ -230,7 +234,7 @@ func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) error { } ix.IndexesNeeded() slices.SortFunc(ix.Indexes, func(a, b int) int { - return cmp(ix.Tensor, ix.Indexes[a], ix.Indexes[b]) + return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! }) return nil } @@ -245,21 +249,11 @@ func (ix *Indexed) SortIndexes() { sort.Ints(ix.Indexes) } -// Sort compare function for string values. -func CompareStrings(a, b string, ascending bool) int { - cmp := strings.Compare(a, b) - if !ascending { - cmp = -cmp - } - return cmp -} - -func CompareNumbers(a, b float64, ascending bool) int { - cmp := cmp.Compare(a, b) - if !ascending { - cmp = -cmp +func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { + if ascending { + return cmp.Compare(a, b) } - return cmp + return cmp.Compare(b, a) } // Sort does default alpha or numeric sort of 1D tensor based on data type. @@ -267,11 +261,11 @@ func CompareNumbers(a, b float64, ascending bool) int { func (ix *Indexed) Sort(ascending bool) error { if ix.Tensor.IsString() { ix.SortFunc(func(tsr Tensor, i, j int) int { - return CompareStrings(tsr.String1D(i), tsr.String1D(j), ascending) + return CompareAscending(tsr.String1D(i), tsr.String1D(j), ascending) }) } else { ix.SortFunc(func(tsr Tensor, i, j int) int { - return CompareNumbers(tsr.Float1D(i), tsr.Float1D(j), ascending) + return CompareAscending(tsr.Float1D(i), tsr.Float1D(j), ascending) }) } return nil @@ -290,7 +284,7 @@ func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) error { } ix.IndexesNeeded() slices.SortStableFunc(ix.Indexes, func(a, b int) int { - return cmp(ix.Tensor, ix.Indexes[a], ix.Indexes[b]) + return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! }) return nil } @@ -301,11 +295,11 @@ func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) error { func (ix *Indexed) SortStable(ascending bool) error { if ix.Tensor.IsString() { ix.SortStableFunc(func(tsr Tensor, i, j int) int { - return CompareStrings(tsr.String1D(i), tsr.String1D(j), ascending) + return CompareAscending(tsr.String1D(i), tsr.String1D(j), ascending) }) } else { ix.SortStableFunc(func(tsr Tensor, i, j int) int { - return CompareNumbers(tsr.Float1D(i), tsr.Float1D(j), ascending) + return CompareAscending(tsr.Float1D(i), tsr.Float1D(j), ascending) }) } return nil diff --git a/tensor/number.go b/tensor/number.go index f31c61b741..6fc8c31491 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -9,7 +9,6 @@ import ( "log" "math" "strconv" - "strings" "cogentcore.org/core/base/num" "gonum.org/v1/gonum/mat" @@ -122,6 +121,11 @@ func NewNumberFromSlice[T num.Number](vals []T) Tensor { return tsr } +// String satisfies the fmt.Stringer interface for string of tensor data. +func (tsr *Number[T]) String() string { + return stringIndexed(tsr, nil) +} + func (tsr *Number[T]) IsString() bool { return false } @@ -145,30 +149,6 @@ func (tsr *Number[T]) SetStringRowCell(val string, row, cell int) { } } -// String satisfies the fmt.Stringer interface for string of tensor data -func (tsr *Number[T]) String() string { - str := tsr.Label() - sz := len(tsr.Values) - if sz > 1000 { - return str - } - var b strings.Builder - b.WriteString(str) - b.WriteString("\n") - oddRow := true - rows, cols, _, _ := Projection2DShape(&tsr.shape, oddRow) - for r := 0; r < rows; r++ { - rc, _ := Projection2DCoords(&tsr.shape, oddRow, r, 0) - b.WriteString(fmt.Sprintf("%v: ", rc)) - for c := 0; c < cols; c++ { - vl := Projection2DValue(tsr, oddRow, r, c) - b.WriteString(fmt.Sprintf("%7g ", vl)) - } - b.WriteString("\n") - } - return b.String() -} - func (tsr *Number[T]) Float(i ...int) float64 { return float64(tsr.Values[tsr.shape.Offset(i...)]) } diff --git a/tensor/slice.go b/tensor/slice.go index 669f5cf4da..7d5a7b87d4 100644 --- a/tensor/slice.go +++ b/tensor/slice.go @@ -90,7 +90,7 @@ func SliceSize(sizes []int, ranges ...Range) ([]int, error) { // Use the [SliceSet] function to copy sliced values back to the original. func Slice(tsr, out *Indexed, ranges ...Range) error { sizes := slices.Clone(tsr.Tensor.Shape().Sizes) - sizes[0] = tsr.Rows() // takes into account indexes + sizes[0] = tsr.NumRows() // takes into account indexes nsz, err := SliceSize(sizes, ranges...) if err != nil { return err @@ -129,7 +129,7 @@ func Slice(tsr, out *Indexed, ranges ...Range) error { // can be different). func SliceSet(tsr, slc *Indexed, ranges ...Range) error { sizes := slices.Clone(tsr.Tensor.Shape().Sizes) - sizes[0] = tsr.Rows() // takes into account indexes + sizes[0] = tsr.NumRows() // takes into account indexes nsz, err := SliceSize(sizes, ranges...) if err != nil { return err diff --git a/tensor/stats/cluster/plot.go b/tensor/stats/cluster/plot.go index 48a535ebf1..3c829ac0d3 100644 --- a/tensor/stats/cluster/plot.go +++ b/tensor/stats/cluster/plot.go @@ -27,7 +27,7 @@ func Plot(pt *table.Table, root *Node, dmat, labels *tensor.Indexed) { // will render this node in a cluster plot when plotted with a standard plotting package. // The lines double-back on themselves to form a continuous line to be plotted. func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { - row := pt.Rows() + row := pt.NumRows() xc := pt.ColumnIndex(0) yc := pt.ColumnIndex(1) lbl := pt.ColumnIndex(2) @@ -47,7 +47,7 @@ func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { xc.SetFloatRowCell(nn.ParDist+nn.Dist, row, 0) yc.SetFloatRowCell(kn.Y, row, 0) kn.Plot(pt, dmat, labels) - row = pt.Rows() + row = pt.NumRows() pt.SetNumRows(row + 1) xc.SetFloatRowCell(nn.ParDist, row, 0) yc.SetFloatRowCell(kn.Y, row, 0) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index e20e811f59..5c2a7322a1 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -101,7 +101,7 @@ func TestPCAIris(t *testing.T) { // https://plot.ly/ipython-notebooks/principal-component-analysis/ dt := table.NewTable() - dt.AddFloat64TensorColumn("data", 4) + dt.AddFloat64Column("data", 4) dt.AddStringColumn("class") err := dt.OpenCSV("testdata/iris.data", tensor.Comma) if err != nil { diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index ccaa6f9257..51e54bd960 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -51,7 +51,7 @@ func NFunc(tsr ...*tensor.Indexed) int { return 0 } a, b, out := tsr[0], tsr[1], tsr[nt-1] - na, nb := a.Rows(), b.Rows() + na, nb := a.NumRows(), b.NumRows() osh := OutShape(a.Tensor.Shape()) out.Tensor.SetShape(osh.Sizes...) out.Tensor.SetNames(osh.Names...) diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 775ebe151c..ead0d3535b 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -79,7 +79,7 @@ func NFunc(tsr ...*tensor.Indexed) int { return 0 } in, out := tsr[0], tsr[nt-1] - n := in.Rows() + n := in.NumRows() osh := OutShape(in.Tensor.Shape()) out.Tensor.SetShape(osh.Sizes...) out.Tensor.SetNames(osh.Names...) diff --git a/tensor/string.go b/tensor/string.go index da5666be0e..5cb198c157 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -9,7 +9,6 @@ import ( "log" "math" "strconv" - "strings" "gonum.org/v1/gonum/mat" ) @@ -62,6 +61,11 @@ func Float64ToString(val float64) string { return strconv.FormatFloat(val, 'g', -1, 64) } +// String satisfies the fmt.Stringer interface for string of tensor data. +func (tsr *String) String() string { + return stringIndexed(tsr, nil) +} + func (tsr *String) IsString() bool { return true } @@ -80,31 +84,6 @@ func (tsr *String) SetStringRowCell(val string, row, cell int) { tsr.Values[row*sz+cell] = val } -// String satisfies the fmt.Stringer interface for string of tensor data -func (tsr *String) String() string { - str := tsr.Label() - sz := len(tsr.Values) - if sz > 1000 { - return str - } - var b strings.Builder - b.WriteString(str) - b.WriteString("\n") - oddRow := true - rows, cols, _, _ := Projection2DShape(&tsr.shape, oddRow) - for r := 0; r < rows; r++ { - rc, _ := Projection2DCoords(&tsr.shape, oddRow, r, 0) - b.WriteString(fmt.Sprintf("%v: ", rc)) - for c := 0; c < cols; c++ { - idx := Projection2DIndex(tsr.Shape(), oddRow, r, c) - vl := tsr.Values[idx] - b.WriteString(vl) - } - b.WriteString("\n") - } - return b.String() -} - func (tsr *String) Float(i ...int) float64 { return StringToFloat64(tsr.Values[tsr.shape.Offset(i...)]) } diff --git a/tensor/table/columns.go b/tensor/table/columns.go index cda3b500d1..64b6550537 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -5,13 +5,11 @@ package table import ( - "fmt" - "cogentcore.org/core/tensor" ) // Columns is the underlying column list and number of rows for Table. -// Table is an Indexed view onto the Columns. +// [Table] is an Indexed view onto the Columns. type Columns struct { tensor.List @@ -25,16 +23,9 @@ func NewColumns() *Columns { return &Columns{} } -// IsValidRow returns error if the row is invalid -func (cl *Columns) IsValidRow(row int) error { - if row < 0 || row >= cl.Rows { - return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, cl.Rows) - } - return nil -} - -// SetNumRows sets the number of rows in the table, across all columns -// if rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0 +// SetNumRows sets the number of rows in the table, across all columns. +// If rows = 0 then the effective number of rows in tensors is 1, +// as this dim cannot be 0. func (cl *Columns) SetNumRows(rows int) *Columns { //types:add cl.Rows = rows // can be 0 rows = max(1, rows) diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index b8b8b1cc05..000aebb40e 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -9,6 +9,8 @@ import ( "slices" "sort" "strings" + + "cogentcore.org/core/tensor" ) // Index returns the actual index into underlying tensor row based on given @@ -20,9 +22,9 @@ func (dt *Table) Index(idx int) int { return dt.Indexes[idx] } -// Rows returns the number of rows, which is the number of Indexes if present, -// else actual number of rows in Tensor data. -func (dt *Table) Rows() int { +// NumRows returns the number of rows, which is the number of Indexes if present, +// else actual number of [Columns.Rows]. +func (dt *Table) NumRows() int { if dt.Indexes == nil { return dt.Columns.Rows } @@ -55,7 +57,7 @@ func (dt *Table) DeleteInvalid() { dt.Indexes = nil return } - ni := dt.Rows() + ni := dt.NumRows() for i := ni - 1; i >= 0; i-- { if dt.Indexes[i] >= dt.Columns.Rows { dt.Indexes = append(dt.Indexes[:i], dt.Indexes[i+1:]...) @@ -80,11 +82,6 @@ func (dt *Table) Permuted() { } } -// AddIndex adds a new index to the list -func (dt *Table) AddIndex(idx int) { - dt.Indexes = append(dt.Indexes, idx) -} - // SortFunc sorts the indexes into our Table using given compare function. // The compare function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. @@ -93,7 +90,7 @@ func (dt *Table) AddIndex(idx int) { func (dt *Table) SortFunc(cmp func(dt *Table, i, j int) int) { dt.IndexesNeeded() slices.SortFunc(dt.Indexes, func(a, b int) int { - return cmp(dt, dt.Indexes[a], dt.Indexes[b]) + return cmp(dt, a, b) // key point: these are already indirected through indexes!! }) } @@ -107,19 +104,6 @@ func (dt *Table) SortIndexes() { sort.Ints(dt.Indexes) } -// SortColumn sorts the indexes into our Table according to values in -// given column name, using either ascending or descending order. -// Only valid for 1-dimensional columns. -// Returns error if column name not found or column is not 1D. -func (dt *Table) SortColumn(column string, ascending bool) error { //types:add - dt.IndexesNeeded() - cl, err := dt.ColumnTry(column) // has our indexes - if err != nil { - return err - } - return cl.Sort(ascending) -} - // SortColumns sorts the indexes into our Table according to values in // given column names, using either ascending or descending order, // and optionally using a stable sort. @@ -148,34 +132,16 @@ func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { } sf(func(dt *Table, i, j int) int { for _, ci := range colIndexes { - cl := dt.Columns.Values[ci] + cl := dt.ColumnIndex(ci).Tensor if cl.IsString() { - if ascending { - if cl.String1D(i) < cl.String1D(j) { - return 1 - } else if cl.String1D(i) > cl.String1D(j) { - return -1 - } // if equal, fallthrough to next col - } else { - if cl.String1D(i) > cl.String1D(j) { - return 1 - } else if cl.String1D(i) < cl.String1D(j) { - return -1 - } // if equal, fallthrough to next col + v := tensor.CompareAscending(cl.StringRowCell(i, 0), cl.StringRowCell(j, 0), ascending) + if v != 0 { + return v } } else { - if ascending { - if cl.Float1D(i) < cl.Float1D(j) { - return 1 - } else if cl.Float1D(i) > cl.Float1D(j) { - return -1 - } // if equal, fallthrough to next col - } else { - if cl.Float1D(i) > cl.Float1D(j) { - return 1 - } else if cl.Float1D(i) < cl.Float1D(j) { - return -1 - } // if equal, fallthrough to next col + v := tensor.CompareAscending(cl.FloatRowCell(i, 0), cl.FloatRowCell(j, 0), ascending) + if v != 0 { + return v } } } @@ -196,7 +162,7 @@ func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { func (dt *Table) SortStableFunc(cmp func(dt *Table, i, j int) int) { dt.IndexesNeeded() slices.SortStableFunc(dt.Indexes, func(a, b int) int { - return cmp(dt, dt.Indexes[a], dt.Indexes[b]) + return cmp(dt, a, b) // key point: these are already indirected through indexes!! }) } @@ -289,7 +255,9 @@ func (dt *Table) NewTable() *Table { return nt } -// DeleteRows deletes n rows of indexes starting at given index in the list of indexes +// DeleteRows deletes n rows of Indexes starting at given index in the list of indexes. +// This does not affect the underlying tensor data; To create an actual in-memory +// ordering with rows deleted, use [Table.NewTable]. func (dt *Table) DeleteRows(at, n int) { dt.IndexesNeeded() dt.Indexes = append(dt.Indexes[:at], dt.Indexes[at+n:]...) @@ -307,10 +275,11 @@ const ( UseCase = false ) -// RowsByString returns the list of *our indexes* whose row in the table has -// given string value in given column name (de-reference our indexes to get actual row). -// if contains, only checks if row contains string; if ignoreCase, ignores case. -// Use named args for greater clarity. +// RowsByString returns the list of row _indexes_ (not necessarily underlying row numbers, +// if Indexes are in place) whose row in the table has given string value in given column name. +// The results can be used as row indexes to Indexed tensor column data. +// If contains, only checks if row contains string; if ignoreCase, ignores case. +// Use the named const args [Contains], [Equals], [IgnoreCase], [UseCase] for greater clarity. func (dt *Table) RowsByString(colname string, str string, contains, ignoreCase bool) []int { col := dt.Column(colname) if col == nil { @@ -318,10 +287,10 @@ func (dt *Table) RowsByString(colname string, str string, contains, ignoreCase b } lowstr := strings.ToLower(str) var indexes []int - rows := dt.Rows() + rows := dt.NumRows() for idx := range rows { srw := dt.Index(idx) - val := col.Tensor.String1D(srw) + val := col.Tensor.StringRowCell(srw, 0) has := false switch { case contains && ignoreCase: diff --git a/tensor/table/io.go b/tensor/table/io.go index 2374e5d62f..143e159220 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -334,7 +334,7 @@ func (dt *Table) WriteCSV(w io.Writer, delim tensor.Delims, headers bool) error } cw := csv.NewWriter(w) cw.Comma = delim.Rune() - nrow := dt.Rows() + nrow := dt.NumRows() for ri := range nrow { ix := dt.Index(ri) err = dt.WriteCSVRowWriter(cw, ix, ncol) diff --git a/tensor/table/slicetable_test.go b/tensor/table/slicetable_test.go index 4d23a9fdab..c280ba68fb 100644 --- a/tensor/table/slicetable_test.go +++ b/tensor/table/slicetable_test.go @@ -26,5 +26,5 @@ func TestSliceTable(t *testing.T) { if err != nil { t.Error(err.Error()) } - assert.Equal(t, 2, dt.Rows()) + assert.Equal(t, 2, dt.NumRows()) } diff --git a/tensor/table/table.go b/tensor/table/table.go index 09bc6bf60e..ac95aa1cdb 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -63,21 +63,18 @@ func NewTableView(src *Table) *Table { // IsValidRow returns error if the row is invalid, if error checking is needed. func (dt *Table) IsValidRow(row int) error { - if row < 0 || row >= dt.Rows() { - return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, dt.Rows()) + if row < 0 || row >= dt.NumRows() { + return fmt.Errorf("table.Table IsValidRow: row %d is out of valid range [0..%d]", row, dt.NumRows()) } return nil } -// NumRows returns the number of rows. -func (dt *Table) NumRows() int { return dt.Columns.Rows } - // NumColumns returns the number of columns. func (dt *Table) NumColumns() int { return dt.Columns.Len() } // Column returns the tensor with given column name, as a [tensor.Indexed] // with the shared [Table.Indexes] from this table. It is best practice to -// access columns by name, and direct access through Columns does not +// access columns by name, and direct access through [Table.Columns] does not // provide the shared table-wide Indexes. // Returns nil if not found. func (dt *Table) Column(name string) *tensor.Indexed { @@ -98,11 +95,11 @@ func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { return nil, fmt.Errorf("table.Table: Column named %q not found", name) } -// ColumnIndex returns the tensor at the given index, as a [tensor.Indexed] -// with the shared [Table.Indexes] from this table. It is best practice to -// access columns by name using [Table.Column] method instead. -// Direct access through Columns does not provide the shared table-wide Indexes. -// Returns nil if not found. +// ColumnIndex returns the tensor at the given column index, +// as a [tensor.Indexed] with the shared [Table.Indexes] from this table. +// It is best practice to instead access columns by name using [Table.Column]. +// Direct access through [Table.Columns} does not provide the shared table-wide Indexes. +// Will panic if out of range. func (dt *Table) ColumnIndex(idx int) *tensor.Indexed { cl := dt.Columns.Values[idx] return tensor.NewIndexed(cl, dt.Indexes) @@ -207,30 +204,32 @@ func (dt *Table) DeleteColumnIndex(i, j int) { // DeleteAll deletes all columns, does full reset. func (dt *Table) DeleteAll() { + dt.Indexes = nil dt.Columns.Reset() } -// AddRows adds n rows to end of underlying Table, and to the indexes in this view +// AddRows adds n rows to end of underlying Table, and to the indexes in this view. func (dt *Table) AddRows(n int) *Table { //types:add - dt.Columns.SetNumRows(dt.Columns.Rows + n) - return dt + return dt.SetNumRows(dt.Columns.Rows + n) } // InsertRows adds n rows to end of underlying Table, and to the indexes starting at // given index in this view, providing an efficient insertion operation that only -// exists in the indexed view. To create an in-memory ordering, use [Table.NewTable]. +// exists in the indexed view. To create an in-memory ordering, use [Table.NewTable]. func (dt *Table) InsertRows(at, n int) *Table { dt.IndexesNeeded() strow := dt.Columns.Rows stidx := len(dt.Indexes) - dt.Columns.SetNumRows(strow + n) // adds n indexes to end of list + dt.SetNumRows(strow + n) // adds n indexes to end of list // move those indexes to at:at+n in index list dt.Indexes = append(dt.Indexes[:at], append(dt.Indexes[stidx:], dt.Indexes[at:]...)...) + dt.Indexes = dt.Indexes[:strow+n] return dt } -// SetNumRows sets the number of rows in the table, across all columns -// if rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0 +// SetNumRows sets the number of rows in the table, across all columns. +// If rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0. +// If indexes are in place and rows are added, indexes for the new rows are added. func (dt *Table) SetNumRows(rows int) *Table { //types:add strow := dt.Columns.Rows dt.Columns.SetNumRows(rows) diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index bdf9214a03..0f25a70c6f 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -8,6 +8,7 @@ import ( "strconv" "testing" + "cogentcore.org/core/tensor" "github.com/stretchr/testify/assert" ) @@ -43,7 +44,7 @@ func NewTestTable() *Table { dt.AddFloat64Column("Flt64") dt.AddIntColumn("Int") dt.SetNumRows(3) - for i := range dt.Rows() { + for i := range dt.NumRows() { dt.Column("Str").SetStringRowCell(strconv.Itoa(i), i, 0) dt.Column("Flt64").SetFloatRowCell(float64(i), i, 0) dt.Column("Int").SetFloatRowCell(float64(i), i, 0) @@ -51,14 +52,14 @@ func NewTestTable() *Table { return dt } -func TestAppendRows(t *testing.T) { +func TestAppendRowsEtc(t *testing.T) { st := NewTestTable() dt := NewTestTable() dt.AppendRows(st) dt.AppendRows(st) dt.AppendRows(st) for j := range 3 { - for i := range st.Rows() { + for i := range st.NumRows() { sr := j*3 + i ss := st.Column("Str").StringRowCell(i, 0) ds := dt.Column("Str").StringRowCell(sr, 0) @@ -73,4 +74,30 @@ func TestAppendRows(t *testing.T) { assert.Equal(t, sf, df) } } + ixs := dt.RowsByString("Str", "1", Equals, UseCase) + assert.Equal(t, []int{1, 4, 7, 10}, ixs) + + dt.IndexesNeeded() + dt.SortColumns(tensor.Descending, true, "Int", "Flt64") + assert.Equal(t, []int{2, 5, 8, 11, 1, 4, 7, 10, 0, 3, 6, 9}, dt.Indexes) +} + +func TestSetNumRows(t *testing.T) { + st := NewTestTable() + dt := NewTestTable() + dt.AppendRows(st) + dt.AppendRows(st) + dt.AppendRows(st) + dt.IndexesNeeded() + dt.SetNumRows(3) + assert.Equal(t, []int{0, 1, 2}, dt.Indexes) +} + +func TestInsertDeleteRows(t *testing.T) { + dt := NewTestTable() + dt.IndexesNeeded() + dt.InsertRows(1, 2) + assert.Equal(t, []int{0, 3, 4, 1, 2}, dt.Indexes) + dt.DeleteRows(1, 2) + assert.Equal(t, []int{0, 1, 2}, dt.Indexes) } diff --git a/tensor/table/util.go b/tensor/table/util.go index 697c230be7..1e8e0451b4 100644 --- a/tensor/table/util.go +++ b/tensor/table/util.go @@ -43,7 +43,7 @@ func (dt *Table) ConfigFromTable(ft *Table) error { nmcol := ft.ColumnIndex(0) tycol := ft.ColumnIndex(1) var errs []error - for i := range ft.Rows() { + for i := range ft.NumRows() { name := nmcol.String1D(i) typ := strings.ToLower(tycol.String1D(i)) kind := reflect.Float64 diff --git a/tensor/tensor.go b/tensor/tensor.go index 111d3cb08b..7b0253a61d 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -38,6 +38,9 @@ type Tensor interface { fmt.Stringer mat.Matrix + // Label satisfies the core.Labeler interface for a summary description of the tensor. + Label() string + // Shape returns a pointer to the Shape that fully parametrizes // the tensor shape. Shape() *Shape diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index d0da67091b..a1ce2272c3 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -63,13 +63,12 @@ func TestTensorString(t *testing.T) { _, err = metadata.Get[string](*tsr.Metadata(), "type") assert.Error(t, err) - var flt []float64 cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) - cln.Floats(&flt) - assert.Equal(t, 3.14, flt[0]) - assert.Equal(t, 0.0, flt[1]) + af := AsFloat64(cln) + assert.Equal(t, 3.14, af.Float1D(0)) + assert.Equal(t, 0.0, af.Float1D(1)) } func TestTensorFloat64(t *testing.T) { @@ -116,17 +115,16 @@ func TestTensorFloat64(t *testing.T) { tsr.SetNumRows(5) assert.Equal(t, 20, tsr.Len()) - var flt []float64 cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) - cln.Floats(&flt) - assert.Equal(t, 3.14, flt[0]) - assert.Equal(t, 0.0, flt[1]) + af := AsFloat64(cln) + assert.Equal(t, 3.14, af.Float1D(0)) + assert.Equal(t, 0.0, af.Float1D(1)) } func TestSlice(t *testing.T) { - ft := NewFloat64(3, 4, 5) + ft := NewFloat64Indexed(3, 4, 5) for r := range 3 { for y := range 4 { for x := range 5 { @@ -136,7 +134,7 @@ func TestSlice(t *testing.T) { } } // fmt.Println(ft.String()) - sf := NewFloat64() + sf := NewFloat64Indexed() Slice(ft, sf, Range{}, Range{Start: 1, End: 2}) // fmt.Println(sf.String()) res := `Tensor: [3, 1, 5] @@ -144,7 +142,7 @@ func TestSlice(t *testing.T) { [1 0]: 210 211 212 213 214 [2 0]: 310 311 312 313 314 ` - assert.Equal(t, res, sf.String()) + assert.Equal(t, res, sf.Tensor.String()) Slice(ft, sf, Range{}, Range{}, Range{Start: 1, End: 2}) // fmt.Println(sf.String()) @@ -162,5 +160,16 @@ func TestSlice(t *testing.T) { [2 2]: 321 [2 3]: 331 ` - assert.Equal(t, res, sf.String()) + assert.Equal(t, res, sf.Tensor.String()) +} + +func TestSortFilter(t *testing.T) { + tsr := NewFloat64Indexed(5) + for i := range 5 { + tsr.SetFloatRowCell(float64(i), i, 0) + } + tsr.Sort(Ascending) + assert.Equal(t, []int{0, 1, 2, 3, 4}, tsr.Indexes) + tsr.Sort(Descending) + assert.Equal(t, []int{4, 3, 2, 1, 0}, tsr.Indexes) } diff --git a/tensor/vectorize.go b/tensor/vectorize.go index 795b07cbc3..9fa6a5390a 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -112,7 +112,7 @@ func NFirstRows(tsr ...*Indexed) int { if len(tsr) == 0 { return 0 } - return tsr[0].Rows() + return tsr[0].NumRows() } // NFirstLen is an N function for Vectorize that returns the number of From 2fa44abf505cb29ae2402f97c7bd7cea063a020c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 14 Sep 2024 22:59:22 -0700 Subject: [PATCH 035/311] rest of tests and README docs update on usage, etc. --- tensor/README.md | 58 +++++++++++++++++-- tensor/indexed.go | 47 ++++++++++++++- tensor/table/README.md | 101 +++++++-------------------------- tensor/table/indexes.go | 113 ++++++++++--------------------------- tensor/table/table_test.go | 21 ++++++- tensor/tensor_test.go | 8 +++ 6 files changed, 178 insertions(+), 170 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index 1a080a1050..b2688db17a 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -37,9 +37,7 @@ All tensor package functions are registered using a single name to function map # Standard shapes and dimensional terminology -In general, **1D** refers to a flat, 1-dimensional list, and **ND** is used by contrast to refer to the _2+_ dimensional case. - -There are various standard shapes of tensor data that different functions expect: +In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: * **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one row-wise dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data (indexes are only available for the outermost row dimension). @@ -49,7 +47,59 @@ There are various standard shapes of tensor data that different functions expect * **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations. - +# Cheat Sheet + +`ix` is the `Indexed` tensor for these examples: + +## Table Access + +### 1D + +```Go +// 5th element in tensor regardless of shape: +val := ix.Float1D(5) +``` + +```Go +// value as a string regardless of underlying data type; numbers converted to strings. +str := ix.String1D(2) +``` + +### 2D Row, Cell + +```Go +// value at row 3, cell 2 (flat index into entire `SubSpace` tensor for this row) +// The row index will be indirected through any `Indexes` present on the Indexed view. +val := ix.FloatRowCell(3, 2) +// string value at row 2, cell 0. this is safe for 1D and 2D+ shapes +// and is a robust way to get 1D data from tensors of unknown shapes. +str := ix.FloatRowCell(2, 0) +``` + +```Go +// get the whole n-dimensional tensor of data cells at given row. +// row is indirected through indexes. +// the resulting tensor is a "subslice" view into the underlying data +// so changes to it will automatically update the parent tensor. +tsr := ix.RowTensor(4) +.... +// set all n-dimensional tensor values at given row from given tensor. +ix.SetRowTensor(tsr, 4) +``` + +```Go +// returns a flat, 1D Indexed view into n-dimensional tensor values at +// given row. This is used in compute routines that operate generically +// on the entire row as a flat pattern. +ci := ix.Cells1D(5) +``` + +### Full N-dimensional Indexes + +```Go +// for 3D data +val := ix.Float(3,2,1) +``` # History diff --git a/tensor/indexed.go b/tensor/indexed.go index 11ff632cfb..b0454a3673 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -11,6 +11,7 @@ import ( "math/rand" "slices" "sort" + "strings" ) // Indexed is an indexed wrapper around a tensor.Tensor that provides a @@ -310,7 +311,7 @@ func (ix *Indexed) SortStable(ascending bool) error { // view of the tensor, and false if it should be removed. type FilterFunc func(tsr Tensor, row int) bool -// Filter filters the indexes into our Tensor using given Filter function. +// Filter filters the indexes using given Filter function. // The Filter function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { @@ -323,6 +324,50 @@ func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { } } +// Named arg values for Contains, IgnoreCase +const ( + // Include means include matches + Include = false + // Exclude means exclude matches + Exclude = true + // Contains means the string only needs to contain the target string (see Equals) + Contains = true + // Equals means the string must equal the target string (see Contains) + Equals = false + // IgnoreCase means that differences in case are ignored in comparing strings + IgnoreCase = true + // UseCase means that case matters when comparing strings + UseCase = false +) + +// FilterString filters the indexes using string values compared to given +// string. Includes rows with matching values unless exclude is set. +// If contains, only checks if row contains string; if ignoreCase, ignores case. +// Use the named const args [Include], [Exclude], [Contains], [Equals], +// [IgnoreCase], [UseCase] for greater clarity. +// Only valid for 1-dimensional columns. +func (ix *Indexed) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add + lowstr := strings.ToLower(str) + ix.Filter(func(tsr Tensor, row int) bool { + val := tsr.StringRowCell(row, 0) + has := false + switch { + case contains && ignoreCase: + has = strings.Contains(strings.ToLower(val), lowstr) + case contains: + has = strings.Contains(val, str) + case ignoreCase: + has = strings.EqualFold(val, str) + default: + has = (val == str) + } + if exclude { + return !has + } + return has + }) +} + // NewTensor returns a new tensor with column data organized according to // the Indexes. If Indexes are nil, a clone of the current tensor is returned // but this function is only sensible if there is an indexed view in place. diff --git a/tensor/table/README.md b/tensor/table/README.md index b5035f0b8d..24ec4c3318 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -6,106 +6,45 @@ See [examples/dataproc](examples/dataproc) for a demo of how to use this system for data analysis, paralleling the example in [Python Data Science](https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html) using pandas, to see directly how that translates into this framework. -Whereas an individual `Tensor` can only hold one data type, the `table` allows coordinated storage and processing of heterogeneous data types, aligned by the outermost row dimension. While the main data processing functions are defined on the individual tensors (which are the universal computational element in the `tensor` system), the coordinated row-wise indexing in the table is important for sorting or filtering a collection of data in the same way, and grouping data by a common set of "splits" for data analysis. Plotting is also driven by the table, with one column providing a shared X axis for the rest of the columns. +Whereas an individual `Tensor` can only hold one data type, the `Table` allows coordinated storage and processing of heterogeneous data types, aligned by the outermost row dimension. The main `tensor` data processing functions are defined on the individual tensors (which are the universal computational element in the `tensor` system), but the coordinated row-wise indexing in the table is important for sorting or filtering a collection of data in the same way, and grouping data by a common set of "splits" for data analysis. Plotting is also driven by the table, with one column providing a shared X axis for the rest of the columns. -As a general convention, it is safest, clearest, and quite fast to access columns by name instead of index (there is a map that caches the column indexes), so the base access method names generally take a column name argument, and those that take a column index have an `Index` suffix. +The `Table` mainly provides "infrastructure" methods for adding tensor columns and CSV (comma separated values, and related tab separated values, TSV) file reading and writing. Any function that can be performed on an individual column should be done using the `tensor.Indexed` and `Tensor` methods directly. -The table itself stores raw data `tensor.Tensor` values, and the `Column` (by index) and `ColumnByName` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). It is best to use the table-wise `Sort` and `Filter` methods (and any others that affect the indexes) to ensure the indexes are properly coordinated. Resetting the column tensor indexes to `nil` (via the `Sequential` method) will break any connection to the table indexes, so that any subsequent index-altering operations on that indexed tensor will be fine. +As a general convention, it is safest, clearest, and quite fast to access columns by name instead of index (there is a `map` from name to index), so the base access method names generally take a column name argument, and those that take a column index have an `Index` suffix. -It is very low-cost to create a new View of a +The table itself stores raw data `tensor.Tensor` values, and the `Column` (by name) and `ColumnIndex` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). -# Cheat Sheet - -`dt` is the Table pointer variable for examples below: - -## Table Access - -Scalar columns: - -```Go -val := dt.Float("ColName", row) -``` - -```Go -str := dt.StringValue("ColName", row) -``` - -Tensor (higher-dimensional) columns: - -```Go -tsr := dt.Tensor("ColName", row) // entire tensor at cell (a row-level SubSpace of column tensor) -``` - -```Go -val := dt.TensorFloat1D("ColName", row, cellidx) // idx is 1D index into cell tensor -``` - -## Set Table Value - -```Go -dt.SetFloat("ColName", row, val) -``` +If you call Sort, Filter or other routines on an individual column tensor, then you can grab the updated indexes via the `IndexesFromTensor` method so that they apply to the entire table. -```Go -dt.SetString("ColName", row, str) -``` - -Tensor (higher-dimensional) columns: - -```Go -dt.SetTensor("ColName", row, tsr) // set entire tensor at cell -``` +There are also multi-column `Sort` and `Filter` methods on the Table itself. -```Go -dt.SetTensorFloat1D("ColName", row, cellidx, val) // idx is 1D index into cell tensor -``` - -## Find Value(s) in Column +It is very low-cost to create a new View of an existing Table, via `NewTableView`, as they can share the underlying `Columns` data. -Returns all rows where value matches given value, in string form (any number will convert to a string) - -```Go -rows := dt.RowsByString("ColName", "value", etable.Contains, etable.IgnoreCase) -``` - -Other options are `etable.Equals` instead of `Contains` to search for an exact full string, and `etable.UseCase` if case should be used instead of ignored. - -## Index Views (Sort, Filter, etc) +# Cheat Sheet -The [Indexed](https://godoc.org/github.com/goki/etable/v2/etable#Indexed) provides a list of row-wise indexes into a table, and Sorting, Filtering and Splitting all operate on this index view without changing the underlying table data, for maximum efficiency and flexibility. +`dt` is the Table pointer variable for examples below: -```Go -ix := etable.NewIndexed(et) // new view with all rows -``` +## Table Access -### Sort +Column data access: ```Go -ix.SortColumnName("Name", etable.Ascending) // etable.Ascending or etable.Descending -SortedTable := ix.NewTable() // turn an Indexed back into a new Table organized in order of indexes +// FloatRowCell is method on the `tensor.Indexed` returned from `Column` method. +// This is the best method to use in general for generic 1D data access, +// as it works on any data from 1D on up (although it only samples the first value +// from higher dimensional data) . +val := dt.Column("Values").FloatRowCell(3, 0) ``` -or: - ```Go -nmcl := dt.Column("Name") // nmcl is an etensor of the Name column, cached -ix.Sort(func(t *Table, i, j int) bool { - return nmcl.StringValue1D(i) < nmcl.StringValue1D(j) -}) +dt.Column("Name").SetStringRowCell(4, 0) ``` -### Filter +todo: more -```Go -nmcl := dt.Column("Name") // column we're filtering on -ix.Filter(func(t *Table, row int) bool { - // filter return value is for what to *keep* (=true), not exclude - // here we keep any row with a name that contains the string "in" - return strings.Contains(nmcl.StringValue1D(row), "in") -}) -``` +## Sorting and Filtering -### Splits ("pivot tables" etc), Aggregation +## Splits ("pivot tables" etc), Aggregation Create a table of mean values of "Data" column grouped by unique entries in "Name" column, resulting table will be called "DataMean": diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 000aebb40e..4a27b03c97 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -40,7 +40,7 @@ func (dt *Table) Sequential() { //types:add // e.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise // current indexes are left as is. Use Sequential, then IndexesNeeded to ensure // all rows are represented. -func (dt *Table) IndexesNeeded() { //types:add +func (dt *Table) IndexesNeeded() { if dt.Indexes != nil { return } @@ -50,6 +50,13 @@ func (dt *Table) IndexesNeeded() { //types:add } } +// IndexesFromTensor copies Indexes from the given [tensor.Indexed] tensor, +// including if they are nil. This allows column-specific Sort, Filter and +// other such methods to be applied to the entire table. +func (dt *Table) IndexesFromTensor(ix *tensor.Indexed) { + dt.Indexes = ix.Indexes +} + // DeleteInvalid deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from table. func (dt *Table) DeleteInvalid() { @@ -94,18 +101,23 @@ func (dt *Table) SortFunc(cmp func(dt *Table, i, j int) int) { }) } -// SortIndexes sorts the indexes into our Table directly in -// numerical order, producing the native ordering, while preserving -// any filtering that might have occurred. -func (dt *Table) SortIndexes() { - if dt.Indexes == nil { - return - } - sort.Ints(dt.Indexes) +// SortStableFunc stably sorts the indexes into our Table using given compare function. +// The compare function operates directly on row numbers into the Table +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +// It is *essential* that it always returns 0 when the two are equal +// for the stable function to actually work. +func (dt *Table) SortStableFunc(cmp func(dt *Table, i, j int) int) { + dt.IndexesNeeded() + slices.SortStableFunc(dt.Indexes, func(a, b int) int { + return cmp(dt, a, b) // key point: these are already indirected through indexes!! + }) } // SortColumns sorts the indexes into our Table according to values in // given column names, using either ascending or descending order, +// (use [tensor.Ascending] or [tensor.Descending] for self-documentation, // and optionally using a stable sort. // Only valid for 1-dimensional columns. // Returns error if column name not found. @@ -149,34 +161,14 @@ func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { }) } -///////////////////////////////////////////////////////////////////////// -// Stable sorts -- sometimes essential.. - -// SortStableFunc stably sorts the indexes into our Table using given compare function. -// The compare function operates directly on row numbers into the Table -// as these row numbers have already been projected through the indexes. -// cmp(a, b) should return a negative number when a < b, a positive -// number when a > b and zero when a == b. -// It is *essential* that it always returns 0 when the two are equal -// for the stable function to actually work. -func (dt *Table) SortStableFunc(cmp func(dt *Table, i, j int) int) { - dt.IndexesNeeded() - slices.SortStableFunc(dt.Indexes, func(a, b int) int { - return cmp(dt, a, b) // key point: these are already indirected through indexes!! - }) -} - -// SortStableColumn sorts the indexes into our Table according to values in -// given column name, using either ascending or descending order. -// Only valid for 1-dimensional columns. -// Returns error if column name not found. -func (dt *Table) SortStableColumn(column string, ascending bool) error { - dt.IndexesNeeded() - cl, err := dt.ColumnTry(column) // has our indexes - if err != nil { - return err +// SortIndexes sorts the indexes into our Table directly in +// numerical order, producing the native ordering, while preserving +// any filtering that might have occurred. +func (dt *Table) SortIndexes() { + if dt.Indexes == nil { + return } - return cl.SortStable(ascending) + sort.Ints(dt.Indexes) } // FilterFunc is a function used for filtering that returns @@ -197,40 +189,6 @@ func (dt *Table) Filter(filterer func(dt *Table, row int) bool) { } } -// FilterColumn filters the indexes into our Table according to values in -// given column name, using string representation of column values. -// Includes rows with matching values unless exclude is set. -// If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use named args for greater clarity. -// Only valid for 1-dimensional columns. -// Returns error if column name not found. -func (dt *Table) FilterColumn(column string, str string, exclude, contains, ignoreCase bool) error { //types:add - col, err := dt.ColumnTry(column) - if err != nil { - return err - } - lowstr := strings.ToLower(str) - dt.Filter(func(dt *Table, row int) bool { - val := col.StringRowCell(row, 0) - has := false - switch { - case contains && ignoreCase: - has = strings.Contains(strings.ToLower(val), lowstr) - case contains: - has = strings.Contains(val, str) - case ignoreCase: - has = strings.EqualFold(val, str) - default: - has = (val == str) - } - if exclude { - return !has - } - return has - }) - return nil -} - // NewTable returns a new table with column data organized according to // the indexes. If Indexes are nil, a clone of the current tensor is returned // but this function is only sensible if there is an indexed view in place. @@ -263,23 +221,12 @@ func (dt *Table) DeleteRows(at, n int) { dt.Indexes = append(dt.Indexes[:at], dt.Indexes[at+n:]...) } -// Named arg values for Contains, IgnoreCase -const ( - // Contains means the string only needs to contain the target string (see Equals) - Contains bool = true - // Equals means the string must equal the target string (see Contains) - Equals = false - // IgnoreCase means that differences in case are ignored in comparing strings - IgnoreCase = true - // UseCase means that case matters when comparing strings - UseCase = false -) - // RowsByString returns the list of row _indexes_ (not necessarily underlying row numbers, // if Indexes are in place) whose row in the table has given string value in given column name. // The results can be used as row indexes to Indexed tensor column data. // If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use the named const args [Contains], [Equals], [IgnoreCase], [UseCase] for greater clarity. +// Use the named const args [tensor.Contains], [tensor.Equals], [tensor.IgnoreCase], +// [tensor.UseCase] for greater clarity. func (dt *Table) RowsByString(colname string, str string, contains, ignoreCase bool) []int { col := dt.Column(colname) if col == nil { diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index 0f25a70c6f..f6e318836d 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -74,12 +74,31 @@ func TestAppendRowsEtc(t *testing.T) { assert.Equal(t, sf, df) } } - ixs := dt.RowsByString("Str", "1", Equals, UseCase) + ixs := dt.RowsByString("Str", "1", tensor.Equals, tensor.UseCase) assert.Equal(t, []int{1, 4, 7, 10}, ixs) + dt.Sequential() dt.IndexesNeeded() + ic := dt.Column("Int") + ic.Sort(tensor.Descending) + dt.IndexesFromTensor(ic) + assert.Equal(t, []int{2, 5, 8, 11, 1, 4, 7, 10, 0, 3, 6, 9}, dt.Indexes) + + dt.Sequential() dt.SortColumns(tensor.Descending, true, "Int", "Flt64") assert.Equal(t, []int{2, 5, 8, 11, 1, 4, 7, 10, 0, 3, 6, 9}, dt.Indexes) + + dt.Sequential() + ic = dt.Column("Int") // note: important to re-get with current indexes + ic.FilterString("1", tensor.Include, tensor.Contains, tensor.IgnoreCase) + dt.IndexesFromTensor(ic) + assert.Equal(t, []int{1, 4, 7, 10}, dt.Indexes) + + dt.Sequential() + dt.Filter(func(dt *Table, row int) bool { + return dt.Column("Flt64").FloatRowCell(row, 0) > 1 + }) + assert.Equal(t, []int{2, 5, 8, 11}, dt.Indexes) } func TestSetNumRows(t *testing.T) { diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index a1ce2272c3..4450c88093 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -172,4 +172,12 @@ func TestSortFilter(t *testing.T) { assert.Equal(t, []int{0, 1, 2, 3, 4}, tsr.Indexes) tsr.Sort(Descending) assert.Equal(t, []int{4, 3, 2, 1, 0}, tsr.Indexes) + + tsr.Sequential() + tsr.FilterString("1", Include, Equals, UseCase) + assert.Equal(t, []int{1}, tsr.Indexes) + + tsr.Sequential() + tsr.FilterString("1", Exclude, Equals, UseCase) + assert.Equal(t, []int{0, 2, 3, 4}, tsr.Indexes) } From 6674f00973df4ae48cee84ccf51a7298d5c2ab5e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 03:56:32 -0700 Subject: [PATCH 036/311] mostly updated datafs and tensorcore, plotting --- core/list.go | 2 +- plot/plotcore/barplot.go | 41 ++- plot/plotcore/options.go | 139 +++----- plot/plotcore/ploteditor.go | 83 ++--- plot/plotcore/tablexy.go | 70 ++-- plot/plotcore/typegen.go | 4 +- plot/plotcore/xyplot.go | 39 ++- tensor/README.md | 3 + tensor/databrowser/datatab.go | 2 +- tensor/datafs/README.md | 10 +- tensor/datafs/data.go | 298 +++--------------- tensor/datafs/dir.go | 180 ++++++----- tensor/datafs/fs.go | 36 +-- tensor/datafs/metadata.go | 70 ++-- tensor/datafs/stats.go | 9 +- tensor/examples/datafs-sim/sim.go | 2 +- tensor/stats/glm/glm.go | 6 +- tensor/stats/split/group.go | 12 +- tensor/stats/split/random.go | 2 +- tensor/stats/split/splits.go | 4 +- tensor/table/README.md | 2 +- tensor/table/table.go | 17 +- tensor/table/typegen.go | 2 +- tensor/tensorcore/simatgrid.go | 4 + tensor/tensorcore/table.go | 104 +++--- tensor/tensorcore/tensoreditor.go | 5 +- tensor/tensorcore/tensorgrid.go | 91 ++---- tensor/tensorcore/typegen.go | 17 - tensor/tensorcore/values.go | 8 +- .../cogentcore_org-core-tensor-table.go | 9 +- 30 files changed, 512 insertions(+), 759 deletions(-) diff --git a/core/list.go b/core/list.go index 8073389e09..832f5ea6bd 100644 --- a/core/list.go +++ b/core/list.go @@ -90,7 +90,7 @@ type Lister interface { // SliceIndex returns the logical slice index: si = i + StartIndex, // the actual value index vi into the slice value (typically = si), // which can be different if there is an index indirection as in - // tensorcore table.Indexed), and a bool that is true if the + // tensorcore table.Table), and a bool that is true if the // index is beyond the available data and is thus invisible, // given the row index provided. SliceIndex(i int) (si, vi int, invis bool) diff --git a/plot/plotcore/barplot.go b/plot/plotcore/barplot.go index dcdfcb8981..786eb10715 100644 --- a/plot/plotcore/barplot.go +++ b/plot/plotcore/barplot.go @@ -8,14 +8,11 @@ import ( "fmt" "log" - "cogentcore.org/core/base/errors" "cogentcore.org/core/colors" "cogentcore.org/core/math32" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" "cogentcore.org/core/plot/plots" - "cogentcore.org/core/tensor/stats/split" - "cogentcore.org/core/tensor/table" ) // bar plot is on integer positions, with different Y values and / or @@ -35,16 +32,16 @@ func (pl *PlotEditor) genPlotBar() { } xp := pl.Columns[xi] - var lsplit *table.Splits + // var lsplit *table.Splits nleg := 1 if pl.Options.Legend != "" { - _, err = pl.table.Table.ColumnIndex(pl.Options.Legend) - if err != nil { - log.Println("plot.Legend: " + err.Error()) + lcol := pl.table.Columns.IndexByKey(pl.Options.Legend) + if lcol < 0 { + log.Println("plot.Legend not found: " + pl.Options.Legend) } else { - xview.SortColumnNames([]string{pl.Options.Legend, xp.Column}, table.Ascending) // make it fit! - lsplit = split.GroupBy(xview, pl.Options.Legend) - nleg = max(lsplit.Len(), 1) + // xview.SortColumnNames([]string{pl.Options.Legend, xp.Column}, tensor.Ascending) // make it fit! + // lsplit = split.GroupBy(xview, pl.Options.Legend) + // nleg = max(lsplit.Len(), 1) } } @@ -60,7 +57,7 @@ func (pl *PlotEditor) genPlotBar() { continue } if cp.TensorIndex < 0 { - yc := errors.Log1(pl.table.Table.ColumnByName(cp.Column)) + yc := pl.table.Column(cp.Column) _, sz := yc.RowCellSize() nys += sz } else { @@ -91,14 +88,14 @@ func (pl *PlotEditor) genPlotBar() { for li := 0; li < nleg; li++ { lview := xview leg := "" - if lsplit != nil && len(lsplit.Values) > li { - leg = lsplit.Values[li][0] - lview = lsplit.Splits[li] - } + // if lsplit != nil && len(lsplit.Values) > li { + // leg = lsplit.Values[li][0] + // lview = lsplit.Splits[li] + // } nidx := 1 stidx := cp.TensorIndex if cp.TensorIndex < 0 { // do all - yc := errors.Log1(pl.table.Table.ColumnByName(cp.Column)) + yc := pl.table.Column(cp.Column) _, sz := yc.RowCellSize() nidx = sz stidx = 0 @@ -109,7 +106,7 @@ func (pl *PlotEditor) genPlotBar() { if xy == nil { continue } - maxx = max(maxx, lview.Len()) + maxx = max(maxx, lview.NumRows()) if firstXY == nil { firstXY = xy } @@ -128,7 +125,7 @@ func (pl *PlotEditor) genPlotBar() { } ec := -1 if cp.ErrColumn != "" { - ec, _ = pl.table.Table.ColumnIndex(cp.ErrColumn) + ec = pl.table.Columns.IndexByKey(cp.ErrColumn) } var bar *plots.BarChart if ec >= 0 { @@ -163,10 +160,10 @@ func (pl *PlotEditor) genPlotBar() { } if firstXY != nil && len(strCols) > 0 { firstXY.table = xview - n := xview.Len() + n := xview.NumRows() for _, cp := range strCols { xy, _ := newTableXY(xview, xi, xp.TensorIndex, firstXY.yColumn, cp.TensorIndex, firstXY.yRange) - xy.labelColumn, _ = xview.Table.ColumnIndex(cp.Column) + xy.labelColumn = xview.Columns.IndexByKey(cp.Column) xy.yIndex = firstXY.yIndex xyl := plots.XYLabels{} @@ -186,8 +183,8 @@ func (pl *PlotEditor) genPlotBar() { } } - netn := pl.table.Len() * stride - xc := pl.table.Table.Columns[xi] + netn := pl.table.NumRows() * stride + xc := pl.table.ColumnIndex(xi) vals := make([]string, netn) for i, dx := range pl.table.Indexes { pi := mid + i*stride diff --git a/plot/plotcore/options.go b/plot/plotcore/options.go index 5b91e3d865..c14e8a68eb 100644 --- a/plot/plotcore/options.go +++ b/plot/plotcore/options.go @@ -6,10 +6,9 @@ package plotcore import ( "image" - "strings" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/option" - "cogentcore.org/core/base/reflectx" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" "cogentcore.org/core/plot/plots" @@ -95,77 +94,54 @@ func (po *PlotOptions) defaults() { // fromMeta sets plot options from meta data. func (po *PlotOptions) fromMeta(dt *table.Table) { - po.FromMetaMap(dt.MetaData) -} - -// metaMapLower tries meta data access by lower-case version of key too -func metaMapLower(meta map[string]string, key string) (string, bool) { - vl, has := meta[key] - if has { - return vl, has - } - vl, has = meta[strings.ToLower(key)] - return vl, has + po.FromMetaMap(dt.Meta) } // FromMetaMap sets plot options from meta data map. -func (po *PlotOptions) FromMetaMap(meta map[string]string) { - if typ, has := metaMapLower(meta, "Type"); has { +func (po *PlotOptions) FromMetaMap(meta metadata.Data) { + if typ, err := metadata.Get[string](meta, "Type"); err == nil { po.Type.SetString(typ) } - if op, has := metaMapLower(meta, "Lines"); has { - if op == "+" || op == "true" { - po.Lines = true - } else { - po.Lines = false - } + if op, err := metadata.Get[bool](meta, "Lines"); err == nil { + po.Lines = op } - if op, has := metaMapLower(meta, "Points"); has { - if op == "+" || op == "true" { - po.Points = true - } else { - po.Points = false - } + if op, err := metadata.Get[bool](meta, "Points"); err == nil { + po.Points = op } - if lw, has := metaMapLower(meta, "LineWidth"); has { - po.LineWidth, _ = reflectx.ToFloat32(lw) + if lw, err := metadata.Get[float64](meta, "LineWidth"); err == nil { + po.LineWidth = float32(lw) } - if ps, has := metaMapLower(meta, "PointSize"); has { - po.PointSize, _ = reflectx.ToFloat32(ps) + if ps, err := metadata.Get[float64](meta, "PointSize"); err == nil { + po.PointSize = float32(ps) } - if bw, has := metaMapLower(meta, "BarWidth"); has { - po.BarWidth, _ = reflectx.ToFloat32(bw) + if bw, err := metadata.Get[float64](meta, "BarWidth"); err == nil { + po.BarWidth = float32(bw) } - if op, has := metaMapLower(meta, "NegativeXDraw"); has { - if op == "+" || op == "true" { - po.NegativeXDraw = true - } else { - po.NegativeXDraw = false - } + if op, err := metadata.Get[bool](meta, "NegativeXDraw"); err == nil { + po.NegativeXDraw = op } - if scl, has := metaMapLower(meta, "Scale"); has { - po.Scale, _ = reflectx.ToFloat32(scl) + if scl, err := metadata.Get[float64](meta, "Scale"); err == nil { + po.Scale = float32(scl) } - if xc, has := metaMapLower(meta, "XAxis"); has { + if xc, err := metadata.Get[string](meta, "XAxis"); err == nil { po.XAxis = xc } - if lc, has := metaMapLower(meta, "Legend"); has { + if lc, err := metadata.Get[string](meta, "Legend"); err == nil { po.Legend = lc } - if xrot, has := metaMapLower(meta, "XAxisRotation"); has { - po.XAxisRotation, _ = reflectx.ToFloat32(xrot) + if xrot, err := metadata.Get[float64](meta, "XAxisRotation"); err == nil { + po.XAxisRotation = float32(xrot) } - if lb, has := metaMapLower(meta, "XAxisLabel"); has { + if lb, err := metadata.Get[string](meta, "XAxisLabel"); err == nil { po.XAxisLabel = lb } - if lb, has := metaMapLower(meta, "YAxisLabel"); has { + if lb, err := metadata.Get[string](meta, "YAxisLabel"); err == nil { po.YAxisLabel = lb } } // ColumnOptions are options for plotting one column of data. type ColumnOptions struct { //types:add - // whether to plot this column On bool @@ -228,64 +204,39 @@ func (co *ColumnOptions) getLabel() string { } // fromMetaMap sets column options from meta data map. -func (co *ColumnOptions) fromMetaMap(meta map[string]string) { - if op, has := metaMapLower(meta, co.Column+":On"); has { - if op == "+" || op == "true" || op == "" { - co.On = true - } else { - co.On = false - } +func (co *ColumnOptions) fromMetaMap(meta metadata.Data) { + if op, err := metadata.Get[bool](meta, co.Column+":On"); err == nil { + co.On = op } - if op, has := metaMapLower(meta, co.Column+":Off"); has { - if op == "+" || op == "true" || op == "" { - co.On = false - } else { - co.On = true - } + if op, err := metadata.Get[bool](meta, co.Column+":Off"); err == nil { + co.On = op } - if op, has := metaMapLower(meta, co.Column+":FixMin"); has { - if op == "+" || op == "true" { - co.Range.FixMin = true - } else { - co.Range.FixMin = false - } + if op, err := metadata.Get[bool](meta, co.Column+":FixMin"); err == nil { + co.Range.FixMin = op } - if op, has := metaMapLower(meta, co.Column+":FixMax"); has { - if op == "+" || op == "true" { - co.Range.FixMax = true - } else { - co.Range.FixMax = false - } + if op, err := metadata.Get[bool](meta, co.Column+":FixMax"); err == nil { + co.Range.FixMax = op } - if op, has := metaMapLower(meta, co.Column+":FloatMin"); has { - if op == "+" || op == "true" { - co.Range.FixMin = false - } else { - co.Range.FixMin = true - } + if op, err := metadata.Get[bool](meta, co.Column+":FloatMin"); err == nil { + co.Range.FixMin = op } - if op, has := metaMapLower(meta, co.Column+":FloatMax"); has { - if op == "+" || op == "true" { - co.Range.FixMax = false - } else { - co.Range.FixMax = true - } + if op, err := metadata.Get[bool](meta, co.Column+":FloatMax"); err == nil { + co.Range.FixMax = op } - if vl, has := metaMapLower(meta, co.Column+":Max"); has { - co.Range.Max, _ = reflectx.ToFloat32(vl) + if vl, err := metadata.Get[float64](meta, co.Column+":Max"); err == nil { + co.Range.Max = float32(vl) } - if vl, has := metaMapLower(meta, co.Column+":Min"); has { - co.Range.Min, _ = reflectx.ToFloat32(vl) + if vl, err := metadata.Get[float64](meta, co.Column+":Min"); err == nil { + co.Range.Min = float32(vl) } - if lb, has := metaMapLower(meta, co.Column+":Label"); has { + if lb, err := metadata.Get[string](meta, co.Column+":Label"); err == nil { co.Label = lb } - if lb, has := metaMapLower(meta, co.Column+":ErrColumn"); has { + if lb, err := metadata.Get[string](meta, co.Column+":ErrColumn"); err == nil { co.ErrColumn = lb } - if vl, has := metaMapLower(meta, co.Column+":TensorIndex"); has { - iv, _ := reflectx.ToInt(vl) - co.TensorIndex = int(iv) + if vl, err := metadata.Get[int](meta, co.Column+":TensorIndex"); err == nil { + co.TensorIndex = vl } } diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 70f09055c5..e7e37dc036 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -17,6 +17,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/base/iox/imagex" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/colors" "cogentcore.org/core/core" "cogentcore.org/core/events" @@ -26,20 +27,21 @@ import ( "cogentcore.org/core/styles" "cogentcore.org/core/styles/states" "cogentcore.org/core/system" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" "cogentcore.org/core/tensor/tensorcore" "cogentcore.org/core/tree" ) // PlotEditor is a widget that provides an interactive 2D plot -// of selected columns of tabular data, represented by a [table.Indexed] into +// of selected columns of tabular data, represented by a [table.Table] into // a [table.Table]. Other types of tabular data can be converted into this format. // The user can change various options for the plot and also modify the underlying data. type PlotEditor struct { //types:add core.Frame // table is the table of data being plotted. - table *table.Indexed + table *table.Table // Options are the overall plot options. Options PlotOptions @@ -104,8 +106,8 @@ func (pl *PlotEditor) Init() { }) pl.Updater(func() { - if pl.table != nil && pl.table.Table != nil { - pl.Options.fromMeta(pl.table.Table) + if pl.table != nil { + pl.Options.fromMeta(pl.table) } }) tree.AddChildAt(pl, "columns", func(w *core.Frame) { @@ -135,7 +137,7 @@ func (pl *PlotEditor) Init() { // to update the Column list, which will also trigger a Layout // and updating of the plot on next render pass. // This is safe to call from a different goroutine. -func (pl *PlotEditor) setIndexed(tab *table.Indexed) *PlotEditor { +func (pl *PlotEditor) setIndexed(tab *table.Table) *PlotEditor { pl.table = tab pl.Update() return pl @@ -146,7 +148,7 @@ func (pl *PlotEditor) setIndexed(tab *table.Indexed) *PlotEditor { // and updating of the plot on next render pass. // This is safe to call from a different goroutine. func (pl *PlotEditor) SetTable(tab *table.Table) *PlotEditor { - pl.table = table.NewIndexed(tab) + pl.table = table.NewView(tab) pl.Update() return pl } @@ -213,7 +215,7 @@ func (pl *PlotEditor) SavePNG(fname core.Filename) { //types:add } // SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim) -func (pl *PlotEditor) SaveCSV(fname core.Filename, delim table.Delims) { //types:add +func (pl *PlotEditor) SaveCSV(fname core.Filename, delim tensor.Delims) { //types:add pl.table.SaveCSV(fname, delim, table.Headers) pl.dataFile = fname } @@ -223,22 +225,22 @@ func (pl *PlotEditor) SaveCSV(fname core.Filename, delim table.Delims) { //types func (pl *PlotEditor) SaveAll(fname core.Filename) { //types:add fn := string(fname) fn = strings.TrimSuffix(fn, filepath.Ext(fn)) - pl.SaveCSV(core.Filename(fn+".tsv"), table.Tab) + pl.SaveCSV(core.Filename(fn+".tsv"), tensor.Tab) pl.SavePNG(core.Filename(fn + ".png")) pl.SaveSVG(core.Filename(fn + ".svg")) } // OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim) -func (pl *PlotEditor) OpenCSV(filename core.Filename, delim table.Delims) { //types:add - pl.table.Table.OpenCSV(filename, delim) +func (pl *PlotEditor) OpenCSV(filename core.Filename, delim tensor.Delims) { //types:add + pl.table.OpenCSV(filename, delim) pl.dataFile = filename pl.UpdatePlot() } // OpenFS opens the Table data from a csv (comma-separated values) file (or any delim) // from the given filesystem. -func (pl *PlotEditor) OpenFS(fsys fs.FS, filename core.Filename, delim table.Delims) { - pl.table.Table.OpenFS(fsys, string(filename), delim) +func (pl *PlotEditor) OpenFS(fsys fs.FS, filename core.Filename, delim tensor.Delims) { + pl.table.OpenFS(fsys, string(filename), delim) pl.dataFile = filename pl.UpdatePlot() } @@ -273,7 +275,7 @@ func (pl *PlotEditor) xLabel() string { // GoUpdatePlot updates the display based on current Indexed into table. // This version can be called from goroutines. It does Sequential() on -// the [table.Indexed], under the assumption that it is used for tracking a +// the [table.Table], under the assumption that it is used for tracking a // the latest updates of a running process. func (pl *PlotEditor) GoUpdatePlot() { if pl == nil || pl.This == nil { @@ -282,7 +284,7 @@ func (pl *PlotEditor) GoUpdatePlot() { if core.TheApp.Platform() == system.Web { time.Sleep(time.Millisecond) // critical to prevent hanging! } - if !pl.IsVisible() || pl.table == nil || pl.table.Table == nil || pl.inPlot { + if !pl.IsVisible() || pl.table == nil || pl.table == nil || pl.inPlot { return } pl.Scene.AsyncLock() @@ -293,19 +295,19 @@ func (pl *PlotEditor) GoUpdatePlot() { } // UpdatePlot updates the display based on current Indexed into table. -// It does not automatically update the [table.Indexed] unless it is +// It does not automatically update the [table.Table] unless it is // nil or out date. func (pl *PlotEditor) UpdatePlot() { if pl == nil || pl.This == nil { return } - if pl.table == nil || pl.table.Table == nil || pl.inPlot { + if pl.table == nil || pl.inPlot { return } - if len(pl.Children) != 2 || len(pl.Columns) != pl.table.Table.NumColumns() { + if len(pl.Children) != 2 || len(pl.Columns) != pl.table.NumColumns() { pl.Update() } - if pl.table.Len() == 0 { + if pl.table.NumRows() == 0 { pl.table.Sequential() } pl.genPlot() @@ -326,8 +328,8 @@ func (pl *PlotEditor) genPlot() { if len(pl.table.Indexes) == 0 { pl.table.Sequential() } else { - lsti := pl.table.Indexes[pl.table.Len()-1] - if lsti >= pl.table.Table.Rows { // out of date + lsti := pl.table.Indexes[pl.table.NumRows()-1] + if lsti >= pl.table.NumRows() { // out of date pl.table.Sequential() } } @@ -341,8 +343,8 @@ func (pl *PlotEditor) genPlot() { pl.plotWidget.Scale = pl.Options.Scale pl.plotWidget.SetRangesFunc = func() { plt := pl.plotWidget.Plot - xi, err := pl.table.Table.ColumnIndex(pl.Options.XAxis) - if err == nil { + xi := pl.table.Columns.IndexByKey(pl.Options.XAxis) + if xi >= 0 { xp := pl.Columns[xi] if xp.Range.FixMin { plt.X.Min = math32.Min(plt.X.Min, float32(xp.Range.Min)) @@ -377,14 +379,14 @@ func (pl *PlotEditor) configPlot(plt *plot.Plot) { } // plotXAxis processes the XAxis and returns its index -func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.Indexed) (xi int, xview *table.Indexed, err error) { - xi, err = ixvw.Table.ColumnIndex(pl.Options.XAxis) - if err != nil { +func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.Table) (xi int, xview *table.Table, err error) { + xi = ixvw.Columns.IndexByKey(pl.Options.XAxis) + if xi < 0 { // log.Println("plot.PlotXAxis: " + err.Error()) return } xview = ixvw - xc := ixvw.Table.Columns[xi] + xc := ixvw.ColumnIndex(xi) xp := pl.Columns[xi] sz := 1 if xp.Range.FixMin { @@ -393,8 +395,8 @@ func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.Indexed) (xi int, xv if xp.Range.FixMax { plt.X.Max = math32.Max(plt.X.Max, float32(xp.Range.Max)) } - if xc.NumDims() > 1 { - sz = xc.Len() / xc.DimSize(0) + if xc.Tensor.NumDims() > 1 { + sz = xc.NumRows() / xc.Tensor.DimSize(0) if xp.TensorIndex > sz || xp.TensorIndex < 0 { slog.Error("plotcore.PlotEditor.plotXAxis: TensorIndex invalid -- reset to 0") xp.TensorIndex = 0 @@ -407,11 +409,11 @@ const plotColumnsHeaderN = 2 // columnsListUpdate updates the list of columns func (pl *PlotEditor) columnsListUpdate() { - if pl.table == nil || pl.table.Table == nil { + if pl.table == nil { pl.Columns = nil return } - dt := pl.table.Table + dt := pl.table nc := dt.NumColumns() if nc == len(pl.Columns) { return @@ -426,8 +428,9 @@ func (pl *PlotEditor) columnsListUpdate() { } cp := &ColumnOptions{Column: cn} cp.defaults() - tcol := dt.Columns[ci] - if tcol.IsString() { + tcol := dt.ColumnIndex(ci) + tc := tcol.Tensor + if tc.IsString() { cp.IsString = true } else { cp.IsString = false @@ -437,9 +440,9 @@ func (pl *PlotEditor) columnsListUpdate() { hasOn = true } } - cp.fromMetaMap(pl.table.Table.MetaData) + cp.fromMetaMap(pl.table.Meta) inc := 1 - if cn == pl.Options.XAxis || tcol.IsString() || tcol.DataType() == reflect.Int || tcol.DataType() == reflect.Int64 || tcol.DataType() == reflect.Int32 || tcol.DataType() == reflect.Uint8 { + if cn == pl.Options.XAxis || tc.IsString() || tc.DataType() == reflect.Int || tc.DataType() == reflect.Int64 || tc.DataType() == reflect.Int32 || tc.DataType() == reflect.Uint8 { inc = 0 } cp.Color = colors.Uniform(colors.Spaced(clri)) @@ -449,7 +452,7 @@ func (pl *PlotEditor) columnsListUpdate() { } // ColumnsFromMetaMap updates all the column settings from given meta map -func (pl *PlotEditor) ColumnsFromMetaMap(meta map[string]string) { +func (pl *PlotEditor) ColumnsFromMetaMap(meta metadata.Data) { for _, cp := range pl.Columns { cp.fromMetaMap(meta) } @@ -630,7 +633,7 @@ func (pl *PlotEditor) MakeToolbar(p *tree.Plan) { SetTooltip("open a Table window of the data"). OnClick(func(e events.Event) { d := core.NewBody(pl.Name + " Data") - tv := tensorcore.NewTable(d).SetTable(pl.table.Table) + tv := tensorcore.NewTable(d).SetTable(pl.table) d.AddTopBar(func(bar *core.Frame) { core.NewToolbar(bar).Maker(tv.MakeToolbar) }) @@ -652,10 +655,10 @@ func (pl *PlotEditor) MakeToolbar(p *tree.Plan) { w.SetFunc(pl.OpenCSV).SetIcon(icons.Open) }) tree.Add(p, func(w *core.Separator) {}) - tree.Add(p, func(w *core.FuncButton) { - w.SetFunc(pl.table.FilterColumnName).SetText("Filter").SetIcon(icons.FilterAlt) - w.SetAfterFunc(pl.UpdatePlot) - }) + // tree.Add(p, func(w *core.FuncButton) { // TODO + // w.SetFunc(pl.table.FilterColumnName).SetText("Filter").SetIcon(icons.FilterAlt) + // w.SetAfterFunc(pl.UpdatePlot) + // }) tree.Add(p, func(w *core.FuncButton) { w.SetFunc(pl.table.Sequential).SetText("Unfilter").SetIcon(icons.FilterAltOff) w.SetAfterFunc(pl.UpdatePlot) diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index 6bcf32406d..071fe23b53 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -21,7 +21,7 @@ import ( type tableXY struct { // the index view of data table to plot from - table *table.Indexed + table *table.Table // the indexes of the tensor columns to use for the X and Y data, respectively xColumn, yColumn int @@ -50,7 +50,7 @@ var _ plots.YErrorer = &tableXY{} // newTableXY returns a new XY plot view onto the given Indexed of table.Table (makes a copy), // from given column indexes, and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. -func newTableXY(dt *table.Indexed, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { +func newTableXY(dt *table.Table, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { txy := &tableXY{table: dt.Clone(), xColumn: xcol, yColumn: ycol, xIndex: xtsrIndex, yIndex: ytsrIndex, yRange: yrng} return txy, txy.validate() } @@ -58,10 +58,10 @@ func newTableXY(dt *table.Indexed, xcol, xtsrIndex, ycol, ytsrIndex int, yrng mi // newTableXYName returns a new XY plot view onto the given Indexed of table.Table (makes a copy), // from given column name and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. -func newTableXYName(dt *table.Indexed, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { - yi, err := dt.Table.ColumnIndex(ycol) - if errors.Log(err) != nil { - return nil, err +func newTableXYName(dt *table.Table, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { + yi := dt.Columns.IndexByKey(ycol) + if yi < 0 { + return nil, nil // todo: err } txy := &tableXY{table: dt.Clone(), xColumn: xi, yColumn: yi, xIndex: xtsrIndex, yIndex: ytsrIndex, yRange: yrng} return txy, txy.validate() @@ -73,7 +73,7 @@ func (txy *tableXY) validate() error { if txy.table == nil { return errors.New("eplot.TableXY table is nil") } - nc := txy.table.Table.NumColumns() + nc := txy.table.NumColumns() if txy.xColumn >= nc || txy.xColumn < 0 { txy.xColumn = 0 return errors.New("eplot.TableXY XColumn index invalid -- reset to 0") @@ -82,13 +82,13 @@ func (txy *tableXY) validate() error { txy.yColumn = 0 return errors.New("eplot.TableXY YColumn index invalid -- reset to 0") } - xc := txy.table.Table.Columns[txy.xColumn] - yc := txy.table.Table.Columns[txy.yColumn] - if xc.NumDims() > 1 { + xc := txy.table.ColumnIndex(txy.xColumn) + yc := txy.table.ColumnIndex(txy.yColumn) + if xc.Tensor.NumDims() > 1 { _, txy.xRowSize = xc.RowCellSize() // note: index already validated } - if yc.NumDims() > 1 { + if yc.Tensor.NumDims() > 1 { _, txy.yRowSize = yc.RowCellSize() if txy.yIndex >= txy.yRowSize || txy.yIndex < 0 { txy.yIndex = 0 @@ -119,20 +119,20 @@ func (txy *tableXY) filterValues() { // Len returns the number of rows in the view of table func (txy *tableXY) Len() int { - if txy.table == nil || txy.table.Table == nil { + if txy.table == nil { return 0 } - return txy.table.Len() + return txy.table.NumRows() } // tRowValue returns the y value at given true table row in table func (txy *tableXY) tRowValue(row int) float32 { - yc := txy.table.Table.Columns[txy.yColumn] + yc := txy.table.ColumnIndex(txy.yColumn) y := float32(0.0) switch { - case yc.IsString(): + case yc.Tensor.IsString(): y = float32(row) - case yc.NumDims() > 1: + case yc.Tensor.NumDims() > 1: _, sz := yc.RowCellSize() if txy.yIndex < sz && txy.yIndex >= 0 { y = float32(yc.FloatRowCell(row, txy.yIndex)) @@ -145,16 +145,16 @@ func (txy *tableXY) tRowValue(row int) float32 { // Value returns the y value at given row in table func (txy *tableXY) Value(row int) float32 { - if txy.table == nil || txy.table.Table == nil || row >= txy.table.Len() { + if txy.table == nil || row >= txy.table.NumRows() { return 0 } trow := txy.table.Indexes[row] // true table row - yc := txy.table.Table.Columns[txy.yColumn] + yc := txy.table.ColumnIndex(txy.yColumn) y := float32(0.0) switch { - case yc.IsString(): + case yc.Tensor.IsString(): y = float32(row) - case yc.NumDims() > 1: + case yc.Tensor.NumDims() > 1: _, sz := yc.RowCellSize() if txy.yIndex < sz && txy.yIndex >= 0 { y = float32(yc.FloatRowCell(trow, txy.yIndex)) @@ -167,15 +167,15 @@ func (txy *tableXY) Value(row int) float32 { // tRowXValue returns an x value at given actual row in table func (txy *tableXY) tRowXValue(row int) float32 { - if txy.table == nil || txy.table.Table == nil { + if txy.table == nil || txy.table == nil { return 0 } - xc := txy.table.Table.Columns[txy.xColumn] + xc := txy.table.ColumnIndex(txy.xColumn) x := float32(0.0) switch { - case xc.IsString(): + case xc.Tensor.IsString(): x = float32(row) - case xc.NumDims() > 1: + case xc.Tensor.NumDims() > 1: _, sz := xc.RowCellSize() if txy.xIndex < sz && txy.xIndex >= 0 { x = float32(xc.FloatRowCell(row, txy.xIndex)) @@ -188,16 +188,16 @@ func (txy *tableXY) tRowXValue(row int) float32 { // xValue returns an x value at given row in table func (txy *tableXY) xValue(row int) float32 { - if txy.table == nil || txy.table.Table == nil || row >= txy.table.Len() { + if txy.table == nil || row >= txy.table.NumRows() { return 0 } trow := txy.table.Indexes[row] // true table row - xc := txy.table.Table.Columns[txy.xColumn] + xc := txy.table.ColumnIndex(txy.xColumn) x := float32(0.0) switch { - case xc.IsString(): + case xc.Tensor.IsString(): x = float32(row) - case xc.NumDims() > 1: + case xc.Tensor.NumDims() > 1: _, sz := xc.RowCellSize() if txy.xIndex < sz && txy.xIndex >= 0 { x = float32(xc.FloatRowCell(trow, txy.xIndex)) @@ -210,7 +210,7 @@ func (txy *tableXY) xValue(row int) float32 { // XY returns an x, y pair at given row in table func (txy *tableXY) XY(row int) (x, y float32) { - if txy.table == nil || txy.table.Table == nil { + if txy.table == nil || txy.table == nil { return 0, 0 } x = txy.xValue(row) @@ -220,25 +220,25 @@ func (txy *tableXY) XY(row int) (x, y float32) { // Label returns a label for given row in table, implementing [plot.Labeler] interface func (txy *tableXY) Label(row int) string { - if txy.table == nil || txy.table.Table == nil || row >= txy.table.Len() { + if txy.table == nil || row >= txy.table.NumRows() { return "" } trow := txy.table.Indexes[row] // true table row - return txy.table.Table.Columns[txy.labelColumn].String1D(trow) + return txy.table.ColumnIndex(txy.labelColumn).String1D(trow) } // YError returns error bars, implementing [plots.YErrorer] interface. func (txy *tableXY) YError(row int) (float32, float32) { - if txy.table == nil || txy.table.Table == nil || row >= txy.table.Len() { + if txy.table == nil || row >= txy.table.NumRows() { return 0, 0 } trow := txy.table.Indexes[row] // true table row - ec := txy.table.Table.Columns[txy.errColumn] + ec := txy.table.ColumnIndex(txy.errColumn) eval := float32(0.0) switch { - case ec.IsString(): + case ec.Tensor.IsString(): eval = float32(row) - case ec.NumDims() > 1: + case ec.Tensor.NumDims() > 1: _, sz := ec.RowCellSize() if txy.yIndex < sz && txy.yIndex >= 0 { eval = float32(ec.FloatRowCell(trow, txy.yIndex)) diff --git a/plot/plotcore/typegen.go b/plot/plotcore/typegen.go index 44d974329f..2dbff3ebbd 100644 --- a/plot/plotcore/typegen.go +++ b/plot/plotcore/typegen.go @@ -26,11 +26,11 @@ func NewPlot(parent ...tree.Node) *Plot { return tree.New[Plot](parent...) } // documents or other cases where the overall plot size will be small. func (t *Plot) SetScale(v float32) *Plot { t.Scale = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Indexed] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "Options", Doc: "Options are the overall plot options."}, {Name: "Columns", Doc: "Columns are the options for each column of the table."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Table] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "Options", Doc: "Options are the overall plot options."}, {Name: "Columns", Doc: "Columns are the options for each column of the table."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) // NewPlotEditor returns a new [PlotEditor] with the given optional parent: // PlotEditor is a widget that provides an interactive 2D plot -// of selected columns of tabular data, represented by a [table.Indexed] into +// of selected columns of tabular data, represented by a [table.Table] into // a [table.Table]. Other types of tabular data can be converted into this format. // The user can change various options for the plot and also modify the underlying data. func NewPlotEditor(parent ...tree.Node) *PlotEditor { return tree.New[PlotEditor](parent...) } diff --git a/plot/plotcore/xyplot.go b/plot/plotcore/xyplot.go index eb405ee337..622c7794e1 100644 --- a/plot/plotcore/xyplot.go +++ b/plot/plotcore/xyplot.go @@ -8,13 +8,10 @@ import ( "fmt" "log/slog" - "cogentcore.org/core/base/errors" "cogentcore.org/core/colors" "cogentcore.org/core/plot" "cogentcore.org/core/plot/plots" "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/split" - "cogentcore.org/core/tensor/table" ) // genPlotXY generates an XY (lines, points) plot, setting Plot variable @@ -28,16 +25,16 @@ func (pl *PlotEditor) genPlotXY() { } xp := pl.Columns[xi] - var lsplit *table.Splits + // var lsplit *table.Splits nleg := 1 if pl.Options.Legend != "" { - _, err = pl.table.Table.ColumnIndex(pl.Options.Legend) - if err != nil { + ix := pl.table.Columns.IndexByKey(pl.Options.Legend) + if ix < 0 { slog.Error("plot.Legend", "err", err.Error()) } else { - errors.Log(xview.SortStableColumnNames([]string{pl.Options.Legend, xp.Column}, table.Ascending)) - lsplit = split.GroupBy(xview, pl.Options.Legend) - nleg = max(lsplit.Len(), 1) + // errors.Log(xview.SortStableColumnNames([]string{pl.Options.Legend, xp.Column}, table.Ascending)) + // lsplit = split.GroupBy(xview, pl.Options.Legend) + // nleg = max(lsplit.Len(), 1) } } @@ -53,7 +50,7 @@ func (pl *PlotEditor) genPlotXY() { continue } if cp.TensorIndex < 0 { - yc := errors.Log1(pl.table.Table.ColumnByName(cp.Column)) + yc := pl.table.Column(cp.Column) _, sz := yc.RowCellSize() nys += sz } else { @@ -77,14 +74,14 @@ func (pl *PlotEditor) genPlotXY() { for li := 0; li < nleg; li++ { lview := xview leg := "" - if lsplit != nil && len(lsplit.Values) > li { - leg = lsplit.Values[li][0] - lview = lsplit.Splits[li] - } + // if lsplit != nil && len(lsplit.Values) > li { + // leg = lsplit.Values[li][0] + // lview = lsplit.Splits[li] + // } nidx := 1 stidx := cp.TensorIndex if cp.TensorIndex < 0 { // do all - yc := errors.Log1(pl.table.Table.ColumnByName(cp.Column)) + yc := pl.table.Column(cp.Column) _, sz := yc.RowCellSize() nidx = sz stidx = 0 @@ -143,7 +140,7 @@ func (pl *PlotEditor) genPlotXY() { } } if cp.ErrColumn != "" { - ec := errors.Log1(pl.table.Table.ColumnIndex(cp.ErrColumn)) + ec := pl.table.Columns.IndexByKey(cp.ErrColumn) if ec >= 0 { xy.errColumn = ec eb, _ := plots.NewYErrorBars(xy) @@ -158,7 +155,7 @@ func (pl *PlotEditor) genPlotXY() { if firstXY != nil && len(strCols) > 0 { for _, cp := range strCols { xy, _ := newTableXY(xview, xi, xp.TensorIndex, firstXY.yColumn, cp.TensorIndex, firstXY.yRange) - xy.labelColumn, _ = xview.Table.ColumnIndex(cp.Column) + xy.labelColumn = xview.Columns.IndexByKey(cp.Column) xy.yIndex = firstXY.yIndex lbls, _ := plots.NewLabels(xy) if lbls != nil { @@ -168,10 +165,10 @@ func (pl *PlotEditor) genPlotXY() { } // Use string labels for X axis if X is a string - xc := pl.table.Table.Columns[xi] - if xc.IsString() { - xcs := xc.(*tensor.String) - vals := make([]string, pl.table.Len()) + xc := pl.table.ColumnIndex(xi) + if xc.Tensor.IsString() { + xcs := xc.Tensor.(*tensor.String) + vals := make([]string, pl.table.NumRows()) for i, dx := range pl.table.Indexes { vals[i] = xcs.Values[dx] } diff --git a/tensor/README.md b/tensor/README.md index b2688db17a..0a687c90de 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -105,3 +105,6 @@ val := ix.Float(3,2,1) This package was originally developed as [etable](https://github.com/emer/etable) as part of the _emergent_ software framework. It always depended on the GUI framework that became Cogent Core, and having it integrated within the Core monorepo makes it easier to integrate updates, and also makes it easier to build advanced data management and visualization applications. For example, the [plot/plotcore](../plot/plotcore) package uses the `Table` to support flexible and powerful plotting functionality. +It was completely rewritten in Sept 2024 to use a single data type (`tensor.Indexed`) and call signature for compute functions taking these args, to provide a simple and efficient data processing framework that greatly simplified the code and enables the [goal](../goal) language to directly transpile simplified math expressions into corresponding tensor compute code. + + diff --git a/tensor/databrowser/datatab.go b/tensor/databrowser/datatab.go index b709cc7436..53b430b9b8 100644 --- a/tensor/databrowser/datatab.go +++ b/tensor/databrowser/datatab.go @@ -27,7 +27,7 @@ func NewTab[T any](br *Browser, label string, mkfun func(tab *core.Frame) T) T { } // NewTabTensorTable creates a tab with a tensorcore.Table widget -// to view given table.Table, using its own table.Indexed as tv.Table. +// to view given table.Table, using its own table.Table as tv.Table. // Use tv.Table.Table to get the underlying *table.Table // Use tv.Table.Sequential to update the Indexed to view // all of the rows when done updating the Table, and then call br.Update() diff --git a/tensor/datafs/README.md b/tensor/datafs/README.md index c2054b8ab6..815df4429a 100644 --- a/tensor/datafs/README.md +++ b/tensor/datafs/README.md @@ -1,9 +1,11 @@ # datafs: a virtual filesystem for data -TODO: write docs +`datafs` is a virtual file system that implements the Go `fs` interface, and can be accessed using fs-general tools, including the cogent core `filetree` and the `goal` shell. -# Constraints +Data is represented using the [tensor] package universal data type: the `tensor.Indexed` `Tensor`, which can represent everything from a single scalar value up to n-dimensional collections of patterns, in a range of data types. + +A given `Data` node either has a tensor `Value` or is a directory with an ordered map of other nodes under it. Each value has a name which must be unique within the directory. The nodes are processed in the order of this list, which initially reflects the order added, and can be re-ordered as needed. + +The hierarchical structure of a filesystem naturally supports various kinds of functions, such as various time scales of logging, with lower-level data aggregated into upper levels. Or hierarchical splits for a pivot-table effect. -* no pointers -- GPU does not like pointers -- use Set / As accessors -* names within directory must be unique diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 8c83a4d94d..85f3002c21 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -6,20 +6,20 @@ package datafs import ( "errors" - "reflect" "time" - "unsafe" "cogentcore.org/core/base/fileinfo" - "cogentcore.org/core/base/metadata" - "cogentcore.org/core/base/reflectx" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) // Data is a single item of data, the "file" or "directory" in the data filesystem. +// Data is represented using the [tensor] package universal data type: the [tensor.Indexed] +// [tensor.Tensor], which can represent everything from a single scalar value up to +// n-dimensional collections of patterns, in a range of data types. +// Directories have an ordered map of items. type Data struct { - // Parent is the parent data directory + // Parent is the parent data directory. Parent *Data // name is the name of this item. it is not a path. @@ -28,20 +28,25 @@ type Data struct { // modTime tracks time added to directory, used for ordering. modTime time.Time - // Meta has metadata, including standardized support for - // plotting options, compute functions. - Meta metadata.Data + // Value is represented using the universal [tensor] data type of + // [tensor.Indexed], which can represent anything from a scalar + // to n-dimensional data, in a range of data types. + Value *tensor.Indexed - // Value is the underlying value of data; - // is a map[string]*Data for directories. - Value any + // Dir is for directory nodes, with all the items in the directory. + Dir *Dir + + // DirTable is a summary [table.Table] with columns comprised of + // Value items in the directory, which can be used for plotting or other + // operations. + DirTable *table.Table } -// NewData returns a new Data item in given directory Data item, -// which can be nil. If not a directory, an error will be generated. -// The modTime is automatically set to now, and can be used for sorting -// by order created. The name must be unique within parent. -func NewData(dir *Data, name string) (*Data, error) { +// newData returns a new Data item in given directory Data item, +// which can be nil. If not a directory, or the name is not unique, +// an error will be generated. +// The modTime is set to now. The name must be unique within parent. +func newData(dir *Data, name string) (*Data, error) { d := &Data{Parent: dir, name: name, modTime: time.Now()} var err error if dir != nil { @@ -50,264 +55,39 @@ func NewData(dir *Data, name string) (*Data, error) { return d, err } -// New adds new data item(s) of given basic type to given directory, -// with given name(s) (at least one is required). -// Values are initialized to zero value for type. -// All names must be unique in the directory. -// Returns the first item created, for immediate use of one value. -func New[T any](dir *Data, names ...string) (*Data, error) { - if len(names) == 0 { - err := errors.New("datafs.New requires at least 1 name") - return nil, err - } - var r *Data - var errs []error - for _, nm := range names { - var v T - d, err := NewData(dir, nm) - if err != nil { - errs = append(errs, err) - continue - } - d.Value = v - if r == nil { - r = d - } - } - return r, errors.Join(errs...) -} - -// NewTensor returns a new Tensor of given data type, shape sizes, -// and optional dimension names, in given directory Data item. +// NewValue returns a new Data value as an [tensor.Indexed] [tensor.Tensor] +// of given data type and shape sizes, in given directory Data item. // The name must be unique in the directory. -func NewTensor[T string | bool | float32 | float64 | int | int32 | byte](dir *Data, name string, sizes []int, names ...string) (tensor.Tensor, error) { - tsr := tensor.New[T](sizes, names...) - d, err := NewData(dir, name) - d.Value = tsr - return tsr, err -} - -// NewTable makes new table.Table(s) in given directory, -// for given name(s) (at least one is required). -// All names must be unique in the directory. -// Returns the first table created, for immediate use of one item. -func NewTable(dir *Data, names ...string) (*table.Table, error) { - if len(names) == 0 { - err := errors.New("datafs.New requires at least 1 name") - return nil, err +func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Indexed { + tsr := tensor.New[T](sizes...) + d, err := newData(dir, name) + if errors.Log(err) != nil { + return nil } - var r *table.Table - var errs []error - for _, nm := range names { - t := table.NewTable(nm) - d, err := NewData(dir, nm) - if err != nil { - errs = append(errs, err) - continue - } - d.Value = t - if r == nil { - r = t - } - } - return r, errors.Join(errs...) -} - -/////////////////////////////// -// Data Access - -// IsNumeric returns true if the [DataType] is a basic scalar -// numerical value, e.g., float32, int, etc. -func (d *Data) IsNumeric() bool { - return reflectx.KindIsNumber(d.DataType()) + d.Value = tensor.NewIndexed(tsr) + return tsr } -// DataType returns the type of the data elements in the tensor. -// Bool is returned for the Bits tensor type. -func (d *Data) DataType() reflect.Kind { +func (d *Data) KnownFileInfo() fileinfo.Known { if d.Value == nil { - return reflect.Invalid + return fileinfo.Unknown } - return reflect.TypeOf(d.Value).Kind() -} - -func (d *Data) KnownFileInfo() fileinfo.Known { - if tsr := d.AsTensor(); tsr != nil { + tsr := d.Value.Tensor + if tsr.Len() > 1 { return fileinfo.Tensor } - kind := d.DataType() - if reflectx.KindIsNumber(kind) { - return fileinfo.Number - } - if kind == reflect.String { + if tsr.IsString() { return fileinfo.String } - return fileinfo.Unknown -} - -// AsTensor returns the data as a tensor if it is one, else nil. -func (d *Data) AsTensor() tensor.Tensor { - tsr, _ := d.Value.(tensor.Tensor) - return tsr -} - -// AsTable returns the data as a table if it is one, else nil. -func (d *Data) AsTable() *table.Table { - dt, _ := d.Value.(*table.Table) - return dt -} - -// AsFloat64 returns data as a float64 if it is a scalar value -// that can be so converted. Returns false if not. -func (d *Data) AsFloat64() (float64, bool) { - // fast path for actual floats - if f, ok := d.Value.(float64); ok { - return f, true - } - if f, ok := d.Value.(float32); ok { - return float64(f), true - } - if tsr := d.AsTensor(); tsr != nil { - return 0, false - } - if dt := d.AsTable(); dt != nil { - return 0, false - } - v, err := reflectx.ToFloat(d.Value) - if err != nil { - return 0, false - } - return v, true -} - -// SetFloat64 sets data from given float64 if it is a scalar value -// that can be so set. Returns false if not. -func (d *Data) SetFloat64(v float64) bool { - // fast path for actual floats - if _, ok := d.Value.(float64); ok { - d.Value = v - return true - } - if _, ok := d.Value.(float32); ok { - d.Value = float32(v) - return true - } - if tsr := d.AsTensor(); tsr != nil { - return false - } - if dt := d.AsTable(); dt != nil { - return false - } - err := reflectx.SetRobust(&d.Value, v) - if err != nil { - return false - } - return true -} - -// AsFloat32 returns data as a float32 if it is a scalar value -// that can be so converted. Returns false if not. -func (d *Data) AsFloat32() (float32, bool) { - v, ok := d.AsFloat64() - return float32(v), ok -} - -// SetFloat32 sets data from given float32 if it is a scalar value -// that can be so set. Returns false if not. -func (d *Data) SetFloat32(v float32) bool { - return d.SetFloat64(float64(v)) -} - -// AsString returns data as a string if it is a scalar value -// that can be so converted. Returns false if not. -func (d *Data) AsString() (string, bool) { - // fast path for actual strings - if s, ok := d.Value.(string); ok { - return s, true - } - if tsr := d.AsTensor(); tsr != nil { - return "", false - } - if dt := d.AsTable(); dt != nil { - return "", false - } - s := reflectx.ToString(d.Value) - return s, true -} - -// SetString sets data from given string if it is a scalar value -// that can be so set. Returns false if not. -func (d *Data) SetString(v string) bool { - // fast path for actual strings - if _, ok := d.Value.(string); ok { - d.Value = v - return true - } - if tsr := d.AsTensor(); tsr != nil { - return false - } - if dt := d.AsTable(); dt != nil { - return false - } - err := reflectx.SetRobust(&d.Value, v) - if err != nil { - return false - } - return true -} - -// AsInt returns data as a int if it is a scalar value -// that can be so converted. Returns false if not. -func (d *Data) AsInt() (int, bool) { - // fast path for actual ints - if f, ok := d.Value.(int); ok { - return f, true - } - if tsr := d.AsTensor(); tsr != nil { - return 0, false - } - if dt := d.AsTable(); dt != nil { - return 0, false - } - v, err := reflectx.ToInt(d.Value) - if err != nil { - return 0, false - } - return int(v), true -} - -// SetInt sets data from given int if it is a scalar value -// that can be so set. Returns false if not. -func (d *Data) SetInt(v int) bool { - // fast path for actual ints - if _, ok := d.Value.(int); ok { - d.Value = v - return true - } - if tsr := d.AsTensor(); tsr != nil { - return false - } - if dt := d.AsTable(); dt != nil { - return false - } - err := reflectx.SetRobust(&d.Value, v) - if err != nil { - return false - } - return true + return fileinfo.Number } // Bytes returns the byte-wise representation of the data Value. // This is the actual underlying data, so make a copy if it can be // unintentionally modified or retained more than for immediate use. func (d *Data) Bytes() []byte { - if tsr := d.AsTensor(); tsr != nil { - return tsr.Bytes() - } - size := d.Size() - switch x := d.Value.(type) { - // todo: other things here? - default: - return unsafe.Slice((*byte)(unsafe.Pointer(&x)), size) + if d.Value == nil { + return nil } + return d.Value.Tensor.Bytes() } diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 9824f66509..7b0c2903df 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -12,9 +12,15 @@ import ( "slices" "sort" - "golang.org/x/exp/maps" + "cogentcore.org/core/base/keylist" + "cogentcore.org/core/tensor" ) +// Dir is a map of directory entry names to Data nodes. +// It retains the order that items were added in, which is +// the natural order items are processed in. +type Dir = keylist.List[string, *Data] + // NewDir returns a new datafs directory with given name. // if parent != nil and a directory, this dir is added to it. // if name is empty, then it is set to "root", the root directory. @@ -28,18 +34,25 @@ func NewDir(name string, parent ...*Data) (*Data, error) { if len(parent) == 1 { par = parent[0] } - d, err := NewData(par, name) - d.Value = make(map[string]*Data) + d, err := newData(par, name) + d.Dir = &Dir{} return d, err } // Item returns data item in given directory by name. // This is for fast access and direct usage of known -// items, and it will crash if item is not found or +// items, and it will panic if item is not found or // this data is not a directory. func (d *Data) Item(name string) *Data { - fm := d.filemap() - return fm[name] + return d.Dir.ValueByKey(name) +} + +// Value returns the [tensor.Indexed] Value for given item +// within this directory. This will panic if item is not +// found, and will return nil if it is not a Value +// (i.e., it is a directory). +func (d *Data) Value(name string) *tensor.Indexed { + return d.Dir.ValueByKey(name).Value } // Items returns data items in given directory by name. @@ -48,11 +61,10 @@ func (d *Data) Items(names ...string) ([]*Data, error) { if err := d.mustDir("Items", ""); err != nil { return nil, err } - fm := d.filemap() var errs []error var its []*Data for _, nm := range names { - dt := fm[nm] + dt := d.Dir.ValueByKey(nm) if dt != nil { its = append(its, dt) } else { @@ -63,8 +75,28 @@ func (d *Data) Items(names ...string) ([]*Data, error) { return its, errors.Join(errs...) } +// Values returns Value items (tensors) in given directory by name. +// error reports any items not found, or if not a directory. +func (d *Data) Values(names ...string) ([]*tensor.Indexed, error) { + if err := d.mustDir("Values", ""); err != nil { + return nil, err + } + var errs []error + var its []*tensor.Indexed + for _, nm := range names { + it := d.Dir.ValueByKey(nm) + if it != nil && it.Value != nil { + its = append(its, it.Value) + } else { + err := fmt.Errorf("datafs Dir %q item not found: %q", d.Path(), nm) + errs = append(errs, err) + } + } + return its, errors.Join(errs...) +} + // ItemsFunc returns data items in given directory -// filtered by given function, in alpha order. +// filtered by given function, in directory order (e.g., order added). // If func is nil, all items are returned. // Any directories within this directory are returned, // unless specifically filtered. @@ -72,92 +104,103 @@ func (d *Data) ItemsFunc(fun func(item *Data) bool) []*Data { if err := d.mustDir("ItemsFunc", ""); err != nil { return nil } - fm := d.filemap() - names := d.DirNamesAlpha() var its []*Data - for _, nm := range names { - dt := fm[nm] - if fun != nil && !fun(dt) { + for _, it := range d.Dir.Values { + if fun != nil && !fun(it) { + continue + } + its = append(its, it) + } + return its +} + +// ValuesFunc returns Value items (tensors) in given directory +// filtered by given function, in directory order (e.g., order added). +// If func is nil, all values are returned. +func (d *Data) ValuesFunc(fun func(item *Data) bool) []*tensor.Indexed { + if err := d.mustDir("ItemsFunc", ""); err != nil { + return nil + } + var its []*tensor.Indexed + for _, it := range d.Dir.Values { + if it.Value == nil { continue } - its = append(its, dt) + if fun != nil && !fun(it) { + continue + } + its = append(its, it.Value) } return its } -// ItemsByTimeFunc returns data items in given directory -// filtered by given function, in time order (i.e., order added). +// ItemsAlphaFunc returns data items in given directory +// filtered by given function, in alphabetical order. // If func is nil, all items are returned. // Any directories within this directory are returned, // unless specifically filtered. -func (d *Data) ItemsByTimeFunc(fun func(item *Data) bool) []*Data { - if err := d.mustDir("ItemsByTimeFunc", ""); err != nil { +func (d *Data) ItemsAlphaFunc(fun func(item *Data) bool) []*Data { + if err := d.mustDir("ItemsAlphaFunc", ""); err != nil { return nil } - fm := d.filemap() - names := d.DirNamesByTime() + names := d.DirNamesAlpha() var its []*Data for _, nm := range names { - dt := fm[nm] - if fun != nil && !fun(dt) { + it := d.Dir.ValueByKey(nm) + if fun != nil && !fun(it) { continue } - its = append(its, dt) + its = append(its, it) } return its } -// FlatItemsFunc returns all "leaf" (non directory) data items -// in given directory, recursively descending into directories -// to return a flat list of the entire subtree, -// filtered by given function, in alpha order. The function can -// filter out directories to prune the tree. -// If func is nil, all items are returned. -func (d *Data) FlatItemsFunc(fun func(item *Data) bool) []*Data { - if err := d.mustDir("FlatItemsFunc", ""); err != nil { +// FlatValuesFunc returns all Value items (tensor) in given directory, +// recursively descending into directories to return a flat list of +// the entire subtree, filtered by given function, in directory order +// (e.g., order added). +// The function can filter out directories to prune the tree. +// If func is nil, all Value items are returned. +func (d *Data) FlatValuesFunc(fun func(item *Data) bool) []*tensor.Indexed { + if err := d.mustDir("FlatValuesFunc", ""); err != nil { return nil } - fm := d.filemap() - names := d.DirNamesAlpha() - var its []*Data - for _, nm := range names { - dt := fm[nm] - if fun != nil && !fun(dt) { + var its []*tensor.Indexed + for _, it := range d.Dir.Values { + if fun != nil && !fun(it) { continue } - if dt.IsDir() { - subs := dt.FlatItemsFunc(fun) + if it.IsDir() { + subs := it.FlatValuesFunc(fun) its = append(its, subs...) } else { - its = append(its, dt) + its = append(its, it.Value) } } return its } -// FlatItemsByTimeFunc returns all "leaf" (non directory) data items -// in given directory, recursively descending into directories -// to return a flat list of the entire subtree, -// filtered by given function, in time order (i.e., order added). +// FlatValuesAlphaFunc returns all Value items (tensors) in given directory, +// recursively descending into directories to return a flat list of +// the entire subtree, filtered by given function, in alphabetical order. // The function can filter out directories to prune the tree. // If func is nil, all items are returned. -func (d *Data) FlatItemsByTimeFunc(fun func(item *Data) bool) []*Data { - if err := d.mustDir("FlatItemsByTimeFunc", ""); err != nil { +func (d *Data) FlatValuesAlphaFunc(fun func(item *Data) bool) []*tensor.Indexed { + if err := d.mustDir("FlatValuesFunc", ""); err != nil { return nil } - fm := d.filemap() - names := d.DirNamesByTime() - var its []*Data + names := d.DirNamesAlpha() + var its []*tensor.Indexed for _, nm := range names { - dt := fm[nm] - if fun != nil && !fun(dt) { + it := d.Dir.ValueByKey(nm) + if fun != nil && !fun(it) { continue } - if dt.IsDir() { - subs := dt.FlatItemsByTimeFunc(fun) + if it.IsDir() { + subs := it.FlatValuesAlphaFunc(fun) its = append(its, subs...) } else { - its = append(its, dt) + its = append(its, it.Value) } } return its @@ -193,31 +236,20 @@ func (d *Data) Path() string { } } -// filemap returns the Value as map[string]*Data, or nil if not a dir -func (d *Data) filemap() map[string]*Data { - fm, ok := d.Value.(map[string]*Data) - if !ok { - return nil - } - return fm -} - // DirNamesAlpha returns the names of items in the directory // sorted alphabetically. Data must be dir by this point. func (d *Data) DirNamesAlpha() []string { - fm := d.filemap() - names := maps.Keys(fm) + names := slices.Clone(d.Dir.Keys) sort.Strings(names) return names } // DirNamesByTime returns the names of items in the directory -// sorted by modTime (order added). Data must be dir by this point. +// sorted by modTime. Data must be dir by this point. func (d *Data) DirNamesByTime() []string { - fm := d.filemap() - names := maps.Keys(fm) + names := slices.Clone(d.Dir.Keys) slices.SortFunc(names, func(a, b string) int { - return fm[a].ModTime().Compare(fm[b].ModTime()) + return d.Dir.ValueByKey(a).ModTime().Compare(d.Dir.ValueByKey(b).ModTime()) }) return names } @@ -239,12 +271,10 @@ func (d *Data) Add(it *Data) error { if err := d.mustDir("Add", it.name); err != nil { return err } - fm := d.filemap() - _, ok := fm[it.name] - if ok { - return &fs.PathError{Op: "add", Path: it.name, Err: errors.New("data item already exists; names must be unique")} + err := d.Dir.Add(name, it) + if err != nil { + return &fs.PathError{Op: "Add", Path: it.name, Err: errors.New("data item already exists; names must be unique")} } - fm[it.name] = it return nil } diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index 034994cb99..3b401aa2b0 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -10,12 +10,10 @@ import ( "io/fs" "path" "slices" - "sort" "time" "unsafe" "cogentcore.org/core/base/fsx" - "golang.org/x/exp/maps" ) // fs.go contains all the io/fs interface implementations @@ -30,8 +28,7 @@ func (d *Data) Open(name string) (fs.File, error) { if err != nil { return nil, err } - fm := sd.filemap() - itm, ok := fm[file] + itm, ok := sd.Dir.ValueByKeyTry(file) if !ok { if dir == "" && (file == d.name || file == ".") { return &DirFile{File: File{Reader: *bytes.NewReader(d.Bytes()), Data: d}}, nil @@ -55,8 +52,7 @@ func (d *Data) Stat(name string) (fs.FileInfo, error) { if err != nil { return nil, err } - fm := sd.filemap() - itm, ok := fm[file] + itm, ok := sd.Dir.ValueByKeyTry(file) if !ok { if dir == "" && (file == d.name || file == ".") { return d, nil @@ -68,11 +64,11 @@ func (d *Data) Stat(name string) (fs.FileInfo, error) { // Sub returns a data FS corresponding to the subtree rooted at dir. func (d *Data) Sub(dir string) (fs.FS, error) { - if err := d.mustDir("sub", dir); err != nil { + if err := d.mustDir("Sub", dir); err != nil { return nil, err } if !fs.ValidPath(dir) { - return nil, &fs.PathError{Op: "sub", Path: dir, Err: errors.New("invalid name")} + return nil, &fs.PathError{Op: "Sub", Path: dir, Err: errors.New("invalid name")} } if dir == "." || dir == "" || dir == d.name { return d, nil @@ -92,8 +88,7 @@ func (d *Data) Sub(dir string) (fs.FS, error) { return cur, nil } cd = rest - fm := cur.filemap() - sd, ok := fm[root] + sd, ok := d.Dir.ValueByKeyTry(root) if !ok { return nil, &fs.PathError{Op: "sub", Path: dir, Err: errors.New("directory not found")} } @@ -111,12 +106,10 @@ func (d *Data) ReadDir(dir string) ([]fs.DirEntry, error) { if err != nil { return nil, err } - fm := sd.filemap() - names := maps.Keys(fm) - sort.Strings(names) + names := sd.DirNamesAlpha() ents := make([]fs.DirEntry, len(names)) for i, nm := range names { - ents[i] = fm[nm] + ents[i] = sd.Dir.ValueByKey(nm) } return ents, nil } @@ -140,8 +133,7 @@ func (d *Data) ReadFile(name string) ([]byte, error) { if err != nil { return nil, err } - fm := sd.filemap() - itm, ok := fm[file] + itm, ok := sd.Dir.ValueByKeyTry(file) if !ok { return nil, &fs.PathError{Op: "readFile", Path: name, Err: errors.New("file not found")} } @@ -185,8 +177,7 @@ func (d *Data) Size() int64 { } func (d *Data) IsDir() bool { - _, ok := d.Value.(map[string]*Data) - return ok + return d.Dir != nil } func (d *Data) ModTime() time.Time { @@ -200,8 +191,13 @@ func (d *Data) Mode() fs.FileMode { return 0444 } -// Sys returns the metadata for Value -func (d *Data) Sys() any { return d.Meta } +// Sys returns the Dir or Value +func (d *Data) Sys() any { + if d.Value != nil { + return d.Value + } + return d.Dir +} /////////////////////////////// // DirEntry interface diff --git a/tensor/datafs/metadata.go b/tensor/datafs/metadata.go index 3ff8d86c8d..548fd40015 100644 --- a/tensor/datafs/metadata.go +++ b/tensor/datafs/metadata.go @@ -15,12 +15,12 @@ import ( // This file provides standardized metadata options for frequent // use cases, using codified key names to eliminate typos. -// SetMetaItems sets given metadata for items in given directory +// SetMetaItems sets given metadata for Value items in given directory // with given names. Returns error for any items not found. func (d *Data) SetMetaItems(key string, value any, names ...string) error { - its, err := d.Items(names...) - for _, it := range its { - it.Meta.Set(key, value) + tsrs, err := d.Value(names...) + for _, tsr := range tsrs { + tsr.Tensor.Meta.Set(key, value) } return err } @@ -41,20 +41,29 @@ func (d *Data) SetPlotColumnOptions(opts *plotcore.ColumnOptions, names ...strin // PlotColumnOptions returns plotting options if they have been set, else nil. func (d *Data) PlotColumnOptions() *plotcore.ColumnOptions { - return errors.Ignore1(metadata.Get[*plotcore.ColumnOptions](d.Meta, "PlotColumnOptions")) + if d.Value == nil { + return + } + return errors.Ignore1(metadata.Get[*plotcore.ColumnOptions](d.Value.Tensor.Meta, "PlotColumnOptions")) } -// SetCalcFunc sets a function to compute an updated Value for this data item. +// SetCalcFunc sets a function to compute an updated Value for this Value item. // Function is stored as CalcFunc in Metadata. Can be called by [Data.Calc] method. func (d *Data) SetCalcFunc(fun func() error) { - d.Meta.Set("CalcFunc", fun) + if d.Value == nil { + return + } + d.Value.Tensor.Meta.Set("CalcFunc", fun) } // Calc calls function set by [Data.SetCalcFunc] to compute an updated Value // for this data item. Returns an error if func not set, or any error from func itself. // Function is stored as CalcFunc in Metadata. func (d *Data) Calc() error { - fun, err := metadata.Get[func() error](d.Meta, "CalcFunc") + if d.Value == nil { + return + } + fun, err := metadata.Get[func() error](d.Value.Tensor.Meta, "CalcFunc") if err != nil { return err } @@ -63,10 +72,10 @@ func (d *Data) Calc() error { // CalcAll calls function set by [Data.SetCalcFunc] for all items // in this directory and all of its subdirectories. -// Calls Calc on items from FlatItemsByTimeFunc(nil) +// Calls Calc on items from FlatValuesFunc(nil) func (d *Data) CalcAll() error { var errs []error - items := d.FlatItemsByTimeFunc(nil) + items := d.FlatValuesFunc(nil) for _, it := range items { err := it.Calc() if err != nil { @@ -76,41 +85,34 @@ func (d *Data) CalcAll() error { return errors.Join(errs...) } -// DirTable returns a table.Table for this directory item, with columns -// as the Tensor elements in the directory and any subdirectories, -// from FlatItemsByTimeFunc using given filter function. +// GetDirTable gets the DirTable as a [table.Table] for this directory item, +// with columns as the Tensor values elements in the directory +// and any subdirectories, from FlatValuesFunc using given filter function. // This is a convenient mechanism for creating a plot of all the data // in a given directory. // If such was previously constructed, it is returned from "DirTable" -// Metadata key where the table is stored. +// where it is stored for later use. // Row count is updated to current max row. -// Delete that key to reconstruct if items have changed. -func (d *Data) DirTable(fun func(item *Data) bool) *table.Table { - dt, err := metadata.Get[*table.Table](d.Meta, "DirTable") - if err == nil { - var maxRow int - for _, tsr := range dt.Columns { - maxRow = max(maxRow, tsr.DimSize(0)) - } - dt.Rows = maxRow +// Set DirTable = nil to regenerate. +func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { + if d.DirTable != nil { + d.DirTable.SetNumRowsToMax() return dt } - items := d.FlatItemsByTimeFunc(fun) - dt = table.NewTable(fsx.DirAndFile(string(d.Path()))) - for _, it := range items { - tsr := it.AsTensor() - if tsr == nil { - continue - } - if dt.Rows == 0 { - dt.Rows = tsr.DimSize(0) + tsrs := d.FlatValuesFunc(fun) + dt := table.NewTable(fsx.DirAndFile(string(d.Path()))) + for _, tsr := range tsrs { + rows := tsr.Tensor.Rows() + if dt.Columns.Rows < rows { + dt.Columns.Rows = rows + dt.SetNumRows(dt.Columns.Rows) } nm := it.Name() if it.Parent != d { nm = fsx.DirAndFile(string(it.Path())) } - dt.AddColumn(tsr, nm) + dt.AddColumn(tsr.Tensor, nm) } - d.Meta.Set("DirTable", dt) + d.DirTable = dt return dt } diff --git a/tensor/datafs/stats.go b/tensor/datafs/stats.go index e6606692ee..4c2a87be88 100644 --- a/tensor/datafs/stats.go +++ b/tensor/datafs/stats.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build not + package datafs import ( @@ -10,6 +12,8 @@ import ( "cogentcore.org/core/tensor/table" ) +/* + // StdStats are the standard descriptive stats computed in StdStatsData, // for one-dimensional tensors. For higher-dimensional cases, the last 3 // quartile-based ones are excluded because they are not compatible. @@ -59,7 +63,7 @@ func StdStatsData(dir *datafs.Data, tsr *tensor.Indexed) { // DescAll returns a table of standard descriptive stats for // all numeric columns in given table, operating over all non-Null, non-NaN elements // in each column. -func DescAll(ix *table.Indexed) *table.Table { +func DescAll(ix *table.Table) *table.Table { /* st := ix.Table nAgg := len(DescStats) @@ -106,3 +110,6 @@ func DescAll(ix *table.Indexed) *table.Table { */ return ix.Table // dt } + +*/ + diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 2062c560b5..8bee76b464 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -28,7 +28,7 @@ type Sim struct { func (ss *Sim) ConfigAll() { ss.Root = errors.Log1(datafs.NewDir("Root")) ss.Config = errors.Log1(ss.Root.Mkdir("Config")) - errors.Log1(datafs.New[int](ss.Config, "NRun", "NEpoch", "NTrial")) + datafs.New[int](ss.Config, "NRun", "NEpoch", "NTrial") ss.Config.Item("NRun").SetInt(5) ss.Config.Item("NEpoch").SetInt(20) ss.Config.Item("NTrial").SetInt(25) diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index 8187c896f3..832611dc96 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -18,7 +18,7 @@ import ( // linear model, which is a general form of multivariate linear // regression, supporting multiple independent and dependent // variables. Make a NewGLM and then do Run() on a tensor -// table.Indexed with the relevant data in columns of the table. +// table.Table with the relevant data in columns of the table. // Batch-mode gradient descent is used and the relevant parameters // can be altered from defaults before calling Run as needed. type GLM struct { @@ -85,7 +85,7 @@ type GLM struct { // Cached values from the table // Table of data - Table *table.Indexed + Table *table.Table // tensor columns from table with the respective variables IndepVars, DepVars, PredVars, ErrVars tensor.Tensor @@ -122,7 +122,7 @@ func (glm *GLM) init(nIv, nDv int) { // each of the Vars args specifies a column in the table, which can have either a // single scalar value for each row, or a tensor cell with multiple values. // predVars and errVars (predicted values and error values) are optional. -func (glm *GLM) SetTable(ix *table.Indexed, indepVars, depVars, predVars, errVars string) error { +func (glm *GLM) SetTable(ix *table.Table, indepVars, depVars, predVars, errVars string) error { dt := ix.Table iv, err := dt.Column(indepVars) if err != nil { diff --git a/tensor/stats/split/group.go b/tensor/stats/split/group.go index c807b473b3..18a5d0c16e 100644 --- a/tensor/stats/split/group.go +++ b/tensor/stats/split/group.go @@ -16,7 +16,7 @@ import ( // All returns a single "split" with all of the rows in given view // useful for leveraging the aggregation management functions in splits -func All(ix *table.Indexed) *table.Splits { +func All(ix *table.Table) *table.Splits { spl := &table.Splits{} spl.Levels = []string{"All"} spl.New(ix.Table, []string{"All"}, ix.Indexes...) @@ -26,7 +26,7 @@ func All(ix *table.Indexed) *table.Splits { // GroupByIndex returns a new Splits set based on the groups of values // across the given set of column indexes. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupByIndex(ix *table.Indexed, colIndexes []int) *table.Splits { +func GroupByIndex(ix *table.Table, colIndexes []int) *table.Splits { nc := len(colIndexes) if nc == 0 || ix.Table == nil { return nil @@ -44,7 +44,7 @@ func GroupByIndex(ix *table.Indexed, colIndexes []int) *table.Splits { srt.SortStableColumns(colIndexes, true) // important for consistency lstValues := make([]string, nc) curValues := make([]string, nc) - var curIx *table.Indexed + var curIx *table.Table for _, rw := range srt.Indexes { diff := false for i, ci := range colIndexes { @@ -71,7 +71,7 @@ func GroupByIndex(ix *table.Indexed, colIndexes []int) *table.Splits { // GroupBy returns a new Splits set based on the groups of values // across the given set of column names. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupBy(ix *table.Indexed, columns ...string) *table.Splits { +func GroupBy(ix *table.Table, columns ...string) *table.Splits { return GroupByIndex(ix, errors.Log1(ix.Table.ColumnIndexesByNames(columns...))) } @@ -80,7 +80,7 @@ func GroupBy(ix *table.Indexed, columns ...string) *table.Splits { // The function should always return the same number of values -- if // it doesn't behavior is undefined. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupByFunc(ix *table.Indexed, fun func(row int) []string) *table.Splits { +func GroupByFunc(ix *table.Table, fun func(row int) []string) *table.Splits { if ix.Table == nil { return nil } @@ -113,7 +113,7 @@ func GroupByFunc(ix *table.Indexed, fun func(row int) []string) *table.Splits { // now do our usual grouping operation spl := &table.Splits{} lstValues := make([]string, nv) - var curIx *table.Indexed + var curIx *table.Table for _, rw := range srt.Indexes { curValues := funvals[rw] diff := (curIx == nil) diff --git a/tensor/stats/split/random.go b/tensor/stats/split/random.go index 047f49ef96..3123c47aa4 100644 --- a/tensor/stats/split/random.go +++ b/tensor/stats/split/random.go @@ -16,7 +16,7 @@ import ( // which will be normalized to sum to 1 (error returned if sum = 0) // names are optional names for each split (e.g., Train, Test) which will be // used to label the Values of the resulting Splits. -func Permuted(ix *table.Indexed, probs []float64, names []string) (*table.Splits, error) { +func Permuted(ix *table.Table, probs []float64, names []string) (*table.Splits, error) { if ix == nil || ix.Len() == 0 { return nil, fmt.Errorf("split.Random table is nil / empty") } diff --git a/tensor/stats/split/splits.go b/tensor/stats/split/splits.go index 99aca74d45..2893f42a7d 100644 --- a/tensor/stats/split/splits.go +++ b/tensor/stats/split/splits.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package table +package split import ( "fmt" @@ -95,7 +95,7 @@ func (spl *Splits) Table() *Table { // Any existing Aggs are deleted by this. func (spl *Splits) New(dt *Table, values []string, rows ...int) *Table { spl.Aggs = nil - ix := NewTableView(dt) + ix := NewView(dt) spl.Splits = append(spl.Splits, ix) if len(rows) > 0 { ix.Indexes = append(ix.Indexes, slices.Clone(rows)...) diff --git a/tensor/table/README.md b/tensor/table/README.md index 24ec4c3318..1caa758562 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -18,7 +18,7 @@ If you call Sort, Filter or other routines on an individual column tensor, then There are also multi-column `Sort` and `Filter` methods on the Table itself. -It is very low-cost to create a new View of an existing Table, via `NewTableView`, as they can share the underlying `Columns` data. +It is very low-cost to create a new View of an existing Table, via `NewView`, as they can share the underlying `Columns` data. # Cheat Sheet diff --git a/tensor/table/table.go b/tensor/table/table.go index ac95aa1cdb..0311e9df9b 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -52,10 +52,10 @@ func NewTable(name ...string) *Table { return dt } -// NewTableView returns a new Table with its own Indexed view into the +// NewView returns a new Table with its own Indexed view into the // same underlying set of Column tensor data as the source table. // Indexes are nil in the new Table, resulting in default full sequential view. -func NewTableView(src *Table) *Table { +func NewView(src *Table) *Table { dt := &Table{Columns: src.Columns} dt.Meta.Copy(src.Meta) return dt @@ -246,6 +246,19 @@ func (dt *Table) SetNumRows(rows int) *Table { //types:add return dt } +// SetNumRowsToMax gets the current max number of rows across all the column tensors, +// and sets the number of rows to that. This will automatically pad shorter columns +// so they all have the same number of rows. If a table has columns that are not fully +// under its own control, they can change size, so this reestablishes +// a common row dimension. +func (dt *Table) SetNumRowsToMax() { + var maxRow int + for _, tsr := range dt.Columns.Values { + maxRow = max(maxRow, tsr.DimSize(0)) + } + dt.SetNumRows(maxRow) +} + // note: no really clean definition of CopyFrom -- no point of re-using existing // table -- just clone it. diff --git a/tensor/table/typegen.go b/tensor/table/typegen.go index eb0a757516..bedd6f2677 100644 --- a/tensor/table/typegen.go +++ b/tensor/table/typegen.go @@ -6,6 +6,6 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "IndexedColumnName", Doc: "IndexedColumnName returns a tensor.Indexed view of given column name,\nusing our current indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column"}, Returns: []string{"Indexed", "error"}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "IndexedColumnName", Doc: "IndexedColumnName returns a tensor.Indexed view of given column name,\nusing our current indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column"}, Returns: []string{"Indexed", "error"}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of data, with columns of tensors,\neach with the same number of Rows (outermost dimension).", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to each of the columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns\nif rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "columns of data, as tensor.Tensor tensors"}, {Name: "ColumnNames", Doc: "the names of the columns"}, {Name: "Rows", Doc: "number of rows, which is enforced to be the size of the outermost dimension of the column tensors"}, {Name: "ColumnNameMap", Doc: "the map of column names to column numbers"}, {Name: "MetaData", Doc: "misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column"}}}) diff --git a/tensor/tensorcore/simatgrid.go b/tensor/tensorcore/simatgrid.go index a134c3199f..a0e2f610ad 100644 --- a/tensor/tensorcore/simatgrid.go +++ b/tensor/tensorcore/simatgrid.go @@ -4,6 +4,8 @@ package tensorcore +/* + import ( "cogentcore.org/core/colors" "cogentcore.org/core/math32" @@ -237,3 +239,5 @@ func (tg *SimMatGrid) Render() { } } } + +*/ diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 9904362bbd..3675c56bfc 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -34,34 +34,34 @@ import ( type Table struct { core.ListBase - // the idx view of the table that we're a view of - Table *table.Indexed `set:"-"` + // Table is the table that we're a view of. + Table *table.Table `set:"-"` - // overall display options for tensor display + // overall display options for tensor display. TensorDisplay TensorDisplay `set:"-"` - // per column tensor display params + // per column tensor display params. ColumnTensorDisplay map[int]*TensorDisplay `set:"-"` - // per column blank tensor values + // per column blank tensor values. ColumnTensorBlank map[int]*tensor.Float64 `set:"-"` - // number of columns in table (as of last update) + // number of columns in table (as of last update). NCols int `edit:"-"` - // current sort index + // current sort index. SortIndex int - // whether current sort order is descending + // whether current sort order is descending. SortDescending bool - // headerWidths has number of characters in each header, per visfields + // headerWidths has number of characters in each header, per visfields. headerWidths []int `copier:"-" display:"-" json:"-" xml:"-"` - // colMaxWidths records maximum width in chars of string type fields + // colMaxWidths records maximum width in chars of string type fields. colMaxWidths []int `set:"-" copier:"-" json:"-" xml:"-"` - // blank values for out-of-range rows + // blank values for out-of-range rows. BlankString string BlankFloat float64 } @@ -133,13 +133,12 @@ func (tb *Table) StyleValue(w core.Widget, s *styles.Style, row, col int) { // SetTable sets the source table that we are viewing, using a sequential Indexed // and then configures the display -func (tb *Table) SetTable(et *table.Table) *Table { - if et == nil { +func (tb *Table) SetTable(dt *table.Table) *Table { + if dt == nil { return nil } - tb.SetSliceBase() - tb.Table = table.NewIndexed(et) + tb.Table = table.NewView(dt) tb.This.(core.Lister).UpdateSliceSize() tb.Update() return tb @@ -163,7 +162,7 @@ func (tb *Table) AsyncUpdateTable() { // SetIndexed sets the source Indexed of a table (using a copy so original is not modified) // and then configures the display -func (tb *Table) SetIndexed(ix *table.Indexed) *Table { +func (tb *Table) SetIndexed(ix *table.Table) *Table { if ix == nil { return tb } @@ -185,11 +184,11 @@ func (tb *Table) SetIndexed(ix *table.Indexed) *Table { func (tb *Table) UpdateSliceSize() int { tb.Table.DeleteInvalid() // table could have changed - if tb.Table.Len() == 0 { + if tb.Table.NumRows() == 0 { tb.Table.Sequential() } - tb.SliceSize = tb.Table.Len() - tb.NCols = tb.Table.Table.NumColumns() + tb.SliceSize = tb.Table.NumRows() + tb.NCols = tb.Table.NumColumns() return tb.SliceSize } @@ -204,7 +203,7 @@ func (tb *Table) UpdateMaxWidths() { } for fli := 0; fli < tb.NCols; fli++ { tb.colMaxWidths[fli] = 0 - col := tb.Table.Table.Columns[fli] + col := tb.Table.Columns.Values[fli] stsr, isstr := col.(*tensor.String) if !isstr { @@ -240,7 +239,7 @@ func (tb *Table) MakeHeader(p *tree.Plan) { }) } for fli := 0; fli < tb.NCols; fli++ { - field := tb.Table.Table.ColumnNames[fli] + field := tb.Table.Columns.Keys[fli] tree.AddAt(p, "head-"+field, func(w *core.Button) { w.SetType(core.ButtonAction) w.Styler(func(s *styles.Style) { @@ -250,7 +249,7 @@ func (tb *Table) MakeHeader(p *tree.Plan) { tb.SortSliceAction(fli) }) w.Updater(func() { - field := tb.Table.Table.ColumnNames[fli] + field := tb.Table.Columns.Keys[fli] w.SetText(field).SetTooltip(field + " (tap to sort by)") tb.headerWidths[fli] = len(field) if fli == tb.SortIndex { @@ -295,7 +294,7 @@ func (tb *Table) MakeRow(p *tree.Plan, i int) { } for fli := 0; fli < tb.NCols; fli++ { - col := tb.Table.Table.Columns[fli] + col := tb.Table.Columns.Values[fli] valnm := fmt.Sprintf("value-%v.%v", fli, itxt) _, isstr := col.(*tensor.String) @@ -314,11 +313,12 @@ func (tb *Table) MakeRow(p *tree.Plan, i int) { w.AsTree().SetProperty(core.ListColProperty, fli) if !tb.IsReadOnly() { wb.OnChange(func(e events.Event) { - if si < len(tb.Table.Indexes) { + _, vi, invis := svi.SliceIndex(i) + if !invis { if isstr { - tb.Table.Table.SetStringIndex(fli, tb.Table.Indexes[si], str) + col.SetString1D(str, vi) } else { - tb.Table.Table.SetFloatIndex(fli, tb.Table.Indexes[si], fval) + col.SetFloat1D(fval, vi) } } tb.SendChange() @@ -328,10 +328,10 @@ func (tb *Table) MakeRow(p *tree.Plan, i int) { _, vi, invis := svi.SliceIndex(i) if !invis { if isstr { - str = tb.Table.Table.StringIndex(fli, vi) + str = col.String1D(vi) core.Bind(&str, w) } else { - fval = tb.Table.Table.FloatIndex(fli, vi) + fval = col.Float1D(vi) core.Bind(&fval, w) } } else { @@ -366,7 +366,7 @@ func (tb *Table) MakeRow(p *tree.Plan, i int) { if invis { cell = tb.ColTensorBlank(fli, col) } else { - cell = tb.Table.Table.TensorIndex(fli, vi) + cell = col.RowTensor(vi) } wb.ValueTitle = tb.ValueTitle + "[" + strconv.Itoa(si) + "]" w.SetState(invis, states.Invisible) @@ -383,7 +383,7 @@ func (tb *Table) ColTensorBlank(cidx int, col tensor.Tensor) *tensor.Float64 { if ctb, has := tb.ColumnTensorBlank[cidx]; has { return ctb } - ctb := tensor.New[float64](col.Shape().Sizes, col.Shape().Names...).(*tensor.Float64) + ctb := tensor.New[float64](col.Shape().Sizes...).(*tensor.Float64) tb.ColumnTensorBlank[cidx] = ctb return ctb } @@ -395,7 +395,7 @@ func (tb *Table) GetColumnTensorDisplay(col int) *TensorDisplay { return ctd } if tb.Table != nil { - cl := tb.Table.Table.Columns[col] + cl := tb.Table.Columns.Values[col] if len(*cl.Metadata()) > 0 { return tb.SetColumnTensorDisplay(col) } @@ -412,7 +412,7 @@ func (tb *Table) SetColumnTensorDisplay(col int) *TensorDisplay { ctd := &TensorDisplay{} *ctd = tb.TensorDisplay if tb.Table != nil { - cl := tb.Table.Table.Columns[col] + cl := tb.Table.Columns.Values[col] ctd.FromMeta(cl) } tb.ColumnTensorDisplay[col] = ctd @@ -463,7 +463,10 @@ func (tb *Table) SortSliceAction(fldIndex int) { if fldIndex == -1 { tb.Table.SortIndexes() } else { - tb.Table.SortColumn(tb.SortIndex, !tb.SortDescending) + tb.Table.IndexesNeeded() + col := tb.Table.ColumnIndex(tb.SortIndex) + col.Sort(!tb.SortDescending) + tb.Table.IndexesFromTensor(col) } tb.Update() // requires full update due to sort button icon } @@ -490,7 +493,7 @@ func (tb *Table) StyleRow(w core.Widget, idx, fidx int) {} // :down depending on descending func (tb *Table) SortFieldName() string { if tb.SortIndex >= 0 && tb.SortIndex < tb.NCols { - nm := tb.Table.Table.ColumnNames[tb.SortIndex] + nm := tb.Table.Columns.Keys[tb.SortIndex] if tb.SortDescending { nm += ":down" } else { @@ -510,7 +513,7 @@ func (tb *Table) SetSortFieldName(nm string) { spnm := strings.Split(nm, ":") got := false for fli := 0; fli < tb.NCols; fli++ { - fld := tb.Table.Table.ColumnNames[fli] + fld := tb.Table.Columns.Keys[fli] if fld == spnm[0] { got = true // fmt.Println("sorting on:", fld.Name, fli, "from:", nm) @@ -613,14 +616,15 @@ func (tb *Table) SizeFinal() { // SelectedColumnStrings returns the string values of given column name. func (tb *Table) SelectedColumnStrings(colName string) []string { - dt := tb.Table.Table + dt := tb.Table jis := tb.SelectedIndexesList(false) if len(jis) == 0 || dt == nil { return nil } var s []string + col := dt.Column(colName) for _, i := range jis { - v := dt.StringValue(colName, i) + v := col.StringRowCell(i, 0) s = append(s, v) } return s @@ -630,21 +634,22 @@ func (tb *Table) SelectedColumnStrings(colName string) []string { // Copy / Cut / Paste func (tb *Table) MakeToolbar(p *tree.Plan) { - if tb.Table == nil || tb.Table.Table == nil { + if tb.Table == nil || tb.Table == nil { return } tree.Add(p, func(w *core.FuncButton) { w.SetFunc(tb.Table.AddRows).SetIcon(icons.Add) w.SetAfterFunc(func() { tb.Update() }) }) - tree.Add(p, func(w *core.FuncButton) { - w.SetFunc(tb.Table.SortColumnName).SetText("Sort").SetIcon(icons.Sort) - w.SetAfterFunc(func() { tb.Update() }) - }) - tree.Add(p, func(w *core.FuncButton) { - w.SetFunc(tb.Table.FilterColumnName).SetText("Filter").SetIcon(icons.FilterAlt) - w.SetAfterFunc(func() { tb.Update() }) - }) + // todo: + // tree.Add(p, func(w *core.FuncButton) { + // w.SetFunc(tb.Table.SortColumnName).SetText("Sort").SetIcon(icons.Sort) + // w.SetAfterFunc(func() { tb.Update() }) + // }) + // tree.Add(p, func(w *core.FuncButton) { + // w.SetFunc(tb.Table.FilterColumnName).SetText("Filter").SetIcon(icons.FilterAlt) + // w.SetAfterFunc(func() { tb.Update() }) + // }) tree.Add(p, func(w *core.FuncButton) { w.SetFunc(tb.Table.Sequential).SetText("Unfilter").SetIcon(icons.FilterAltOff) w.SetAfterFunc(func() { tb.Update() }) @@ -669,8 +674,7 @@ func (tb *Table) CopySelectToMime() mimedata.Mimes { if nitms == 0 { return nil } - ix := &table.Indexed{} - ix.Table = tb.Table.Table + ix := table.NewView(tb.Table) idx := tb.SelectedIndexesList(false) // ascending iidx := make([]int, len(idx)) for i, di := range idx { @@ -709,7 +713,7 @@ func (tb *Table) PasteAssign(md mimedata.Mimes, idx int) { if len(recs) == 0 { return } - tb.Table.Table.ReadCSVRow(recs[1], tb.Table.Indexes[idx]) + tb.Table.ReadCSVRow(recs[1], tb.Table.Indexes[idx]) tb.UpdateChange() } @@ -725,7 +729,7 @@ func (tb *Table) PasteAtIndex(md mimedata.Mimes, idx int) { for ri := 0; ri < nr; ri++ { rec := recs[1+ri] rw := tb.Table.Indexes[idx+ri] - tb.Table.Table.ReadCSVRow(rec, rw) + tb.Table.ReadCSVRow(rec, rw) } tb.SendChange() tb.SelectIndexEvent(idx, events.SelectOne) diff --git a/tensor/tensorcore/tensoreditor.go b/tensor/tensorcore/tensoreditor.go index 7c57bea909..08ac7f264c 100644 --- a/tensor/tensorcore/tensoreditor.go +++ b/tensor/tensorcore/tensoreditor.go @@ -19,7 +19,6 @@ import ( "cogentcore.org/core/styles/states" "cogentcore.org/core/styles/units" "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/table" "cogentcore.org/core/tree" ) @@ -372,7 +371,7 @@ func (tb *TensorEditor) SizeFinal() { // Outer-most dims are rows in the file, and inner-most is column -- // Reading just grabs all values and doesn't care about shape. func (tb *TensorEditor) SaveCSV(filename core.Filename) error { //types:add - return tensor.SaveCSV(tb.Tensor, filename, table.Tab.Rune()) + return tensor.SaveCSV(tb.Tensor, filename, tensor.Tab) } // OpenTSV reads a tensor from a tab-separated-values (TSV) file. @@ -380,7 +379,7 @@ func (tb *TensorEditor) SaveCSV(filename core.Filename) error { //types:add // to the official CSV standard. // Reads all values and assigns as many as fit. func (tb *TensorEditor) OpenCSV(filename core.Filename) error { //types:add - return tensor.OpenCSV(tb.Tensor, filename, table.Tab.Rune()) + return tensor.OpenCSV(tb.Tensor, filename, tensor.Tab) } func (tb *TensorEditor) MakeToolbar(p *tree.Plan) { diff --git a/tensor/tensorcore/tensorgrid.go b/tensor/tensorcore/tensorgrid.go index a6d054a744..37ed6df590 100644 --- a/tensor/tensorcore/tensorgrid.go +++ b/tensor/tensorcore/tensorgrid.go @@ -7,8 +7,8 @@ package tensorcore import ( "image/color" "log" - "strconv" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/colors" "cogentcore.org/core/colors/colormap" "cogentcore.org/core/core" @@ -108,65 +108,44 @@ func (td *TensorDisplay) Defaults() { // FromMeta sets display options from Tensor meta-data func (td *TensorDisplay) FromMeta(tsr tensor.Tensor) { - if op, has := tsr.MetaData("top-zero"); has { - if op == "+" || op == "true" { - td.TopZero = true - } + if op, err := metadata.Get[bool](*tsr.Metadata(), "top-zero"); err == nil { + td.TopZero = op } - if op, has := tsr.MetaData("odd-row"); has { - if op == "+" || op == "true" { - td.OddRow = true - } + if op, err := metadata.Get[bool](*tsr.Metadata(), "odd-row"); err == nil { + td.OddRow = op } - if op, has := tsr.MetaData("image"); has { - if op == "+" || op == "true" { - td.Image = true - } + if op, err := metadata.Get[bool](*tsr.Metadata(), "image"); err == nil { + td.Image = op } - if op, has := tsr.MetaData("min"); has { - mv, _ := strconv.ParseFloat(op, 64) - td.Range.Min = mv + if op, err := metadata.Get[float64](*tsr.Metadata(), "min"); err == nil { + td.Range.Min = op } - if op, has := tsr.MetaData("max"); has { - mv, _ := strconv.ParseFloat(op, 64) - td.Range.Max = mv + if op, err := metadata.Get[float64](*tsr.Metadata(), "max"); err == nil { + td.Range.Max = op } - if op, has := tsr.MetaData("fix-min"); has { - if op == "+" || op == "true" { - td.Range.FixMin = true - } else { - td.Range.FixMin = false - } + if op, err := metadata.Get[bool](*tsr.Metadata(), "fix-min"); err == nil { + td.Range.FixMin = op } - if op, has := tsr.MetaData("fix-max"); has { - if op == "+" || op == "true" { - td.Range.FixMax = true - } else { - td.Range.FixMax = false - } + if op, err := metadata.Get[bool](*tsr.Metadata(), "fix-max"); err == nil { + td.Range.FixMax = op } - if op, has := tsr.MetaData("colormap"); has { + if op, err := metadata.Get[string](*tsr.Metadata(), "colormap"); err == nil { td.ColorMap = core.ColorMapName(op) } - if op, has := tsr.MetaData("grid-fill"); has { - mv, _ := strconv.ParseFloat(op, 32) - td.GridFill = float32(mv) + if op, err := metadata.Get[float64](*tsr.Metadata(), "grid-fill"); err == nil { + td.GridFill = float32(op) } - if op, has := tsr.MetaData("grid-min"); has { - mv, _ := strconv.ParseFloat(op, 32) - td.GridMinSize = float32(mv) + if op, err := metadata.Get[float64](*tsr.Metadata(), "grid-min"); err == nil { + td.GridMinSize = float32(op) } - if op, has := tsr.MetaData("grid-max"); has { - mv, _ := strconv.ParseFloat(op, 32) - td.GridMaxSize = float32(mv) + if op, err := metadata.Get[float64](*tsr.Metadata(), "grid-max"); err == nil { + td.GridMaxSize = float32(op) } - if op, has := tsr.MetaData("dim-extra"); has { - mv, _ := strconv.ParseFloat(op, 32) - td.DimExtra = float32(mv) + if op, err := metadata.Get[float64](*tsr.Metadata(), "dim-extra"); err == nil { + td.DimExtra = float32(op) } - if op, has := tsr.MetaData("font-size"); has { - mv, _ := strconv.ParseFloat(op, 32) - td.FontSize = float32(mv) + if op, err := metadata.Get[float64](*tsr.Metadata(), "font-size"); err == nil { + td.FontSize = float32(op) } } @@ -351,11 +330,11 @@ func (tg *TensorGrid) Render() { case outclr: var r, g, b, a float64 a = 1 - r = tg.Display.Range.ClipNormValue(tsr.Float([]int{0, y, x})) - g = tg.Display.Range.ClipNormValue(tsr.Float([]int{1, y, x})) - b = tg.Display.Range.ClipNormValue(tsr.Float([]int{2, y, x})) + r = tg.Display.Range.ClipNormValue(tsr.Float(0, y, x)) + g = tg.Display.Range.ClipNormValue(tsr.Float(1, y, x)) + b = tg.Display.Range.ClipNormValue(tsr.Float(2, y, x)) if nclr > 3 { - a = tg.Display.Range.ClipNormValue(tsr.Float([]int{3, y, x})) + a = tg.Display.Range.ClipNormValue(tsr.Float(3, y, x)) } cr := math32.Vec2(float32(x), float32(ey)) pr := pos.Add(cr.Mul(gsz)) @@ -364,18 +343,18 @@ func (tg *TensorGrid) Render() { case nclr > 1: var r, g, b, a float64 a = 1 - r = tg.Display.Range.ClipNormValue(tsr.Float([]int{y, x, 0})) - g = tg.Display.Range.ClipNormValue(tsr.Float([]int{y, x, 1})) - b = tg.Display.Range.ClipNormValue(tsr.Float([]int{y, x, 2})) + r = tg.Display.Range.ClipNormValue(tsr.Float(y, x, 0)) + g = tg.Display.Range.ClipNormValue(tsr.Float(y, x, 1)) + b = tg.Display.Range.ClipNormValue(tsr.Float(y, x, 2)) if nclr > 3 { - a = tg.Display.Range.ClipNormValue(tsr.Float([]int{y, x, 3})) + a = tg.Display.Range.ClipNormValue(tsr.Float(y, x, 3)) } cr := math32.Vec2(float32(x), float32(ey)) pr := pos.Add(cr.Mul(gsz)) pc.StrokeStyle.Color = colors.Uniform(colors.FromFloat64(r, g, b, a)) pc.FillBox(pr, gsz, pc.StrokeStyle.Color) default: - val := tg.Display.Range.ClipNormValue(tsr.Float([]int{y, x})) + val := tg.Display.Range.ClipNormValue(tsr.Float(y, x)) cr := math32.Vec2(float32(x), float32(ey)) pr := pos.Add(cr.Mul(gsz)) pc.StrokeStyle.Color = colors.Uniform(colors.FromFloat64(val, val, val, 1)) diff --git a/tensor/tensorcore/typegen.go b/tensor/tensorcore/typegen.go index bbea16ff82..8a42d7c352 100644 --- a/tensor/tensorcore/typegen.go +++ b/tensor/tensorcore/typegen.go @@ -5,19 +5,11 @@ package tensorcore import ( "cogentcore.org/core/colors/colormap" "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/simat" "cogentcore.org/core/tensor/table" "cogentcore.org/core/tree" "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/tensorcore.SimMatGrid", IDName: "sim-mat-grid", Doc: "SimMatGrid is a widget that displays a similarity / distance matrix\nwith tensor values as a grid of colored squares, and labels for rows and columns.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Embeds: []types.Field{{Name: "TensorGrid"}}, Fields: []types.Field{{Name: "SimMat", Doc: "the similarity / distance matrix"}, {Name: "rowMaxSz"}, {Name: "rowMinBlank"}, {Name: "rowNGps"}, {Name: "colMaxSz"}, {Name: "colMinBlank"}, {Name: "colNGps"}}}) - -// NewSimMatGrid returns a new [SimMatGrid] with the given optional parent: -// SimMatGrid is a widget that displays a similarity / distance matrix -// with tensor values as a grid of colored squares, and labels for rows and columns. -func NewSimMatGrid(parent ...tree.Node) *SimMatGrid { return tree.New[SimMatGrid](parent...) } - var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/tensorcore.Table", IDName: "table", Doc: "Table provides a GUI widget for representing [table.Table] values.", Embeds: []types.Field{{Name: "ListBase"}}, Fields: []types.Field{{Name: "Table", Doc: "the idx view of the table that we're a view of"}, {Name: "TensorDisplay", Doc: "overall display options for tensor display"}, {Name: "ColumnTensorDisplay", Doc: "per column tensor display params"}, {Name: "ColumnTensorBlank", Doc: "per column blank tensor values"}, {Name: "NCols", Doc: "number of columns in table (as of last update)"}, {Name: "SortIndex", Doc: "current sort index"}, {Name: "SortDescending", Doc: "whether current sort order is descending"}, {Name: "headerWidths", Doc: "headerWidths has number of characters in each header, per visfields"}, {Name: "colMaxWidths", Doc: "colMaxWidths records maximum width in chars of string type fields"}, {Name: "BlankString", Doc: "\tblank values for out-of-range rows"}, {Name: "BlankFloat"}}}) // NewTable returns a new [Table] with the given optional parent: @@ -98,12 +90,3 @@ func NewTableButton(parent ...tree.Node) *TableButton { return tree.New[TableBut // SetTable sets the [TableButton.Table] func (t *TableButton) SetTable(v *table.Table) *TableButton { t.Table = v; return t } - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/tensorcore.SimMatButton", IDName: "sim-mat-button", Doc: "SimMatValue presents a button that pulls up the [SimMatGrid] viewer for a [table.Table].", Embeds: []types.Field{{Name: "Button"}}, Fields: []types.Field{{Name: "SimMat"}}}) - -// NewSimMatButton returns a new [SimMatButton] with the given optional parent: -// SimMatValue presents a button that pulls up the [SimMatGrid] viewer for a [table.Table]. -func NewSimMatButton(parent ...tree.Node) *SimMatButton { return tree.New[SimMatButton](parent...) } - -// SetSimMat sets the [SimMatButton.SimMat] -func (t *SimMatButton) SetSimMat(v *simat.SimMat) *SimMatButton { t.SimMat = v; return t } diff --git a/tensor/tensorcore/values.go b/tensor/tensorcore/values.go index 26eee622cd..de27c12fdd 100644 --- a/tensor/tensorcore/values.go +++ b/tensor/tensorcore/values.go @@ -5,10 +5,10 @@ package tensorcore import ( + "cogentcore.org/core/base/metadata" "cogentcore.org/core/core" "cogentcore.org/core/icons" "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/simat" "cogentcore.org/core/tensor/table" ) @@ -21,7 +21,7 @@ func init() { core.AddValueType[tensor.Byte, TensorButton]() core.AddValueType[tensor.String, TensorButton]() core.AddValueType[tensor.Bits, TensorButton]() - core.AddValueType[simat.SimMat, SimMatButton]() + // core.AddValueType[simat.SimMat, SimMatButton]() } // TensorButton represents a Tensor with a button for making a [TensorGrid] @@ -62,7 +62,7 @@ func (tb *TableButton) Init() { tb.Updater(func() { text := "None" if tb.Table != nil { - if nm, has := tb.Table.MetaData["name"]; has { + if nm, err := metadata.Get[string](tb.Table.Meta, "name"); err == nil { text = nm } else { text = "Table" @@ -75,6 +75,7 @@ func (tb *TableButton) Init() { }) } +/* // SimMatValue presents a button that pulls up the [SimMatGrid] viewer for a [table.Table]. type SimMatButton struct { core.Button @@ -101,3 +102,4 @@ func (tb *SimMatButton) Init() { NewSimMatGrid(d).SetSimMat(tb.SimMat) }) } +*/ diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-table.go b/yaegicore/symbols/cogentcore_org-core-tensor-table.go index 751886cc5a..211744754a 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-table.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-table.go @@ -5,6 +5,7 @@ package symbols import ( "reflect" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) @@ -19,8 +20,8 @@ func init() { "ConfigFromHeaders": reflect.ValueOf(table.ConfigFromHeaders), "ConfigFromTableHeaders": reflect.ValueOf(table.ConfigFromTableHeaders), "Contains": reflect.ValueOf(table.Contains), - "DelimsN": reflect.ValueOf(table.DelimsN), - "DelimsValues": reflect.ValueOf(table.DelimsValues), + "DelimsN": reflect.ValueOf(tensor.DelimsN), + "DelimsValues": reflect.ValueOf(tensor.DelimsValues), "Descending": reflect.ValueOf(table.Descending), "Detect": reflect.ValueOf(table.Detect), "DetectTableHeaders": reflect.ValueOf(table.DetectTableHeaders), @@ -42,9 +43,9 @@ func init() { "UseCase": reflect.ValueOf(table.UseCase), // type definitions - "Delims": reflect.ValueOf((*table.Delims)(nil)), + "Delims": reflect.ValueOf((*tensor.Delims)(nil)), "Filterer": reflect.ValueOf((*table.Filterer)(nil)), - "Indexed": reflect.ValueOf((*table.Indexed)(nil)), + "Indexed": reflect.ValueOf((*table.Table)(nil)), "LessFunc": reflect.ValueOf((*table.LessFunc)(nil)), "SplitAgg": reflect.ValueOf((*table.SplitAgg)(nil)), "Splits": reflect.ValueOf((*table.Splits)(nil)), From ff58d6b201d5a81acf480457893897c413143a38 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 11:19:02 -0700 Subject: [PATCH 037/311] plot and table editors working with new table -- except indexed view not working in plot. added back column-wise sort and filter methods that wrap the tensor and get indexes back. --- examples/plot/plot.go | 3 +- plot/plotcore/barplot.go | 8 +-- plot/plotcore/metadata.go | 121 +++++++++++++++++++++++++++++++++++ plot/plotcore/options.go | 89 +------------------------- plot/plotcore/ploteditor.go | 16 ++--- plot/plotcore/tablexy.go | 18 +++--- plot/plotcore/xyplot.go | 8 +-- plot/plots/table.go | 2 +- tensor/datafs/dir.go | 34 ++++++++++ tensor/datafs/metadata.go | 57 ----------------- tensor/indexed.go | 56 ++++++---------- tensor/stats/cluster/plot.go | 6 +- tensor/stats/split/agg.go | 4 +- tensor/stats/split/splits.go | 2 +- tensor/table/README.md | 4 +- tensor/table/indexes.go | 81 +++++++++++------------ tensor/table/table.go | 11 +++- tensor/table/table_test.go | 12 +--- tensor/table/typegen.go | 4 +- tensor/table/util.go | 4 +- tensor/tensorcore/table.go | 29 ++++----- 21 files changed, 278 insertions(+), 291 deletions(-) create mode 100644 plot/plotcore/metadata.go diff --git a/examples/plot/plot.go b/examples/plot/plot.go index 226f565e72..b84b97d419 100644 --- a/examples/plot/plot.go +++ b/examples/plot/plot.go @@ -9,6 +9,7 @@ import ( "cogentcore.org/core/core" "cogentcore.org/core/plot/plotcore" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) @@ -19,7 +20,7 @@ func main() { b := core.NewBody("Plot Example") epc := table.NewTable("epc") - epc.OpenFS(tsv, "ra25epoch.tsv", table.Tab) + epc.OpenFS(tsv, "ra25epoch.tsv", tensor.Tab) pl := plotcore.NewPlotEditor(b) pl.Options.Title = "RA25 Epoch Train" diff --git a/plot/plotcore/barplot.go b/plot/plotcore/barplot.go index 786eb10715..2a68987f51 100644 --- a/plot/plotcore/barplot.go +++ b/plot/plotcore/barplot.go @@ -35,7 +35,7 @@ func (pl *PlotEditor) genPlotBar() { // var lsplit *table.Splits nleg := 1 if pl.Options.Legend != "" { - lcol := pl.table.Columns.IndexByKey(pl.Options.Legend) + lcol := pl.table.ColumnIndex(pl.Options.Legend) if lcol < 0 { log.Println("plot.Legend not found: " + pl.Options.Legend) } else { @@ -125,7 +125,7 @@ func (pl *PlotEditor) genPlotBar() { } ec := -1 if cp.ErrColumn != "" { - ec = pl.table.Columns.IndexByKey(cp.ErrColumn) + ec = pl.table.ColumnIndex(cp.ErrColumn) } var bar *plots.BarChart if ec >= 0 { @@ -163,7 +163,7 @@ func (pl *PlotEditor) genPlotBar() { n := xview.NumRows() for _, cp := range strCols { xy, _ := newTableXY(xview, xi, xp.TensorIndex, firstXY.yColumn, cp.TensorIndex, firstXY.yRange) - xy.labelColumn = xview.Columns.IndexByKey(cp.Column) + xy.labelColumn = xview.ColumnIndex(cp.Column) xy.yIndex = firstXY.yIndex xyl := plots.XYLabels{} @@ -184,7 +184,7 @@ func (pl *PlotEditor) genPlotBar() { } netn := pl.table.NumRows() * stride - xc := pl.table.ColumnIndex(xi) + xc := pl.table.ColumnByIndex(xi) vals := make([]string, netn) for i, dx := range pl.table.Indexes { pi := mid + i*stride diff --git a/plot/plotcore/metadata.go b/plot/plotcore/metadata.go new file mode 100644 index 0000000000..72585311bc --- /dev/null +++ b/plot/plotcore/metadata.go @@ -0,0 +1,121 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package plotcore + +import ( + "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/metadata" + "cogentcore.org/core/tensor/table" +) + +// todo: make a more systematic version driven by reflect fields +// key issue is the same nullable type issue: you don't want to set everything +// in md. + +// also the Column name and IsString should not be in the struct! + +// fromMeta sets plot options from meta data. +func (po *PlotOptions) fromMeta(dt *table.Table) { + po.FromMetaMap(dt.Meta) +} + +// FromMetaMap sets plot options from meta data map. +func (po *PlotOptions) FromMetaMap(meta metadata.Data) { + if typ, err := metadata.Get[string](meta, "Type"); err == nil { + po.Type.SetString(typ) + } + if op, err := metadata.Get[bool](meta, "Lines"); err == nil { + po.Lines = op + } + if op, err := metadata.Get[bool](meta, "Points"); err == nil { + po.Points = op + } + if lw, err := metadata.Get[float64](meta, "LineWidth"); err == nil { + po.LineWidth = float32(lw) + } + if ps, err := metadata.Get[float64](meta, "PointSize"); err == nil { + po.PointSize = float32(ps) + } + if bw, err := metadata.Get[float64](meta, "BarWidth"); err == nil { + po.BarWidth = float32(bw) + } + if op, err := metadata.Get[bool](meta, "NegativeXDraw"); err == nil { + po.NegativeXDraw = op + } + if scl, err := metadata.Get[float64](meta, "Scale"); err == nil { + po.Scale = float32(scl) + } + if xc, err := metadata.Get[string](meta, "XAxis"); err == nil { + po.XAxis = xc + } + if lc, err := metadata.Get[string](meta, "Legend"); err == nil { + po.Legend = lc + } + if xrot, err := metadata.Get[float64](meta, "XAxisRotation"); err == nil { + po.XAxisRotation = float32(xrot) + } + if lb, err := metadata.Get[string](meta, "XAxisLabel"); err == nil { + po.XAxisLabel = lb + } + if lb, err := metadata.Get[string](meta, "YAxisLabel"); err == nil { + po.YAxisLabel = lb + } +} + +// fromMetaMap sets column options from meta data map. +func (co *ColumnOptions) fromMetaMap(meta metadata.Data) { + if op, err := metadata.Get[bool](meta, co.Column+":On"); err == nil { + co.On = op + } + if op, err := metadata.Get[bool](meta, co.Column+":Off"); err == nil { + co.On = op + } + if op, err := metadata.Get[bool](meta, co.Column+":FixMin"); err == nil { + co.Range.FixMin = op + } + if op, err := metadata.Get[bool](meta, co.Column+":FixMax"); err == nil { + co.Range.FixMax = op + } + if op, err := metadata.Get[bool](meta, co.Column+":FloatMin"); err == nil { + co.Range.FixMin = op + } + if op, err := metadata.Get[bool](meta, co.Column+":FloatMax"); err == nil { + co.Range.FixMax = op + } + if vl, err := metadata.Get[float64](meta, co.Column+":Max"); err == nil { + co.Range.Max = float32(vl) + } + if vl, err := metadata.Get[float64](meta, co.Column+":Min"); err == nil { + co.Range.Min = float32(vl) + } + if lb, err := metadata.Get[string](meta, co.Column+":Label"); err == nil { + co.Label = lb + } + if lb, err := metadata.Get[string](meta, co.Column+":ErrColumn"); err == nil { + co.ErrColumn = lb + } + if vl, err := metadata.Get[int](meta, co.Column+":TensorIndex"); err == nil { + co.TensorIndex = vl + } +} + +// PlotColumnZeroOne returns plot options with a fixed 0-1 range +func PlotColumnZeroOne() *ColumnOptions { + opts := &ColumnOptions{} + opts.Range.SetMin(0) + opts.Range.SetMax(1) + return opts +} + +// SetPlotColumnOptions sets given plotting options for named items +// within this directory, in Metadata. +func SetPlotColumnOptions(md metadata.Data, opts *ColumnOptions) { + md.Set("PlotColumnOptions", opts) +} + +// PlotColumnOptions returns plotting options if they have been set, else nil. +func PlotColumnOptions(md metadata.Data) *ColumnOptions { + return errors.Ignore1(metadata.Get[*ColumnOptions](md, "PlotColumnOptions")) +} diff --git a/plot/plotcore/options.go b/plot/plotcore/options.go index c14e8a68eb..66f798fed9 100644 --- a/plot/plotcore/options.go +++ b/plot/plotcore/options.go @@ -7,12 +7,10 @@ package plotcore import ( "image" - "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/option" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" "cogentcore.org/core/plot/plots" - "cogentcore.org/core/tensor/table" ) // PlotOptions are options for the overall plot. @@ -92,60 +90,12 @@ func (po *PlotOptions) defaults() { } } -// fromMeta sets plot options from meta data. -func (po *PlotOptions) fromMeta(dt *table.Table) { - po.FromMetaMap(dt.Meta) -} - -// FromMetaMap sets plot options from meta data map. -func (po *PlotOptions) FromMetaMap(meta metadata.Data) { - if typ, err := metadata.Get[string](meta, "Type"); err == nil { - po.Type.SetString(typ) - } - if op, err := metadata.Get[bool](meta, "Lines"); err == nil { - po.Lines = op - } - if op, err := metadata.Get[bool](meta, "Points"); err == nil { - po.Points = op - } - if lw, err := metadata.Get[float64](meta, "LineWidth"); err == nil { - po.LineWidth = float32(lw) - } - if ps, err := metadata.Get[float64](meta, "PointSize"); err == nil { - po.PointSize = float32(ps) - } - if bw, err := metadata.Get[float64](meta, "BarWidth"); err == nil { - po.BarWidth = float32(bw) - } - if op, err := metadata.Get[bool](meta, "NegativeXDraw"); err == nil { - po.NegativeXDraw = op - } - if scl, err := metadata.Get[float64](meta, "Scale"); err == nil { - po.Scale = float32(scl) - } - if xc, err := metadata.Get[string](meta, "XAxis"); err == nil { - po.XAxis = xc - } - if lc, err := metadata.Get[string](meta, "Legend"); err == nil { - po.Legend = lc - } - if xrot, err := metadata.Get[float64](meta, "XAxisRotation"); err == nil { - po.XAxisRotation = float32(xrot) - } - if lb, err := metadata.Get[string](meta, "XAxisLabel"); err == nil { - po.XAxisLabel = lb - } - if lb, err := metadata.Get[string](meta, "YAxisLabel"); err == nil { - po.YAxisLabel = lb - } -} - // ColumnOptions are options for plotting one column of data. type ColumnOptions struct { //types:add // whether to plot this column On bool - // name of column being plotting + // name of column being plotted Column string // whether to plot lines; uses the overall plot option if unset @@ -203,43 +153,6 @@ func (co *ColumnOptions) getLabel() string { return co.Column } -// fromMetaMap sets column options from meta data map. -func (co *ColumnOptions) fromMetaMap(meta metadata.Data) { - if op, err := metadata.Get[bool](meta, co.Column+":On"); err == nil { - co.On = op - } - if op, err := metadata.Get[bool](meta, co.Column+":Off"); err == nil { - co.On = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FixMin"); err == nil { - co.Range.FixMin = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FixMax"); err == nil { - co.Range.FixMax = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FloatMin"); err == nil { - co.Range.FixMin = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FloatMax"); err == nil { - co.Range.FixMax = op - } - if vl, err := metadata.Get[float64](meta, co.Column+":Max"); err == nil { - co.Range.Max = float32(vl) - } - if vl, err := metadata.Get[float64](meta, co.Column+":Min"); err == nil { - co.Range.Min = float32(vl) - } - if lb, err := metadata.Get[string](meta, co.Column+":Label"); err == nil { - co.Label = lb - } - if lb, err := metadata.Get[string](meta, co.Column+":ErrColumn"); err == nil { - co.ErrColumn = lb - } - if vl, err := metadata.Get[int](meta, co.Column+":TensorIndex"); err == nil { - co.TensorIndex = vl - } -} - // PlotTypes are different types of plots. type PlotTypes int32 //enums:enum diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index e7e37dc036..0d68d7fb2d 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -343,7 +343,7 @@ func (pl *PlotEditor) genPlot() { pl.plotWidget.Scale = pl.Options.Scale pl.plotWidget.SetRangesFunc = func() { plt := pl.plotWidget.Plot - xi := pl.table.Columns.IndexByKey(pl.Options.XAxis) + xi := pl.table.ColumnIndex(pl.Options.XAxis) if xi >= 0 { xp := pl.Columns[xi] if xp.Range.FixMin { @@ -380,13 +380,13 @@ func (pl *PlotEditor) configPlot(plt *plot.Plot) { // plotXAxis processes the XAxis and returns its index func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.Table) (xi int, xview *table.Table, err error) { - xi = ixvw.Columns.IndexByKey(pl.Options.XAxis) + xi = ixvw.ColumnIndex(pl.Options.XAxis) if xi < 0 { // log.Println("plot.PlotXAxis: " + err.Error()) return } xview = ixvw - xc := ixvw.ColumnIndex(xi) + xc := ixvw.ColumnByIndex(xi) xp := pl.Columns[xi] sz := 1 if xp.Range.FixMin { @@ -428,7 +428,7 @@ func (pl *PlotEditor) columnsListUpdate() { } cp := &ColumnOptions{Column: cn} cp.defaults() - tcol := dt.ColumnIndex(ci) + tcol := dt.ColumnByIndex(ci) tc := tcol.Tensor if tc.IsString() { cp.IsString = true @@ -655,10 +655,10 @@ func (pl *PlotEditor) MakeToolbar(p *tree.Plan) { w.SetFunc(pl.OpenCSV).SetIcon(icons.Open) }) tree.Add(p, func(w *core.Separator) {}) - // tree.Add(p, func(w *core.FuncButton) { // TODO - // w.SetFunc(pl.table.FilterColumnName).SetText("Filter").SetIcon(icons.FilterAlt) - // w.SetAfterFunc(pl.UpdatePlot) - // }) + tree.Add(p, func(w *core.FuncButton) { + w.SetFunc(pl.table.FilterString).SetText("Filter").SetIcon(icons.FilterAlt) + w.SetAfterFunc(pl.UpdatePlot) + }) tree.Add(p, func(w *core.FuncButton) { w.SetFunc(pl.table.Sequential).SetText("Unfilter").SetIcon(icons.FilterAltOff) w.SetAfterFunc(pl.UpdatePlot) diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index 071fe23b53..da94a4e0e2 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -59,7 +59,7 @@ func newTableXY(dt *table.Table, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minm // from given column name and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. func newTableXYName(dt *table.Table, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { - yi := dt.Columns.IndexByKey(ycol) + yi := dt.ColumnIndex(ycol) if yi < 0 { return nil, nil // todo: err } @@ -82,8 +82,8 @@ func (txy *tableXY) validate() error { txy.yColumn = 0 return errors.New("eplot.TableXY YColumn index invalid -- reset to 0") } - xc := txy.table.ColumnIndex(txy.xColumn) - yc := txy.table.ColumnIndex(txy.yColumn) + xc := txy.table.ColumnByIndex(txy.xColumn) + yc := txy.table.ColumnByIndex(txy.yColumn) if xc.Tensor.NumDims() > 1 { _, txy.xRowSize = xc.RowCellSize() // note: index already validated @@ -127,7 +127,7 @@ func (txy *tableXY) Len() int { // tRowValue returns the y value at given true table row in table func (txy *tableXY) tRowValue(row int) float32 { - yc := txy.table.ColumnIndex(txy.yColumn) + yc := txy.table.ColumnByIndex(txy.yColumn) y := float32(0.0) switch { case yc.Tensor.IsString(): @@ -149,7 +149,7 @@ func (txy *tableXY) Value(row int) float32 { return 0 } trow := txy.table.Indexes[row] // true table row - yc := txy.table.ColumnIndex(txy.yColumn) + yc := txy.table.ColumnByIndex(txy.yColumn) y := float32(0.0) switch { case yc.Tensor.IsString(): @@ -170,7 +170,7 @@ func (txy *tableXY) tRowXValue(row int) float32 { if txy.table == nil || txy.table == nil { return 0 } - xc := txy.table.ColumnIndex(txy.xColumn) + xc := txy.table.ColumnByIndex(txy.xColumn) x := float32(0.0) switch { case xc.Tensor.IsString(): @@ -192,7 +192,7 @@ func (txy *tableXY) xValue(row int) float32 { return 0 } trow := txy.table.Indexes[row] // true table row - xc := txy.table.ColumnIndex(txy.xColumn) + xc := txy.table.ColumnByIndex(txy.xColumn) x := float32(0.0) switch { case xc.Tensor.IsString(): @@ -224,7 +224,7 @@ func (txy *tableXY) Label(row int) string { return "" } trow := txy.table.Indexes[row] // true table row - return txy.table.ColumnIndex(txy.labelColumn).String1D(trow) + return txy.table.ColumnByIndex(txy.labelColumn).String1D(trow) } // YError returns error bars, implementing [plots.YErrorer] interface. @@ -233,7 +233,7 @@ func (txy *tableXY) YError(row int) (float32, float32) { return 0, 0 } trow := txy.table.Indexes[row] // true table row - ec := txy.table.ColumnIndex(txy.errColumn) + ec := txy.table.ColumnByIndex(txy.errColumn) eval := float32(0.0) switch { case ec.Tensor.IsString(): diff --git a/plot/plotcore/xyplot.go b/plot/plotcore/xyplot.go index 622c7794e1..7d99de5334 100644 --- a/plot/plotcore/xyplot.go +++ b/plot/plotcore/xyplot.go @@ -28,7 +28,7 @@ func (pl *PlotEditor) genPlotXY() { // var lsplit *table.Splits nleg := 1 if pl.Options.Legend != "" { - ix := pl.table.Columns.IndexByKey(pl.Options.Legend) + ix := pl.table.ColumnIndex(pl.Options.Legend) if ix < 0 { slog.Error("plot.Legend", "err", err.Error()) } else { @@ -140,7 +140,7 @@ func (pl *PlotEditor) genPlotXY() { } } if cp.ErrColumn != "" { - ec := pl.table.Columns.IndexByKey(cp.ErrColumn) + ec := pl.table.ColumnIndex(cp.ErrColumn) if ec >= 0 { xy.errColumn = ec eb, _ := plots.NewYErrorBars(xy) @@ -155,7 +155,7 @@ func (pl *PlotEditor) genPlotXY() { if firstXY != nil && len(strCols) > 0 { for _, cp := range strCols { xy, _ := newTableXY(xview, xi, xp.TensorIndex, firstXY.yColumn, cp.TensorIndex, firstXY.yRange) - xy.labelColumn = xview.Columns.IndexByKey(cp.Column) + xy.labelColumn = xview.ColumnIndex(cp.Column) xy.yIndex = firstXY.yIndex lbls, _ := plots.NewLabels(xy) if lbls != nil { @@ -165,7 +165,7 @@ func (pl *PlotEditor) genPlotXY() { } // Use string labels for X axis if X is a string - xc := pl.table.ColumnIndex(xi) + xc := pl.table.ColumnByIndex(xi) if xc.Tensor.IsString() { xcs := xc.Tensor.(*tensor.String) vals := make([]string, pl.table.NumRows()) diff --git a/plot/plots/table.go b/plot/plots/table.go index 41644ce7a6..484b66a534 100644 --- a/plot/plots/table.go +++ b/plot/plots/table.go @@ -22,7 +22,7 @@ type Table interface { PlotData(column, row int) float32 } -func TableColumnIndex(tab Table, name string) int { +func TableColumnByIndex(tab Table, name string) int { for i := range tab.NumColumns() { if tab.ColumnName(i) == name { return i diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 7b0c2903df..a1c28730ad 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -12,8 +12,10 @@ import ( "slices" "sort" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/keylist" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" ) // Dir is a map of directory entry names to Data nodes. @@ -286,3 +288,35 @@ func (d *Data) Mkdir(name string) (*Data, error) { } return NewDir(name, d) } + +// GetDirTable gets the DirTable as a [table.Table] for this directory item, +// with columns as the Tensor values elements in the directory +// and any subdirectories, from FlatValuesFunc using given filter function. +// This is a convenient mechanism for creating a plot of all the data +// in a given directory. +// If such was previously constructed, it is returned from "DirTable" +// where it is stored for later use. +// Row count is updated to current max row. +// Set DirTable = nil to regenerate. +func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { + if d.DirTable != nil { + d.DirTable.SetNumRowsToMax() + return dt + } + tsrs := d.FlatValuesFunc(fun) + dt := table.NewTable(fsx.DirAndFile(string(d.Path()))) + for _, tsr := range tsrs { + rows := tsr.Tensor.Rows() + if dt.Columns.Rows < rows { + dt.Columns.Rows = rows + dt.SetNumRows(dt.Columns.Rows) + } + nm := it.Name() + if it.Parent != d { + nm = fsx.DirAndFile(string(it.Path())) + } + dt.AddColumn(tsr.Tensor, nm) + } + d.DirTable = dt + return dt +} diff --git a/tensor/datafs/metadata.go b/tensor/datafs/metadata.go index 548fd40015..3b8ab180fb 100644 --- a/tensor/datafs/metadata.go +++ b/tensor/datafs/metadata.go @@ -6,10 +6,7 @@ package datafs import ( "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/metadata" - "cogentcore.org/core/plot/plotcore" - "cogentcore.org/core/tensor/table" ) // This file provides standardized metadata options for frequent @@ -25,28 +22,6 @@ func (d *Data) SetMetaItems(key string, value any, names ...string) error { return err } -// PlotColumnZeroOne returns plot options with a fixed 0-1 range -func PlotColumnZeroOne() *plotcore.ColumnOptions { - opts := &plotcore.ColumnOptions{} - opts.Range.SetMin(0) - opts.Range.SetMax(1) - return opts -} - -// SetPlotColumnOptions sets given plotting options for named items -// within this directory (stored in Metadata). -func (d *Data) SetPlotColumnOptions(opts *plotcore.ColumnOptions, names ...string) error { - return d.SetMetaItems("PlotColumnOptions", opts, names...) -} - -// PlotColumnOptions returns plotting options if they have been set, else nil. -func (d *Data) PlotColumnOptions() *plotcore.ColumnOptions { - if d.Value == nil { - return - } - return errors.Ignore1(metadata.Get[*plotcore.ColumnOptions](d.Value.Tensor.Meta, "PlotColumnOptions")) -} - // SetCalcFunc sets a function to compute an updated Value for this Value item. // Function is stored as CalcFunc in Metadata. Can be called by [Data.Calc] method. func (d *Data) SetCalcFunc(fun func() error) { @@ -84,35 +59,3 @@ func (d *Data) CalcAll() error { } return errors.Join(errs...) } - -// GetDirTable gets the DirTable as a [table.Table] for this directory item, -// with columns as the Tensor values elements in the directory -// and any subdirectories, from FlatValuesFunc using given filter function. -// This is a convenient mechanism for creating a plot of all the data -// in a given directory. -// If such was previously constructed, it is returned from "DirTable" -// where it is stored for later use. -// Row count is updated to current max row. -// Set DirTable = nil to regenerate. -func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { - if d.DirTable != nil { - d.DirTable.SetNumRowsToMax() - return dt - } - tsrs := d.FlatValuesFunc(fun) - dt := table.NewTable(fsx.DirAndFile(string(d.Path()))) - for _, tsr := range tsrs { - rows := tsr.Tensor.Rows() - if dt.Columns.Rows < rows { - dt.Columns.Rows = rows - dt.SetNumRows(dt.Columns.Rows) - } - nm := it.Name() - if it.Parent != d { - nm = fsx.DirAndFile(string(it.Path())) - } - dt.AddColumn(tsr.Tensor, nm) - } - d.DirTable = dt - return dt -} diff --git a/tensor/indexed.go b/tensor/indexed.go index b0454a3673..42ae460d1b 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -6,7 +6,6 @@ package tensor import ( "cmp" - "errors" "math" "math/rand" "slices" @@ -179,20 +178,17 @@ func (ix *Indexed) IndexesNeeded() { //types:add } } -// ExcludeMissing1D deletes indexes for a 1D tensor (only) where -// the values are missing, as indicated by NaN. -func (ix *Indexed) ExcludeMissing1D() { //types:add +// ExcludeMissing deletes indexes where the values are missing, as indicated by NaN. +// Uses first cell of higher dimensional data. +func (ix *Indexed) ExcludeMissing() { //types:add if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return } - if ix.Tensor.NumDims() > 1 { - return - } ix.IndexesNeeded() ni := ix.NumRows() for i := ni - 1; i >= 0; i-- { - if math.IsNaN(ix.Float1D(ix.Indexes[i])) { + if math.IsNaN(ix.Tensor.FloatRowCell(ix.Indexes[i], 0)) { ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) } } @@ -223,21 +219,16 @@ const ( Descending = false ) -// SortFunc sorts the indexes into 1D Tensor using given compare function. -// Returns an error if called on a higher-dimensional tensor. +// SortFunc sorts the row-wise indexes using given compare function. // The compare function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. // cmp(a, b) should return a negative number when a < b, a positive // number when a > b and zero when a == b. -func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) error { - if ix.Tensor.NumDims() > 1 { - return errors.New("tensor Sorting is only for 1D tensors") - } +func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) { ix.IndexesNeeded() slices.SortFunc(ix.Indexes, func(a, b int) int { return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! }) - return nil } // SortIndexes sorts the indexes into our Tensor directly in @@ -257,53 +248,46 @@ func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { return cmp.Compare(b, a) } -// Sort does default alpha or numeric sort of 1D tensor based on data type. -// Returns an error if called on a higher-dimensional tensor. -func (ix *Indexed) Sort(ascending bool) error { +// Sort does default alpha or numeric sort of row-wise data. +// Uses first cell of higher dimensional data. +func (ix *Indexed) Sort(ascending bool) { if ix.Tensor.IsString() { ix.SortFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.String1D(i), tsr.String1D(j), ascending) + return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) }) } else { ix.SortFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.Float1D(i), tsr.Float1D(j), ascending) + return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) }) } - return nil } -// SortStableFunc stably sorts the indexes of 1D Tensor using given compare function. +// SortStableFunc stably sorts the row-wise indexes using given compare function. // The compare function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. // cmp(a, b) should return a negative number when a < b, a positive // number when a > b and zero when a == b. // It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) error { - if ix.Tensor.NumDims() > 1 { - return errors.New("tensor Sorting is only for 1D tensors") - } +func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { ix.IndexesNeeded() slices.SortStableFunc(ix.Indexes, func(a, b int) int { return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! }) - return nil } -// SortStable does default alpha or numeric stable sort -// of 1D tensor based on data type. -// Returns an error if called on a higher-dimensional tensor. -func (ix *Indexed) SortStable(ascending bool) error { +// SortStable does stable default alpha or numeric sort. +// Uses first cell of higher dimensional data. +func (ix *Indexed) SortStable(ascending bool) { if ix.Tensor.IsString() { ix.SortStableFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.String1D(i), tsr.String1D(j), ascending) + return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) }) } else { ix.SortStableFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.Float1D(i), tsr.Float1D(j), ascending) + return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) }) } - return nil } // FilterFunc is a function used for filtering that returns @@ -324,7 +308,7 @@ func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { } } -// Named arg values for Contains, IgnoreCase +// Named arg values for FilterString const ( // Include means include matches Include = false @@ -345,7 +329,7 @@ const ( // If contains, only checks if row contains string; if ignoreCase, ignores case. // Use the named const args [Include], [Exclude], [Contains], [Equals], // [IgnoreCase], [UseCase] for greater clarity. -// Only valid for 1-dimensional columns. +// Uses first cell of higher dimensional data. func (ix *Indexed) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add lowstr := strings.ToLower(str) ix.Filter(func(tsr Tensor, row int) bool { diff --git a/tensor/stats/cluster/plot.go b/tensor/stats/cluster/plot.go index 3c829ac0d3..e269b1ea9d 100644 --- a/tensor/stats/cluster/plot.go +++ b/tensor/stats/cluster/plot.go @@ -28,9 +28,9 @@ func Plot(pt *table.Table, root *Node, dmat, labels *tensor.Indexed) { // The lines double-back on themselves to form a continuous line to be plotted. func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { row := pt.NumRows() - xc := pt.ColumnIndex(0) - yc := pt.ColumnIndex(1) - lbl := pt.ColumnIndex(2) + xc := pt.ColumnByIndex(0) + yc := pt.ColumnByIndex(1) + lbl := pt.ColumnByIndex(2) if nn.IsLeaf() { pt.SetNumRows(row + 1) xc.SetFloatRowCell(nn.ParDist, row, 0) diff --git a/tensor/stats/split/agg.go b/tensor/stats/split/agg.go index cea87595be..a27de3436b 100644 --- a/tensor/stats/split/agg.go +++ b/tensor/stats/split/agg.go @@ -31,7 +31,7 @@ func AggColumn(spl *table.Splits, column string, stat stats.Stats) (*table.Split if dt == nil { return nil, fmt.Errorf("split.AggTry: No splits to aggregate over") } - colIndex, err := dt.ColumnIndex(column) + colIndex, err := dt.ColumnByIndex(column) if err != nil { return nil, err } @@ -78,7 +78,7 @@ func DescColumn(spl *table.Splits, column string) error { if dt == nil { return fmt.Errorf("split.DescTry: No splits to aggregate over") } - colIndex, err := dt.ColumnIndex(column) + colIndex, err := dt.ColumnByIndex(column) if err != nil { return err } diff --git a/tensor/stats/split/splits.go b/tensor/stats/split/splits.go index 2893f42a7d..56f5743c16 100644 --- a/tensor/stats/split/splits.go +++ b/tensor/stats/split/splits.go @@ -345,7 +345,7 @@ func (spl *Splits) AggByColumnName(name string) (*SplitAgg, error) { return nil, fmt.Errorf("table.Splits AggByColumnName: table nil") } nmsp := strings.Split(name, ":") - colIndex := dt.Columns.IndexByKey(nmsp[0]) + colIndex := dt.ColumnIndex(nmsp[0]) if colIndex < 0 { return nil, nil } diff --git a/tensor/table/README.md b/tensor/table/README.md index 1caa758562..dc22378801 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -12,7 +12,7 @@ The `Table` mainly provides "infrastructure" methods for adding tensor columns a As a general convention, it is safest, clearest, and quite fast to access columns by name instead of index (there is a `map` from name to index), so the base access method names generally take a column name argument, and those that take a column index have an `Index` suffix. -The table itself stores raw data `tensor.Tensor` values, and the `Column` (by name) and `ColumnIndex` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). +The table itself stores raw data `tensor.Tensor` values, and the `Column` (by name) and `ColumnByIndex` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). If you call Sort, Filter or other routines on an individual column tensor, then you can grab the updated indexes via the `IndexesFromTensor` method so that they apply to the entire table. @@ -29,7 +29,7 @@ It is very low-cost to create a new View of an existing Table, via `NewView`, as Column data access: ```Go -// FloatRowCell is method on the `tensor.Indexed` returned from `Column` method. +// FloatRowCell is a method on the `tensor.Indexed` returned from the `Column` method. // This is the best method to use in general for generic 1D data access, // as it works on any data from 1D on up (although it only samples the first value // from higher dimensional data) . diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 4a27b03c97..0b2ab87b6d 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -8,7 +8,6 @@ import ( "math/rand" "slices" "sort" - "strings" "cogentcore.org/core/tensor" ) @@ -89,6 +88,22 @@ func (dt *Table) Permuted() { } } +// SortColumn sorts the indexes into our Table according to values in +// given column, using either ascending or descending order, +// (use [tensor.Ascending] or [tensor.Descending] for self-documentation). +// Uses first cell of higher dimensional data. +// Returns error if column name not found. +func (dt *Table) SortColumn(columnName string, ascending bool) error { //types:add + dt.IndexesNeeded() + cl, err := dt.ColumnTry(columnName) + if err != nil { + return err + } + cl.Sort(ascending) + dt.IndexesFromTensor(cl) + return nil +} + // SortFunc sorts the indexes into our Table using given compare function. // The compare function operates directly on row numbers into the Table // as these row numbers have already been projected through the indexes. @@ -119,13 +134,12 @@ func (dt *Table) SortStableFunc(cmp func(dt *Table, i, j int) int) { // given column names, using either ascending or descending order, // (use [tensor.Ascending] or [tensor.Descending] for self-documentation, // and optionally using a stable sort. -// Only valid for 1-dimensional columns. -// Returns error if column name not found. -func (dt *Table) SortColumns(ascending, stable bool, columns ...string) { +// Uses first cell of higher dimensional data. +func (dt *Table) SortColumns(ascending, stable bool, columns ...string) { //types:add nc := len(columns) cis := make([]int, 0, nc) for _, cn := range columns { - ci := dt.Columns.IndexByKey(cn) + ci := dt.ColumnIndex(cn) if ci >= 0 { cis = append(cis, ci) } @@ -135,7 +149,7 @@ func (dt *Table) SortColumns(ascending, stable bool, columns ...string) { // SortColumnIndexes sorts the indexes into our Table according to values in // given list of column indexes, using either ascending or descending order for -// all of the columns. Only valid for 1-dimensional columns. +// all of the columns. Uses first cell of higher dimensional data. func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { dt.IndexesNeeded() sf := dt.SortFunc @@ -144,7 +158,7 @@ func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { } sf(func(dt *Table, i, j int) int { for _, ci := range colIndexes { - cl := dt.ColumnIndex(ci).Tensor + cl := dt.ColumnByIndex(ci).Tensor if cl.IsString() { v := tensor.CompareAscending(cl.StringRowCell(i, 0), cl.StringRowCell(j, 0), ascending) if v != 0 { @@ -189,6 +203,24 @@ func (dt *Table) Filter(filterer func(dt *Table, row int) bool) { } } +// FilterString filters the indexes using string values in column compared to given +// string. Includes rows with matching values unless exclude is set. +// If contains, only checks if row contains string; if ignoreCase, ignores case. +// Use the named const args [tensor.Include], [tensor.Exclude], [tensor.Contains], +// [tensor.Equals], [tensor.IgnoreCase], [tensor.UseCase] for greater clarity. +// Uses first cell from higher dimensions. +// Returns error if column name not found. +func (dt *Table) FilterString(columnName string, str string, exclude, contains, ignoreCase bool) error { //types:add + dt.IndexesNeeded() + cl, err := dt.ColumnTry(columnName) + if err != nil { + return err + } + cl.FilterString(str, exclude, contains, ignoreCase) + dt.IndexesFromTensor(cl) + return nil +} + // NewTable returns a new table with column data organized according to // the indexes. If Indexes are nil, a clone of the current tensor is returned // but this function is only sensible if there is an indexed view in place. @@ -221,41 +253,6 @@ func (dt *Table) DeleteRows(at, n int) { dt.Indexes = append(dt.Indexes[:at], dt.Indexes[at+n:]...) } -// RowsByString returns the list of row _indexes_ (not necessarily underlying row numbers, -// if Indexes are in place) whose row in the table has given string value in given column name. -// The results can be used as row indexes to Indexed tensor column data. -// If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use the named const args [tensor.Contains], [tensor.Equals], [tensor.IgnoreCase], -// [tensor.UseCase] for greater clarity. -func (dt *Table) RowsByString(colname string, str string, contains, ignoreCase bool) []int { - col := dt.Column(colname) - if col == nil { - return nil - } - lowstr := strings.ToLower(str) - var indexes []int - rows := dt.NumRows() - for idx := range rows { - srw := dt.Index(idx) - val := col.Tensor.StringRowCell(srw, 0) - has := false - switch { - case contains && ignoreCase: - has = strings.Contains(strings.ToLower(val), lowstr) - case contains: - has = strings.Contains(val, str) - case ignoreCase: - has = strings.EqualFold(val, str) - default: - has = (val == str) - } - if has { - indexes = append(indexes, idx) - } - } - return indexes -} - // Swap switches the indexes for i and j func (dt *Table) Swap(i, j int) { dt.Indexes[i], dt.Indexes[j] = dt.Indexes[j], dt.Indexes[i] diff --git a/tensor/table/table.go b/tensor/table/table.go index 0311e9df9b..a276ee6d1f 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -100,16 +100,21 @@ func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { // It is best practice to instead access columns by name using [Table.Column]. // Direct access through [Table.Columns} does not provide the shared table-wide Indexes. // Will panic if out of range. -func (dt *Table) ColumnIndex(idx int) *tensor.Indexed { +func (dt *Table) ColumnByIndex(idx int) *tensor.Indexed { cl := dt.Columns.Values[idx] return tensor.NewIndexed(cl, dt.Indexes) } -// ColumnName returns the name of given column +// ColumnName returns the name of given column. func (dt *Table) ColumnName(i int) string { return dt.Columns.Keys[i] } +// ColumnIndex returns the index for given column name. +func (dt *Table) ColumnIndex(name string) int { + return dt.Columns.IndexByKey(name) +} + // AddColumn adds a new column to the table, of given type and column name // (which must be unique). If no cellSizes are specified, it holds scalar values, // otherwise the cells are n-dimensional tensors of given size. @@ -198,7 +203,7 @@ func (dt *Table) DeleteColumnName(name string) bool { } // DeleteColumnIndex deletes column within the index range [i:j]. -func (dt *Table) DeleteColumnIndex(i, j int) { +func (dt *Table) DeleteColumnByIndex(i, j int) { dt.Columns.DeleteIndex(i, j) } diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index f6e318836d..2320c9867f 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -74,14 +74,8 @@ func TestAppendRowsEtc(t *testing.T) { assert.Equal(t, sf, df) } } - ixs := dt.RowsByString("Str", "1", tensor.Equals, tensor.UseCase) - assert.Equal(t, []int{1, 4, 7, 10}, ixs) - dt.Sequential() - dt.IndexesNeeded() - ic := dt.Column("Int") - ic.Sort(tensor.Descending) - dt.IndexesFromTensor(ic) + dt.SortColumn("Int", tensor.Descending) assert.Equal(t, []int{2, 5, 8, 11, 1, 4, 7, 10, 0, 3, 6, 9}, dt.Indexes) dt.Sequential() @@ -89,9 +83,7 @@ func TestAppendRowsEtc(t *testing.T) { assert.Equal(t, []int{2, 5, 8, 11, 1, 4, 7, 10, 0, 3, 6, 9}, dt.Indexes) dt.Sequential() - ic = dt.Column("Int") // note: important to re-get with current indexes - ic.FilterString("1", tensor.Include, tensor.Contains, tensor.IgnoreCase) - dt.IndexesFromTensor(ic) + dt.FilterString("Int", "1", tensor.Include, tensor.Contains, tensor.IgnoreCase) assert.Equal(t, []int{1, 4, 7, 10}, dt.Indexes) dt.Sequential() diff --git a/tensor/table/typegen.go b/tensor/table/typegen.go index bedd6f2677..fa644e2e9e 100644 --- a/tensor/table/typegen.go +++ b/tensor/table/typegen.go @@ -6,6 +6,4 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a table.Table that provides a\nspecific view onto the Table defined by the set of indexes.\nThis provides an efficient way of sorting and filtering a table by only\nupdating the indexes while doing nothing to the Table itself.\nTo produce a table that has data actually organized according to the\nindexed order, call the NewTable method.\nIndexed views on a table can also be organized together as Splits\nof the table rows, e.g., by grouping values along a given column.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets indexes to sequential row-wise indexes into table", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "IndexedColumnName", Doc: "IndexedColumnName returns a tensor.Indexed view of given column name,\nusing our current indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column"}, Returns: []string{"Indexed", "error"}}, {Name: "SortColumnName", Doc: "SortColumnName sorts the indexes into our Table according to values in\ngiven column name, using either ascending or descending order.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "ascending"}, Returns: []string{"error"}}, {Name: "FilterColumnName", Doc: "FilterColumnName filters the indexes into our Table according to values in\ngiven column name, using string representation of column values.\nIncludes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse named args for greater clarity.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"column", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table index view to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table idx view from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}}, Fields: []types.Field{{Name: "Table", Doc: "Table that we are an indexed view onto"}, {Name: "Indexes", Doc: "current indexes into Table"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of data, with columns of tensors,\neach with the same number of Rows (outermost dimension).", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to each of the columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns\nif rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "columns of data, as tensor.Tensor tensors"}, {Name: "ColumnNames", Doc: "the names of the columns"}, {Name: "Rows", Doc: "number of rows, which is enforced to be the size of the outermost dimension of the column tensors"}, {Name: "ColumnNameMap", Doc: "the map of column names to column numbers"}, {Name: "MetaData", Doc: "misc meta data for the table. We use lower-case key names following the struct tag convention: name = name of table; desc = description; read-only = gui is read-only; precision = n for precision to write out floats in csv. For Column-specific data, we look for ColumnName: prefix, specifically ColumnName:desc = description of the column contents, which is shown as tooltip in the tensorcore.Table, and :width for width of a column"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of Tensor columns aligned by a common outermost row dimension.\nUse the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a\n[tensor.Indexed] view of the column, using the shared [Table.Indexes] of the Table.\nThus, a coordinated sorting and filtered view of the column data is automatically\navailable for any of the tensor package functions that use [tensor.Indexed] as the one\ncommon data representation for all operations.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumns", Doc: "SortColumns sorts the indexes into our Table according to values in\ngiven column names, using either ascending or descending order,\n(use [tensor.Ascending] or [tensor.Descending] for self-documentation,\nand optionally using a stable sort.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"ascending", "stable", "columns"}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values in column compared to given\nstring. Includes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse the named const args [tensor.Include], [tensor.Exclude], [tensor.Contains],\n[tensor.Equals], [tensor.IgnoreCase], [tensor.UseCase] for greater clarity.\nOnly valid for 1-dimensional columns (uses first cell from higher dimensions).\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"columnName", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}, Returns: []string{"Table"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns.\nIf rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0.\nIf indexes are in place and rows are added, indexes for the new rows are added.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "Columns has the list of column tensor data for this table.\nDifferent tables can provide different indexed views onto the same Columns."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nThese indexes are shared into the `tensor.Indexed` Column values\nto provide a coordinated indexed view into the underlying data."}, {Name: "Meta", Doc: "Meta is misc metadata for the table. Use lower-case key names\nfollowing the struct tag convention:\n\t- name string = name of table\n\t- doc string = documentation, description\n\t- read-only bool = gui is read-only\n\t- precision int = n for precision to write out floats in csv."}}}) diff --git a/tensor/table/util.go b/tensor/table/util.go index 1e8e0451b4..97a3632885 100644 --- a/tensor/table/util.go +++ b/tensor/table/util.go @@ -40,8 +40,8 @@ func (dt *Table) InsertKeyColumns(args ...string) *Table { // values in the first two columns of given format table, conventionally named // Name, Type (but names are not used), which must be of the string type. func (dt *Table) ConfigFromTable(ft *Table) error { - nmcol := ft.ColumnIndex(0) - tycol := ft.ColumnIndex(1) + nmcol := ft.ColumnByIndex(0) + tycol := ft.ColumnByIndex(1) var errs []error for i := range ft.NumRows() { name := nmcol.String1D(i) diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 3675c56bfc..9ed8888153 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -110,8 +110,8 @@ func (tb *Table) Init() { func (tb *Table) SliceIndex(i int) (si, vi int, invis bool) { si = tb.StartIndex + i vi = -1 - if si < len(tb.Table.Indexes) { - vi = tb.Table.Indexes[si] + if si < tb.Table.NumRows() { + vi = tb.Table.Index(si) } invis = vi < 0 return @@ -464,7 +464,7 @@ func (tb *Table) SortSliceAction(fldIndex int) { tb.Table.SortIndexes() } else { tb.Table.IndexesNeeded() - col := tb.Table.ColumnIndex(tb.SortIndex) + col := tb.Table.ColumnByIndex(tb.SortIndex) col.Sort(!tb.SortDescending) tb.Table.IndexesFromTensor(col) } @@ -641,15 +641,14 @@ func (tb *Table) MakeToolbar(p *tree.Plan) { w.SetFunc(tb.Table.AddRows).SetIcon(icons.Add) w.SetAfterFunc(func() { tb.Update() }) }) - // todo: - // tree.Add(p, func(w *core.FuncButton) { - // w.SetFunc(tb.Table.SortColumnName).SetText("Sort").SetIcon(icons.Sort) - // w.SetAfterFunc(func() { tb.Update() }) - // }) - // tree.Add(p, func(w *core.FuncButton) { - // w.SetFunc(tb.Table.FilterColumnName).SetText("Filter").SetIcon(icons.FilterAlt) - // w.SetAfterFunc(func() { tb.Update() }) - // }) + tree.Add(p, func(w *core.FuncButton) { + w.SetFunc(tb.Table.SortColumns).SetText("Sort").SetIcon(icons.Sort) + w.SetAfterFunc(func() { tb.Update() }) + }) + tree.Add(p, func(w *core.FuncButton) { + w.SetFunc(tb.Table.FilterString).SetText("Filter").SetIcon(icons.FilterAlt) + w.SetAfterFunc(func() { tb.Update() }) + }) tree.Add(p, func(w *core.FuncButton) { w.SetFunc(tb.Table.Sequential).SetText("Unfilter").SetIcon(icons.FilterAltOff) w.SetAfterFunc(func() { tb.Update() }) @@ -678,7 +677,7 @@ func (tb *Table) CopySelectToMime() mimedata.Mimes { idx := tb.SelectedIndexesList(false) // ascending iidx := make([]int, len(idx)) for i, di := range idx { - iidx[i] = tb.Table.Indexes[di] + iidx[i] = tb.Table.Index(di) } ix.Indexes = iidx var b bytes.Buffer @@ -713,7 +712,7 @@ func (tb *Table) PasteAssign(md mimedata.Mimes, idx int) { if len(recs) == 0 { return } - tb.Table.ReadCSVRow(recs[1], tb.Table.Indexes[idx]) + tb.Table.ReadCSVRow(recs[1], tb.Table.Index(idx)) tb.UpdateChange() } @@ -728,7 +727,7 @@ func (tb *Table) PasteAtIndex(md mimedata.Mimes, idx int) { tb.Table.InsertRows(idx, nr) for ri := 0; ri < nr; ri++ { rec := recs[1+ri] - rw := tb.Table.Indexes[idx+ri] + rw := tb.Table.Index(idx + ri) tb.Table.ReadCSVRow(rec, rw) } tb.SendChange() From 0b6f3404f343ef6a0f7f4246c9e212d8362e415f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 12:20:52 -0700 Subject: [PATCH 038/311] datafs building --- base/metadata/metadata.go | 32 ++++++++++++++++++++++ tensor/base.go | 2 +- tensor/datafs/README.md | 6 ++++- tensor/datafs/data.go | 22 ++++++++------- tensor/datafs/dir.go | 56 ++++++++++++++++++++++++++++----------- tensor/datafs/fs.go | 28 +++----------------- tensor/datafs/metadata.go | 18 ++++++------- tensor/table/README.md | 2 +- 8 files changed, 105 insertions(+), 61 deletions(-) diff --git a/base/metadata/metadata.go b/base/metadata/metadata.go index 1fbc1852e2..85c3bb7f03 100644 --- a/base/metadata/metadata.go +++ b/base/metadata/metadata.go @@ -2,15 +2,27 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// Package metadata provides a map of named any elements +// with generic support for type-safe Get and nil-safe Set. +// Metadata keys often function as optional fields in a struct, +// and therefore a CamelCase naming convention is typical. +// Provides default support for "Name" and "Doc" standard keys. package metadata import ( "fmt" "maps" + + "cogentcore.org/core/base/errors" ) // Data is metadata as a map of named any elements // with generic support for type-safe Get and nil-safe Set. +// Metadata keys often function as optional fields in a struct, +// and therefore a CamelCase naming convention is typical. +// Provides default support for "Name" and "Doc" standard keys. +// In general it is good practice to provide access functions +// that establish standard key names, to avoid issues with typos. type Data map[string]any func (md *Data) init() { @@ -52,3 +64,23 @@ func (md *Data) Copy(src Data) { md.init() maps.Copy(*md, src) } + +// SetName sets the "Name" standard key. +func (md *Data) SetName(name string) { + md.Set("Name", name) +} + +// GetName returns the "Name" standard key value (empty if not set). +func (md *Data) GetName() string { + return errors.Ignore1(Get[string](*md, "Name")) +} + +// SetDoc sets the "Doc" standard key. +func (md *Data) SetDoc(doc string) { + md.Set("Doc", doc) +} + +// GetDoc returns the "Doc" standard key value (empty if not set). +func (md *Data) GetDoc() string { + return errors.Ignore1(Get[string](*md, "Doc")) +} diff --git a/tensor/base.go b/tensor/base.go index 0f2b02958b..da108f05b8 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -142,7 +142,7 @@ func (tsr *Base[T]) StringRowCell(row, cell int) string { // Label satisfies the core.Labeler interface for a summary description of the tensor. func (tsr *Base[T]) Label() string { - nm, _ := metadata.Get[string](tsr.Meta, "name") + nm := tsr.Meta.GetName() if nm != "" { nm += " " + tsr.shape.String() } else { diff --git a/tensor/datafs/README.md b/tensor/datafs/README.md index 815df4429a..cae33f83e5 100644 --- a/tensor/datafs/README.md +++ b/tensor/datafs/README.md @@ -4,7 +4,11 @@ Data is represented using the [tensor] package universal data type: the `tensor.Indexed` `Tensor`, which can represent everything from a single scalar value up to n-dimensional collections of patterns, in a range of data types. -A given `Data` node either has a tensor `Value` or is a directory with an ordered map of other nodes under it. Each value has a name which must be unique within the directory. The nodes are processed in the order of this list, which initially reflects the order added, and can be re-ordered as needed. +A given `Data` node is either: +* A _Value_, with a tensor encoding its `Data` value. These are terminal "leaves" in the hierarchical data tree, equivalent to "files" in a standard filesystem. +* A _Directory_, with an ordered map of other Data nodes under it. + +Each Data node has a name which must be unique within the directory. The nodes in a directory are processed in the order of its ordered map list, which initially reflects the order added, and can be re-ordered as needed. An alphabetical sort is also available with the `Alpha` versions of methods, and is the default sort for standard FS operations. The hierarchical structure of a filesystem naturally supports various kinds of functions, such as various time scales of logging, with lower-level data aggregated into upper levels. Or hierarchical splits for a pivot-table effect. diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 85f3002c21..cc7573167b 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -5,9 +5,9 @@ package datafs import ( - "errors" "time" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fileinfo" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" @@ -28,10 +28,11 @@ type Data struct { // modTime tracks time added to directory, used for ordering. modTime time.Time - // Value is represented using the universal [tensor] data type of + // Data is the data value for "files" / "leaves" in the FS, + // represented using the universal [tensor] data type of // [tensor.Indexed], which can represent anything from a scalar // to n-dimensional data, in a range of data types. - Value *tensor.Indexed + Data *tensor.Indexed // Dir is for directory nodes, with all the items in the directory. Dir *Dir @@ -58,21 +59,22 @@ func newData(dir *Data, name string) (*Data, error) { // NewValue returns a new Data value as an [tensor.Indexed] [tensor.Tensor] // of given data type and shape sizes, in given directory Data item. // The name must be unique in the directory. -func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Indexed { +func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) *tensor.Indexed { tsr := tensor.New[T](sizes...) + tsr.Metadata().SetName(name) d, err := newData(dir, name) if errors.Log(err) != nil { return nil } - d.Value = tensor.NewIndexed(tsr) - return tsr + d.Data = tensor.NewIndexed(tsr) + return d.Data } func (d *Data) KnownFileInfo() fileinfo.Known { - if d.Value == nil { + if d.Data == nil { return fileinfo.Unknown } - tsr := d.Value.Tensor + tsr := d.Data.Tensor if tsr.Len() > 1 { return fileinfo.Tensor } @@ -86,8 +88,8 @@ func (d *Data) KnownFileInfo() fileinfo.Known { // This is the actual underlying data, so make a copy if it can be // unintentionally modified or retained more than for immediate use. func (d *Data) Bytes() []byte { - if d.Value == nil { + if d.Data == nil { return nil } - return d.Value.Tensor.Bytes() + return d.Data.Tensor.Bytes() } diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index a1c28730ad..b99bd302dd 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -54,7 +54,7 @@ func (d *Data) Item(name string) *Data { // found, and will return nil if it is not a Value // (i.e., it is a directory). func (d *Data) Value(name string) *tensor.Indexed { - return d.Dir.ValueByKey(name).Value + return d.Dir.ValueByKey(name).Data } // Items returns data items in given directory by name. @@ -87,8 +87,8 @@ func (d *Data) Values(names ...string) ([]*tensor.Indexed, error) { var its []*tensor.Indexed for _, nm := range names { it := d.Dir.ValueByKey(nm) - if it != nil && it.Value != nil { - its = append(its, it.Value) + if it != nil && it.Data != nil { + its = append(its, it.Data) } else { err := fmt.Errorf("datafs Dir %q item not found: %q", d.Path(), nm) errs = append(errs, err) @@ -125,13 +125,13 @@ func (d *Data) ValuesFunc(fun func(item *Data) bool) []*tensor.Indexed { } var its []*tensor.Indexed for _, it := range d.Dir.Values { - if it.Value == nil { + if it.Data == nil { continue } if fun != nil && !fun(it) { continue } - its = append(its, it.Value) + its = append(its, it.Data) } return its } @@ -176,7 +176,32 @@ func (d *Data) FlatValuesFunc(fun func(item *Data) bool) []*tensor.Indexed { subs := it.FlatValuesFunc(fun) its = append(its, subs...) } else { - its = append(its, it.Value) + its = append(its, it.Data) + } + } + return its +} + +// FlatItemsFunc returns all Value items (tensor) as Data items +// in given directory, recursively descending into directories +// to return a flat list of the entire subtree, filtered by +// given function, in directory order (e.g., order added). +// The function can filter out directories to prune the tree. +// If func is nil, all Value items are returned. +func (d *Data) FlatItemsFunc(fun func(item *Data) bool) []*Data { + if err := d.mustDir("FlatItemsFunc", ""); err != nil { + return nil + } + var its []*Data + for _, it := range d.Dir.Values { + if fun != nil && !fun(it) { + continue + } + if it.IsDir() { + subs := it.FlatItemsFunc(fun) + its = append(its, subs...) + } else { + its = append(its, it) } } return its @@ -202,7 +227,7 @@ func (d *Data) FlatValuesAlphaFunc(fun func(item *Data) bool) []*tensor.Indexed subs := it.FlatValuesAlphaFunc(fun) its = append(its, subs...) } else { - its = append(its, it.Value) + its = append(its, it.Data) } } return its @@ -273,7 +298,7 @@ func (d *Data) Add(it *Data) error { if err := d.mustDir("Add", it.name); err != nil { return err } - err := d.Dir.Add(name, it) + err := d.Dir.Add(it.name, it) if err != nil { return &fs.PathError{Op: "Add", Path: it.name, Err: errors.New("data item already exists; names must be unique")} } @@ -291,7 +316,7 @@ func (d *Data) Mkdir(name string) (*Data, error) { // GetDirTable gets the DirTable as a [table.Table] for this directory item, // with columns as the Tensor values elements in the directory -// and any subdirectories, from FlatValuesFunc using given filter function. +// and any subdirectories, from FlatItemsFunc using given filter function. // This is a convenient mechanism for creating a plot of all the data // in a given directory. // If such was previously constructed, it is returned from "DirTable" @@ -301,21 +326,22 @@ func (d *Data) Mkdir(name string) (*Data, error) { func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { if d.DirTable != nil { d.DirTable.SetNumRowsToMax() - return dt + return d.DirTable } - tsrs := d.FlatValuesFunc(fun) + its := d.FlatItemsFunc(fun) dt := table.NewTable(fsx.DirAndFile(string(d.Path()))) - for _, tsr := range tsrs { - rows := tsr.Tensor.Rows() + for _, it := range its { + tsr := it.Data + rows := tsr.NumRows() if dt.Columns.Rows < rows { dt.Columns.Rows = rows dt.SetNumRows(dt.Columns.Rows) } - nm := it.Name() + nm := it.name if it.Parent != d { nm = fsx.DirAndFile(string(it.Path())) } - dt.AddColumn(tsr.Tensor, nm) + dt.AddColumn(nm, tsr.Tensor) } d.DirTable = dt return dt diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index 3b401aa2b0..0eeed6f223 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -11,7 +11,6 @@ import ( "path" "slices" "time" - "unsafe" "cogentcore.org/core/base/fsx" ) @@ -146,34 +145,15 @@ func (d *Data) ReadFile(name string) ([]byte, error) { /////////////////////////////// // FileInfo interface: -// Sizer is an interface to allow an arbitrary data Value -// to report its size in bytes. Size is automatically computed for -// known basic data Values supported by datafs directly. -type Sizer interface { - Sizeof() int64 -} - func (d *Data) Name() string { return d.name } // Size returns the size of known data Values, or it uses // the Sizer interface, otherwise returns 0. func (d *Data) Size() int64 { - if szr, ok := d.Value.(Sizer); ok { // tensor implements Sizer - return szr.Sizeof() - } - switch x := d.Value.(type) { - case float32, int32, uint32: - return 4 - case float64, int64: - return 8 - case int: - return int64(unsafe.Sizeof(x)) - case complex64: - return 16 - case complex128: - return 32 - } - return 0 + if d.Data == nil { + return 0 + } + return d.Data.Tensor.Sizeof() } func (d *Data) IsDir() bool { diff --git a/tensor/datafs/metadata.go b/tensor/datafs/metadata.go index 3b8ab180fb..7250c73072 100644 --- a/tensor/datafs/metadata.go +++ b/tensor/datafs/metadata.go @@ -15,9 +15,9 @@ import ( // SetMetaItems sets given metadata for Value items in given directory // with given names. Returns error for any items not found. func (d *Data) SetMetaItems(key string, value any, names ...string) error { - tsrs, err := d.Value(names...) + tsrs, err := d.Values(names...) for _, tsr := range tsrs { - tsr.Tensor.Meta.Set(key, value) + tsr.Tensor.Metadata().Set(key, value) } return err } @@ -25,20 +25,20 @@ func (d *Data) SetMetaItems(key string, value any, names ...string) error { // SetCalcFunc sets a function to compute an updated Value for this Value item. // Function is stored as CalcFunc in Metadata. Can be called by [Data.Calc] method. func (d *Data) SetCalcFunc(fun func() error) { - if d.Value == nil { + if d.Data == nil { return } - d.Value.Tensor.Meta.Set("CalcFunc", fun) + d.Data.Tensor.Metadata().Set("CalcFunc", fun) } // Calc calls function set by [Data.SetCalcFunc] to compute an updated Value // for this data item. Returns an error if func not set, or any error from func itself. // Function is stored as CalcFunc in Metadata. func (d *Data) Calc() error { - if d.Value == nil { - return + if d.Data == nil { + return nil } - fun, err := metadata.Get[func() error](d.Value.Tensor.Meta, "CalcFunc") + fun, err := metadata.Get[func() error](*d.Data.Tensor.Metadata(), "CalcFunc") if err != nil { return err } @@ -47,10 +47,10 @@ func (d *Data) Calc() error { // CalcAll calls function set by [Data.SetCalcFunc] for all items // in this directory and all of its subdirectories. -// Calls Calc on items from FlatValuesFunc(nil) +// Calls Calc on items from FlatItemsFunc(nil) func (d *Data) CalcAll() error { var errs []error - items := d.FlatValuesFunc(nil) + items := d.FlatItemsFunc(nil) for _, it := range items { err := it.Calc() if err != nil { diff --git a/tensor/table/README.md b/tensor/table/README.md index dc22378801..952a7ee9fe 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -14,7 +14,7 @@ As a general convention, it is safest, clearest, and quite fast to access column The table itself stores raw data `tensor.Tensor` values, and the `Column` (by name) and `ColumnByIndex` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). -If you call Sort, Filter or other routines on an individual column tensor, then you can grab the updated indexes via the `IndexesFromTensor` method so that they apply to the entire table. +If you call Sort, Filter or other routines on an individual column tensor, then you can grab the updated indexes via the `IndexesFromTensor` method so that they apply to the entire table. The `SortColumn` and `FilterString` methods do this for you. There are also multi-column `Sort` and `Filter` methods on the Table itself. From 31a0ae0b04d02f1fafe515582e4dc8df3f5cf24f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 14:41:15 -0700 Subject: [PATCH 039/311] sim example building and mostly working -- data browser not quite working.. --- plot/plotcore/metadata.go | 7 ++- tensor/base.go | 2 + tensor/bits.go | 91 +++++++++++++++++++-------- tensor/databrowser/filetree.go | 44 ++++++------- tensor/datafs/data.go | 98 ++++++++++++++++++++++++++++- tensor/datafs/fs.go | 4 +- tensor/datafs/fs_test.go | 6 +- tensor/datafs/metadata.go | 29 +-------- tensor/examples/datafs-sim/sim.go | 99 ++++++++++++++--------------- tensor/funcs.go | 19 ++++++ tensor/indexed.go | 101 ++++++++++++++++++++++++------ tensor/number.go | 33 ++++++++++ tensor/stats/stats/quantiles.go | 2 +- tensor/string.go | 33 ++++++++++ tensor/tensor.go | 30 +++++++++ 15 files changed, 437 insertions(+), 161 deletions(-) diff --git a/plot/plotcore/metadata.go b/plot/plotcore/metadata.go index 72585311bc..9b0009f0f7 100644 --- a/plot/plotcore/metadata.go +++ b/plot/plotcore/metadata.go @@ -101,12 +101,13 @@ func (co *ColumnOptions) fromMetaMap(meta metadata.Data) { } } -// PlotColumnZeroOne returns plot options with a fixed 0-1 range -func PlotColumnZeroOne() *ColumnOptions { +// PlotColumnZeroOne returns plot options with a fixed 0-1 range, +// and the standard key: "PlotColumnOptions" +func PlotColumnZeroOne() (*ColumnOptions, string) { opts := &ColumnOptions{} opts.Range.SetMin(0) opts.Range.SetMax(1) - return opts + return opts, "PlotColumnOptions" } // SetPlotColumnOptions sets given plotting options for named items diff --git a/tensor/base.go b/tensor/base.go index da108f05b8..8890a01976 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -129,6 +129,8 @@ func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { return stsr } +///////////////////// Strings + func (tsr *Base[T]) StringValue(i ...int) string { return reflectx.ToString(tsr.Values[tsr.shape.Offset(i...)]) } diff --git a/tensor/bits.go b/tensor/bits.go index 978a118a53..5b7e778815 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -10,6 +10,7 @@ import ( "reflect" "cogentcore.org/core/base/metadata" + "cogentcore.org/core/base/num" "cogentcore.org/core/base/reflectx" "cogentcore.org/core/base/slicesx" "cogentcore.org/core/tensor/bitslice" @@ -43,20 +44,24 @@ func NewBitsShape(shape *Shape) *Bits { return tsr } +// Float64ToBool converts float64 value to bool. func Float64ToBool(val float64) bool { - bv := true - if val == 0 { - bv = false - } - return bv + return num.ToBool(val) } +// BoolToFloat64 converts bool to float64 value. func BoolToFloat64(bv bool) float64 { - if bv { - return 1 - } else { - return 0 - } + return num.FromBool[float64](bv) +} + +// IntToBool converts int value to bool. +func IntToBool(val int) bool { + return num.ToBool(val) +} + +// BoolToInt converts bool to int value. +func BoolToInt(bv bool) int { + return num.FromBool[int](bv) } // String satisfies the fmt.Stringer interface for string of tensor data. @@ -155,12 +160,16 @@ func (tsr *Bits) SetRowTensor(val Tensor, row int) { } -func (tsr *Bits) Float(i ...int) float64 { - return BoolToFloat64(tsr.Values.Index(tsr.shape.Offset(i...))) +///////////////////// Strings + +func (tsr *Bits) String1D(off int) string { + return reflectx.ToString(tsr.Values.Index(off)) } -func (tsr *Bits) SetFloat(val float64, i ...int) { - tsr.Values.Set(Float64ToBool(val), tsr.shape.Offset(i...)) +func (tsr *Bits) SetString1D(val string, off int) { + if bv, err := reflectx.ToBool(val); err == nil { + tsr.Values.Set(bv, off) + } } func (tsr *Bits) StringValue(i ...int) string { @@ -173,6 +182,28 @@ func (tsr *Bits) SetString(val string, i ...int) { } } +func (tsr *Bits) StringRowCell(row, cell int) string { + _, sz := tsr.RowCellSize() + return reflectx.ToString(tsr.Values.Index(row*sz + cell)) +} + +func (tsr *Bits) SetStringRowCell(val string, row, cell int) { + if bv, err := reflectx.ToBool(val); err == nil { + _, sz := tsr.RowCellSize() + tsr.Values.Set(bv, row*sz+cell) + } +} + +///////////////////// Floats + +func (tsr *Bits) Float(i ...int) float64 { + return BoolToFloat64(tsr.Values.Index(tsr.shape.Offset(i...))) +} + +func (tsr *Bits) SetFloat(val float64, i ...int) { + tsr.Values.Set(Float64ToBool(val), tsr.shape.Offset(i...)) +} + func (tsr *Bits) Float1D(off int) float64 { return BoolToFloat64(tsr.Values.Index(off)) } @@ -191,26 +222,32 @@ func (tsr *Bits) SetFloatRowCell(val float64, row, cell int) { tsr.Values.Set(Float64ToBool(val), row*sz+cell) } -func (tsr *Bits) String1D(off int) string { - return reflectx.ToString(tsr.Values.Index(off)) +///////////////////// Ints + +func (tsr *Bits) Int(i ...int) int { + return BoolToInt(tsr.Values.Index(tsr.shape.Offset(i...))) } -func (tsr *Bits) SetString1D(val string, off int) { - if bv, err := reflectx.ToBool(val); err == nil { - tsr.Values.Set(bv, off) - } +func (tsr *Bits) SetInt(val int, i ...int) { + tsr.Values.Set(IntToBool(val), tsr.shape.Offset(i...)) } -func (tsr *Bits) StringRowCell(row, cell int) string { +func (tsr *Bits) Int1D(off int) int { + return BoolToInt(tsr.Values.Index(off)) +} + +func (tsr *Bits) SetInt1D(val int, off int) { + tsr.Values.Set(IntToBool(val), off) +} + +func (tsr *Bits) IntRowCell(row, cell int) int { _, sz := tsr.RowCellSize() - return reflectx.ToString(tsr.Values.Index(row*sz + cell)) + return BoolToInt(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bits) SetStringRowCell(val string, row, cell int) { - if bv, err := reflectx.ToBool(val); err == nil { - _, sz := tsr.RowCellSize() - tsr.Values.Set(bv, row*sz+cell) - } +func (tsr *Bits) SetIntRowCell(val int, row, cell int) { + _, sz := tsr.RowCellSize() + tsr.Values.Set(IntToBool(val), row*sz+cell) } // Label satisfies the core.Labeler interface for a summary description of the tensor diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index ad289ac312..14a8de0c79 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -7,7 +7,6 @@ package databrowser import ( "image" "log" - "reflect" "strings" "cogentcore.org/core/base/errors" @@ -19,6 +18,7 @@ import ( "cogentcore.org/core/icons" "cogentcore.org/core/styles" "cogentcore.org/core/styles/states" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" "cogentcore.org/core/texteditor/diffbrowser" @@ -41,7 +41,7 @@ func (fn *FileNode) WidgetTooltip(pos image.Point) (string, image.Point) { switch fn.Info.Known { case fileinfo.Number, fileinfo.String: dv := DataFS(ofn) - v, _ := dv.AsString() + v := dv.AsString() if res != "" { res += " " } @@ -102,20 +102,15 @@ func (fn *FileNode) OpenFile() error { switch fn.Info.Known { case fileinfo.Tensor: d := DataFS(ofn) - tsr := d.AsTensor() - if tsr.IsString() || tsr.DataType() < reflect.Float32 { - br.NewTabTensorEditor(df, tsr) - } else { - br.NewTabTensorGrid(df, tsr) - } - case fileinfo.Table: - d := DataFS(ofn) - dt := d.AsTable() - br.NewTabTensorTable(df, dt) - br.Update() + br.NewTabTensorEditor(df, d.Data.Tensor) + // case fileinfo.Table: + // d := DataFS(ofn) + // dt := d.AsTable() + // br.NewTabTensorTable(df, dt) + // br.Update() case fileinfo.Number: dv := DataFS(ofn) - v, _ := dv.AsFloat32() + v := dv.AsFloat32() d := core.NewBody(df) core.NewText(d).SetType(core.TextSupporting).SetText(df) sp := core.NewSpinner(d).SetValue(v) @@ -128,7 +123,7 @@ func (fn *FileNode) OpenFile() error { d.RunDialog(br) case fileinfo.String: dv := DataFS(ofn) - v, _ := dv.AsString() + v := dv.AsString() d := core.NewBody(df) core.NewText(d).SetType(core.TextSupporting).SetText(df) tf := core.NewTextField(d).SetText(v) @@ -142,7 +137,7 @@ func (fn *FileNode) OpenFile() error { default: dt := table.NewTable() - err := dt.OpenCSV(fn.Filepath, table.Tab) // todo: need more flexible data handling mode + err := dt.OpenCSV(fn.Filepath, tensor.Tab) // todo: need more flexible data handling mode if err != nil { core.ErrorSnackbar(br, err) } else { @@ -217,23 +212,24 @@ func (fn *FileNode) PlotFile() { var dt *table.Table switch { case fn.IsDir(): - dt = d.DirTable(nil) + dt = d.GetDirTable(nil) case fn.Info.Cat == fileinfo.Data: switch fn.Info.Known { case fileinfo.Tensor: - tsr := d.AsTensor() + tsr := d.Data dt = table.NewTable(df) - dt.Rows = tsr.DimSize(0) + dt.Columns.Rows = tsr.NumRows() + dt.Indexes = tsr.Indexes rc := dt.AddIntColumn("Row") - for r := range dt.Rows { + for r := range dt.Columns.Rows { rc.Values[r] = r } - dt.AddColumn(tsr, fn.Name) - case fileinfo.Table: - dt = d.AsTable() + dt.AddColumn(fn.Name, tsr.Tensor) + // case fileinfo.Table: + // dt = d.AsTable() default: dt = table.NewTable(df) - err := dt.OpenCSV(fn.Filepath, table.Tab) // todo: need more flexible data handling mode + err := dt.OpenCSV(fn.Filepath, tensor.Tab) // todo: need more flexible data handling mode if err != nil { core.ErrorSnackbar(br, err) dt = nil diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index cc7573167b..7f1fed6f16 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -5,6 +5,7 @@ package datafs import ( + "reflect" "time" "cogentcore.org/core/base/errors" @@ -56,7 +57,28 @@ func newData(dir *Data, name string) (*Data, error) { return d, err } -// NewValue returns a new Data value as an [tensor.Indexed] [tensor.Tensor] +// NewScalar returns new scalar Data value(s) (as a [tensor.Indexed]) +// of given data type, in given directory. +// The names must be unique in the directory. +// Returns the first item created, for immediate use of one value. +func NewScalar[T tensor.DataTypes](dir *Data, names ...string) *tensor.Indexed { + var first *tensor.Indexed + for _, nm := range names { + tsr := tensor.New[T](1) + tsr.Metadata().SetName(nm) + d, err := newData(dir, nm) + if errors.Log(err) != nil { + return nil + } + d.Data = tensor.NewIndexed(tsr) + if first == nil { + first = d.Data + } + } + return first +} + +// NewValue returns a new Data value as a [tensor.Indexed] [tensor.Tensor] // of given data type and shape sizes, in given directory Data item. // The name must be unique in the directory. func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) *tensor.Indexed { @@ -70,6 +92,21 @@ func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) *tensor. return d.Data } +// NewOfType returns a new Data value as a [tensor.Indexed] +// of given reflect.Kind type and shape sizes per dimension, in given directory Data item. +// Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. +// The name must be unique in the directory. +func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) *tensor.Indexed { + tsr := tensor.NewOfType(typ, sizes...) + tsr.Metadata().SetName(name) + nd, err := newData(d, name) + if errors.Log(err) != nil { + return nil + } + nd.Data = tensor.NewIndexed(tsr) + return nd.Data +} + func (d *Data) KnownFileInfo() fileinfo.Known { if d.Data == nil { return fileinfo.Unknown @@ -78,6 +115,7 @@ func (d *Data) KnownFileInfo() fileinfo.Known { if tsr.Len() > 1 { return fileinfo.Tensor } + // scalars by type if tsr.IsString() { return fileinfo.String } @@ -93,3 +131,61 @@ func (d *Data) Bytes() []byte { } return d.Data.Tensor.Bytes() } + +// AsString returns data as scalar string. +func (d *Data) AsString() string { + if d.Data == nil { + return "" + } + return d.Data.StringRowCell(0, 0) +} + +// SetString sets scalar data value from given string. +func (d *Data) SetString(v string) { + if d.Data == nil { + return + } + d.Data.SetStringRowCell(v, 0, 0) +} + +// AsFloat64 returns data as a scalar float64 (first element of tensor). +func (d *Data) AsFloat64() float64 { + if d.Data == nil { + return 0 + } + return d.Data.FloatRowCell(0, 0) +} + +// SetFloat64 sets scalar data value from given float64. +func (d *Data) SetFloat64(v float64) { + if d.Data == nil { + return + } + d.Data.SetFloatRowCell(v, 0, 0) +} + +// AsFloat32 returns data as a scalar float32 (first element of tensor). +func (d *Data) AsFloat32() float32 { + return float32(d.AsFloat64()) +} + +// SetFloat32 sets scalar data value from given float32. +func (d *Data) SetFloat32(v float32) { + d.SetFloat64(float64(v)) +} + +// AsInt returns data as a scalar int (first element of tensor). +func (d *Data) AsInt() int { + if d.Data == nil { + return 0 + } + return d.Data.IntRowCell(0, 0) +} + +// SetInt sets scalar data value from given int. +func (d *Data) SetInt(v int) { + if d.Data == nil { + return + } + d.Data.SetIntRowCell(v, 0, 0) +} diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index 0eeed6f223..b3962b7e15 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -173,8 +173,8 @@ func (d *Data) Mode() fs.FileMode { // Sys returns the Dir or Value func (d *Data) Sys() any { - if d.Value != nil { - return d.Value + if d.Data != nil { + return d.Data } return d.Dir } diff --git a/tensor/datafs/fs_test.go b/tensor/datafs/fs_test.go index e44f9a2d27..44fc131a2d 100644 --- a/tensor/datafs/fs_test.go +++ b/tensor/datafs/fs_test.go @@ -17,10 +17,8 @@ func makeTestData(t *testing.T) *Data { assert.NoError(t, err) net, err := dfs.Mkdir("network") assert.NoError(t, err) - NewTensor[float32](net, "units", []int{50, 50}) - log, err := dfs.Mkdir("log") - assert.NoError(t, err) - _, err = NewTable(log, "Trial") + NewValue[float32](net, "units", 50, 50) + _, err = dfs.Mkdir("log") assert.NoError(t, err) return dfs } diff --git a/tensor/datafs/metadata.go b/tensor/datafs/metadata.go index 7250c73072..5ac08998d4 100644 --- a/tensor/datafs/metadata.go +++ b/tensor/datafs/metadata.go @@ -6,7 +6,7 @@ package datafs import ( "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/metadata" + "cogentcore.org/core/tensor" ) // This file provides standardized metadata options for frequent @@ -22,37 +22,14 @@ func (d *Data) SetMetaItems(key string, value any, names ...string) error { return err } -// SetCalcFunc sets a function to compute an updated Value for this Value item. -// Function is stored as CalcFunc in Metadata. Can be called by [Data.Calc] method. -func (d *Data) SetCalcFunc(fun func() error) { - if d.Data == nil { - return - } - d.Data.Tensor.Metadata().Set("CalcFunc", fun) -} - -// Calc calls function set by [Data.SetCalcFunc] to compute an updated Value -// for this data item. Returns an error if func not set, or any error from func itself. -// Function is stored as CalcFunc in Metadata. -func (d *Data) Calc() error { - if d.Data == nil { - return nil - } - fun, err := metadata.Get[func() error](*d.Data.Tensor.Metadata(), "CalcFunc") - if err != nil { - return err - } - return fun() -} - // CalcAll calls function set by [Data.SetCalcFunc] for all items // in this directory and all of its subdirectories. // Calls Calc on items from FlatItemsFunc(nil) func (d *Data) CalcAll() error { var errs []error - items := d.FlatItemsFunc(nil) + items := d.FlatValuesFunc(nil) for _, it := range items { - err := it.Calc() + err := tensor.Calc(it.Tensor) if err != nil { errs = append(errs, err) } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 8bee76b464..b9f1d465a7 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -9,8 +9,8 @@ import ( "reflect" "strconv" - "cogentcore.org/core/base/errors" "cogentcore.org/core/core" + "cogentcore.org/core/plot/plotcore" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" "cogentcore.org/core/tensor/datafs" @@ -26,9 +26,9 @@ type Sim struct { // ConfigAll configures the sim func (ss *Sim) ConfigAll() { - ss.Root = errors.Log1(datafs.NewDir("Root")) - ss.Config = errors.Log1(ss.Root.Mkdir("Config")) - datafs.New[int](ss.Config, "NRun", "NEpoch", "NTrial") + ss.Root, _ = datafs.NewDir("Root") + ss.Config, _ = ss.Root.Mkdir("Config") + datafs.NewScalar[int](ss.Config, "NRun", "NEpoch", "NTrial") ss.Config.Item("NRun").SetInt(5) ss.Config.Item("NEpoch").SetInt(20) ss.Config.Item("NTrial").SetInt(25) @@ -39,21 +39,21 @@ func (ss *Sim) ConfigAll() { // ConfigStats adds basic stats that we record for our simulation. func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { - stats := errors.Log1(dir.Mkdir("Stats")) - errors.Log1(datafs.New[int](stats, "Run", "Epoch", "Trial")) // counters - errors.Log1(datafs.New[string](stats, "TrialName")) - errors.Log1(datafs.New[float32](stats, "SSE", "AvgSSE", "TrlErr")) - z1 := datafs.PlotColumnZeroOne() - stats.SetPlotColumnOptions(z1, "AvgErr", "TrlErr") - zmax := datafs.PlotColumnZeroOne() + stats, _ := dir.Mkdir("Stats") + datafs.NewScalar[int](stats, "Run", "Epoch", "Trial") // counters + datafs.NewScalar[string](stats, "TrialName") + datafs.NewScalar[float32](stats, "SSE", "AvgSSE", "TrlErr") + z1, key := plotcore.PlotColumnZeroOne() + stats.SetMetaItems(key, z1, "AvgErr", "TrlErr") + zmax, _ := plotcore.PlotColumnZeroOne() zmax.Range.FixMax = false - stats.SetPlotColumnOptions(z1, "SSE") + stats.SetMetaItems(key, z1, "SSE") return stats } // ConfigLogs adds first-level logging of stats into tensors func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { - logd := errors.Log1(dir.Mkdir("Log")) + logd, _ := dir.Mkdir("Log") trial := ss.ConfigTrialLog(logd) ss.ConfigAggLog(logd, "Epoch", trial, stats.Mean, stats.Sem, stats.Min) return logd @@ -61,22 +61,18 @@ func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { // ConfigTrialLog adds first-level logging of stats into tensors func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { - logd := errors.Log1(dir.Mkdir("Trial")) - ntrial, _ := ss.Config.Item("NTrial").AsInt() - sitems := ss.Stats.ItemsByTimeFunc(nil) + logd, _ := dir.Mkdir("Trial") + ntrial := ss.Config.Item("NTrial").AsInt() + sitems := ss.Stats.ValuesFunc(nil) for _, st := range sitems { - dt := errors.Log1(datafs.NewData(logd, st.Name())) - tsr := tensor.NewOfType(st.DataType(), []int{ntrial}, "row") - dt.Value = tsr - dt.Meta.Copy(st.Meta) // key affordance: we get meta data from source - dt.SetCalcFunc(func() error { - trl, _ := ss.Stats.Item("Trial").AsInt() - if st.IsNumeric() { - v, _ := st.AsFloat64() - tsr.SetFloat1D(trl, v) + lt := logd.NewOfType(st.Tensor.Metadata().GetName(), st.Tensor.DataType(), ntrial) + lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source + tensor.SetCalcFunc(lt.Tensor, func() error { + trl := ss.Stats.Item("Trial").AsInt() + if st.Tensor.IsString() { + lt.SetString1D(st.StringRowCell(0, 0), trl) } else { - v, _ := st.AsString() - tsr.SetString1D(trl, v) + lt.SetFloat1D(st.FloatRowCell(0, 0), trl) } return nil }) @@ -86,38 +82,35 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { // ConfigAggLog adds a higher-level logging of lower-level into higher-level tensors func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, aggs ...stats.Stats) *datafs.Data { - logd := errors.Log1(dir.Mkdir(level)) - sitems := ss.Stats.ItemsByTimeFunc(nil) - nctr, _ := ss.Config.Item("N" + level).AsInt() + logd, _ := dir.Mkdir(level) + sitems := ss.Stats.ValuesFunc(nil) + nctr := ss.Config.Item("N" + level).AsInt() + stout := tensor.NewFloat64Scalar(0) for _, st := range sitems { - if !st.IsNumeric() { + if st.Tensor.IsString() { continue } - src := from.Item(st.Name()).AsTensor() - if st.DataType() >= reflect.Float32 { - dd := errors.Log1(logd.Mkdir(st.Name())) + nm := st.Tensor.Metadata().GetName() + src := from.Value(nm) + if st.Tensor.DataType() >= reflect.Float32 { + dd, _ := logd.Mkdir(nm) for _, ag := range aggs { // key advantage of dir structure: multiple stats per item - dt := errors.Log1(datafs.NewData(dd, ag.String())) - tsr := tensor.NewOfType(st.DataType(), []int{nctr}, "row") - dt.Value = tsr - dt.Meta.Copy(st.Meta) - dt.SetCalcFunc(func() error { - ctr, _ := ss.Stats.Item(level).AsInt() - v := stats.StatTensor(src, ag) - tsr.SetFloat1D(ctr, v) + lt := dd.NewOfType(ag.String(), st.Tensor.DataType(), nctr) + lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) + tensor.SetCalcFunc(lt.Tensor, func() error { + stats.Stat(ag, src, stout) + ctr := ss.Stats.Item(level).AsInt() + lt.SetFloatRowCell(stout.FloatRowCell(0, 0), ctr, 0) return nil }) } } else { - dt := errors.Log1(datafs.NewData(logd, st.Name())) - tsr := tensor.NewOfType(st.DataType(), []int{nctr}, "row") - // todo: set level counter as default x axis in plot config - dt.Value = tsr - dt.Meta.Copy(st.Meta) - dt.SetCalcFunc(func() error { - ctr, _ := ss.Stats.Item(level).AsInt() - v, _ := st.AsFloat64() - tsr.SetFloat1D(ctr, v) + lt := logd.NewOfType(nm, st.Tensor.DataType(), nctr) + lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) + tensor.SetCalcFunc(lt.Tensor, func() error { + v := st.FloatRowCell(0, 0) + ctr := ss.Stats.Item(level).AsInt() + lt.SetFloatRowCell(v, ctr, 0) return nil }) } @@ -126,8 +119,8 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a } func (ss *Sim) Run() { - nepc, _ := ss.Config.Item("NEpoch").AsInt() - ntrl, _ := ss.Config.Item("NTrial").AsInt() + nepc := ss.Config.Item("NEpoch").AsInt() + ntrl := ss.Config.Item("NTrial").AsInt() for epc := range nepc { ss.Stats.Item("Epoch").SetInt(epc) for trl := range ntrl { diff --git a/tensor/funcs.go b/tensor/funcs.go index 79036c445e..4be480c480 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -10,6 +10,7 @@ import ( "strings" "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/metadata" ) // StringFirstArg should be used to set StringFirst functions @@ -265,3 +266,21 @@ func (fn *Func) CallOut(tsr ...*Indexed) ([]*Indexed, error) { err := fn.Call(tsr...) return outs, err } + +// SetCalcFunc sets a function to calculate updated value for given tensor, +// storing the function pointer in the Metadata "CalcFunc" key for the tensor. +// Can be called by [Calc] function. +func SetCalcFunc(tsr Tensor, fun func() error) { + tsr.Metadata().Set("CalcFunc", fun) +} + +// Calc calls function set by [SetCalcFunc] to compute an updated value for +// given tensor. Returns an error if func not set, or any error from func itself. +// Function is stored as CalcFunc in Metadata. +func Calc(tsr Tensor) error { + fun, err := metadata.Get[func() error](*tsr.Metadata(), "CalcFunc") + if err != nil { + return err + } + return fun() +} diff --git a/tensor/indexed.go b/tensor/indexed.go index 42ae460d1b..14141a3ad9 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -444,6 +444,62 @@ func (ix *Indexed) Swap(i, j int) { /////////////////////////////////////////////// // Indexed access +///////////////////// Strings + +// StringValue returns the value of given index as a string. +// The first index value is indirected through the indexes. +func (ix *Indexed) StringValue(i ...int) string { + if ix.Indexes == nil { + return ix.Tensor.StringValue(i...) + } + ic := slices.Clone(i) + ic[0] = ix.Indexes[ic[0]] + return ix.Tensor.StringValue(ic...) +} + +// SetString sets the value of given index as a string +// The first index value is indirected through the [Indexed.Indexes]. +func (ix *Indexed) SetString(val string, i ...int) { + if ix.Indexes == nil { + ix.Tensor.SetString(val, i...) + } + ic := slices.Clone(i) + ic[0] = ix.Indexes[ic[0]] + ix.Tensor.SetString(val, ic...) +} + +// StringRowCell returns the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Indexed.Indexes]. +// This is the preferred interface for all Indexed operations. +func (ix *Indexed) StringRowCell(row, cell int) string { + return ix.Tensor.StringRowCell(ix.Index(row), cell) +} + +// SetStringRowCell sets the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Indexed.Indexes]. +// This is the preferred interface for all Indexed operations. +func (ix *Indexed) SetStringRowCell(val string, row, cell int) { + ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) +} + +// String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. +// This is just a convenience pass-through to the Tensor, and does _not_ use +// the [Indexed.Indexes]. +func (ix *Indexed) String1D(i int) string { + return ix.Tensor.String1D(i) +} + +// SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. +// This is just a convenience pass-through to the Tensor, and does _not_ use +// the [Indexed.Indexes]. +func (ix *Indexed) SetString1D(val string, i int) { + ix.Tensor.SetString1D(val, i) +} + +///////////////////// Floats + // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. func (ix *Indexed) Float(i ...int) float64 { @@ -497,58 +553,63 @@ func (ix *Indexed) SetFloat1D(val float64, i int) { ix.Tensor.SetFloat1D(val, i) } -// StringValue returns the value of given index as a string. +///////////////////// Ints + +// Int returns the value of given index as a float64. // The first index value is indirected through the indexes. -func (ix *Indexed) StringValue(i ...int) string { +func (ix *Indexed) Int(i ...int) int { if ix.Indexes == nil { - return ix.Tensor.StringValue(i...) + return ix.Tensor.Int(i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.StringValue(ic...) + return ix.Tensor.Int(ic...) } -// SetString sets the value of given index as a string +// SetInt sets the value of given index as a int // The first index value is indirected through the [Indexed.Indexes]. -func (ix *Indexed) SetString(val string, i ...int) { +func (ix *Indexed) SetInt(val int, i ...int) { if ix.Indexes == nil { - ix.Tensor.SetString(val, i...) + ix.Tensor.SetInt(val, i...) + return } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetString(val, ic...) + ix.Tensor.SetInt(val, ic...) } -// StringRowCell returns the value at given row and cell, +// IntRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) StringRowCell(row, cell int) string { - return ix.Tensor.StringRowCell(ix.Index(row), cell) +func (ix *Indexed) IntRowCell(row, cell int) int { + return ix.Tensor.IntRowCell(ix.Index(row), cell) } -// SetStringRowCell sets the value at given row and cell, +// SetIntRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetStringRowCell(val string, row, cell int) { - ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) +func (ix *Indexed) SetIntRowCell(val int, row, cell int) { + ix.Tensor.SetIntRowCell(val, ix.Index(row), cell) } -// String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. +// Int1D returns the value of given 1-dimensional index (0-Len()-1) as a int. // This is just a convenience pass-through to the Tensor, and does _not_ use // the [Indexed.Indexes]. -func (ix *Indexed) String1D(i int) string { - return ix.Tensor.String1D(i) +func (ix *Indexed) Int1D(i int) int { + return ix.Tensor.Int1D(i) } -// SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. +// SetInt1D sets the value of given 1-dimensional index (0-Len()-1) as a int. // This is just a convenience pass-through to the Tensor, and does _not_ use // the [Indexed.Indexes]. -func (ix *Indexed) SetString1D(val string, i int) { - ix.Tensor.SetString1D(val, i) +func (ix *Indexed) SetInt1D(val int, i int) { + ix.Tensor.SetInt1D(val, i) } +///////////////////// SubSpaces + // SubSpace returns a new tensor with innermost subspace at given // offset(s) in outermost dimension(s) (len(offs) < NumDims). // The new tensor points to the values of the this tensor (i.e., modifications diff --git a/tensor/number.go b/tensor/number.go index 6fc8c31491..501b1e9f1f 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -130,6 +130,8 @@ func (tsr *Number[T]) IsString() bool { return false } +///////////////////// Strings + func (tsr *Number[T]) SetString(val string, i ...int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { tsr.Values[tsr.shape.Offset(i...)] = T(fv) @@ -149,6 +151,8 @@ func (tsr *Number[T]) SetStringRowCell(val string, row, cell int) { } } +///////////////////// Floats + func (tsr *Number[T]) Float(i ...int) float64 { return float64(tsr.Values[tsr.shape.Offset(i...)]) } @@ -176,6 +180,35 @@ func (tsr *Number[T]) SetFloatRowCell(val float64, row, cell int) { tsr.Values[row*sz+cell] = T(val) } +///////////////////// Ints + +func (tsr *Number[T]) Int(i ...int) int { + return int(tsr.Values[tsr.shape.Offset(i...)]) +} + +func (tsr *Number[T]) SetInt(val int, i ...int) { + tsr.Values[tsr.shape.Offset(i...)] = T(val) +} + +func (tsr *Number[T]) Int1D(i int) int { + return int(tsr.Values[i]) +} + +func (tsr *Number[T]) SetInt1D(val int, i int) { + tsr.Values[i] = T(val) +} + +func (tsr *Number[T]) IntRowCell(row, cell int) int { + _, sz := tsr.shape.RowCellSize() + i := row*sz + cell + return int(tsr.Values[i]) +} + +func (tsr *Number[T]) SetIntRowCell(val int, row, cell int) { + _, sz := tsr.shape.RowCellSize() + tsr.Values[row*sz+cell] = T(val) +} + // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. func (tsr *Number[T]) At(i, j int) float64 { diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 8f7f75fd58..e05e3eb346 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -27,7 +27,7 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } sin := in.Clone() - sin.ExcludeMissing1D() + sin.ExcludeMissing() sin.Sort(tensor.Ascending) sz := len(sin.Indexes) - 1 // length of our own index list fsz := float64(sz) diff --git a/tensor/string.go b/tensor/string.go index 5cb198c157..24b8c2b62e 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -10,6 +10,7 @@ import ( "math" "strconv" + "cogentcore.org/core/base/errors" "gonum.org/v1/gonum/mat" ) @@ -70,6 +71,8 @@ func (tsr *String) IsString() bool { return true } +///////////////////// Strings + func (tsr *String) SetString(val string, i ...int) { j := tsr.shape.Offset(i...) tsr.Values[j] = val @@ -84,6 +87,8 @@ func (tsr *String) SetStringRowCell(val string, row, cell int) { tsr.Values[row*sz+cell] = val } +///////////////////// Floats + func (tsr *String) Float(i ...int) float64 { return StringToFloat64(tsr.Values[tsr.shape.Offset(i...)]) } @@ -110,6 +115,34 @@ func (tsr *String) SetFloatRowCell(val float64, row, cell int) { tsr.Values[row*sz+cell] = Float64ToString(val) } +///////////////////// Ints + +func (tsr *String) Int(i ...int) int { + return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.Offset(i...)])) +} + +func (tsr *String) SetInt(val int, i ...int) { + tsr.Values[tsr.shape.Offset(i...)] = strconv.Itoa(val) +} + +func (tsr *String) Int1D(off int) int { + return errors.Ignore1(strconv.Atoi(tsr.Values[off])) +} + +func (tsr *String) SetInt1D(val int, off int) { + tsr.Values[off] = strconv.Itoa(val) +} + +func (tsr *String) IntRowCell(row, cell int) int { + _, sz := tsr.shape.RowCellSize() + return errors.Ignore1(strconv.Atoi(tsr.Values[row*sz+cell])) +} + +func (tsr *String) SetIntRowCell(val int, row, cell int) { + _, sz := tsr.shape.RowCellSize() + tsr.Values[row*sz+cell] = strconv.Itoa(val) +} + // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. func (tsr *String) At(i, j int) float64 { diff --git a/tensor/tensor.go b/tensor/tensor.go index 7b0253a61d..3a9442c109 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -84,6 +84,8 @@ type Tensor interface { // IsString returns true if the data type is a String; otherwise it is numeric. IsString() bool + ///////////////////// Floats + // Float returns the value of given n-dimensional index (matching Shape) as a float64. Float(i ...int) float64 @@ -133,6 +135,34 @@ type Tensor interface { // and use this interface extensively. SetStringRowCell(val string, row, cell int) + ///////////////////// Ints + + // Int returns the value of given n-dimensional index (matching Shape) as a int. + Int(i ...int) int + + // SetInt sets the value of given n-dimensional index (matching Shape) as a int. + SetInt(val int, i ...int) + + // Int1D returns the value of given 1-dimensional index (0-Len()-1) as a int. + Int1D(i int) int + + // SetInt1D sets the value of given 1-dimensional index (0-Len()-1) as a int. + SetInt1D(val int, i int) + + // IntRowCell returns the value at given row and cell, where row is outermost dim, + // and cell is 1D index into remaining inner dims. This is useful for lists of + // patterns, and the [table.Table] container. [Indexed] tensors index along the row, + // and use this interface extensively. + IntRowCell(row, cell int) int + + // SetIntRowCell sets the value at given row and cell, where row is outermost dim, + // and cell is 1D index into remaining inner dims. This is useful for lists of + // patterns, and the [table.Table] container. [Indexed] tensors index along the row, + // and use this interface extensively. + SetIntRowCell(val int, row, cell int) + + ///////////////////// SubSpaces + // SubSpace returns a new tensor with innermost subspace at given // offset(s) in outermost dimension(s) (len(offs) < [NumDims]). // The new tensor points to the values of the this tensor (i.e., modifications From e73654c3ccc4b3a6a196640d8f8b596abc0d5179 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 14:51:56 -0700 Subject: [PATCH 040/311] sim example fully working --- tensor/datafs/fs.go | 16 ++++++++-------- tensor/tensorcore/tensoreditor.go | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index b3962b7e15..95d04107d4 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -20,7 +20,7 @@ import ( // Open opens the given data Value within this datafs filesystem. func (d *Data) Open(name string) (fs.File, error) { if !fs.ValidPath(name) { - return nil, &fs.PathError{Op: "open", Path: name, Err: errors.New("invalid name")} + return nil, &fs.PathError{Op: "Open", Path: name, Err: errors.New("invalid name")} } dir, file := path.Split(name) sd, err := d.DirAtPath(dir) @@ -32,7 +32,7 @@ func (d *Data) Open(name string) (fs.File, error) { if dir == "" && (file == d.name || file == ".") { return &DirFile{File: File{Reader: *bytes.NewReader(d.Bytes()), Data: d}}, nil } - return nil, &fs.PathError{Op: "open", Path: name, Err: errors.New("file not found")} + return nil, &fs.PathError{Op: "Open", Path: name, Err: errors.New("file not found")} } if itm.IsDir() { return &DirFile{File: File{Reader: *bytes.NewReader(itm.Bytes()), Data: itm}}, nil @@ -44,7 +44,7 @@ func (d *Data) Open(name string) (fs.File, error) { // If there is an error, it should be of type *PathError. func (d *Data) Stat(name string) (fs.FileInfo, error) { if !fs.ValidPath(name) { - return nil, &fs.PathError{Op: "open", Path: name, Err: errors.New("invalid name")} + return nil, &fs.PathError{Op: "Open", Path: name, Err: errors.New("invalid name")} } dir, file := path.Split(name) sd, err := d.DirAtPath(dir) @@ -56,7 +56,7 @@ func (d *Data) Stat(name string) (fs.FileInfo, error) { if dir == "" && (file == d.name || file == ".") { return d, nil } - return nil, &fs.PathError{Op: "stat", Path: name, Err: errors.New("file not found")} + return nil, &fs.PathError{Op: "Stat", Path: name, Err: errors.New("file not found")} } return itm, nil } @@ -87,12 +87,12 @@ func (d *Data) Sub(dir string) (fs.FS, error) { return cur, nil } cd = rest - sd, ok := d.Dir.ValueByKeyTry(root) + sd, ok := cur.Dir.ValueByKeyTry(root) if !ok { - return nil, &fs.PathError{Op: "sub", Path: dir, Err: errors.New("directory not found")} + return nil, &fs.PathError{Op: "Sub", Path: dir, Err: errors.New("directory not found")} } if !sd.IsDir() { - return nil, &fs.PathError{Op: "sub", Path: dir, Err: errors.New("is not a directory")} + return nil, &fs.PathError{Op: "Sub", Path: dir, Err: errors.New("is not a directory")} } cur = sd } @@ -121,7 +121,7 @@ func (d *Data) ReadDir(dir string) ([]fs.DirEntry, error) { // The caller is permitted to modify the returned byte slice. // This method should return a copy of the underlying data. func (d *Data) ReadFile(name string) ([]byte, error) { - if err := d.mustDir("readFile", name); err != nil { + if err := d.mustDir("ReadFile", name); err != nil { return nil, err } if !fs.ValidPath(name) { diff --git a/tensor/tensorcore/tensoreditor.go b/tensor/tensorcore/tensoreditor.go index 08ac7f264c..0e024e4c7f 100644 --- a/tensor/tensorcore/tensoreditor.go +++ b/tensor/tensorcore/tensoreditor.go @@ -51,6 +51,7 @@ var _ core.Lister = (*TensorEditor)(nil) func (tb *TensorEditor) Init() { tb.ListBase.Init() + tb.Layout.OddRow = true tb.Makers.Normal[0] = func(p *tree.Plan) { // TODO: reduce redundancy with ListBase Maker svi := tb.This.(core.Lister) svi.UpdateSliceSize() From d9c6dd61b9908e11995a6be2a891f844a1f6edb2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 14:55:45 -0700 Subject: [PATCH 041/311] sim example cleanup -- looks cleaner than before.. --- tensor/examples/datafs-sim/sim.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index b9f1d465a7..114dd77c2c 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -65,14 +65,15 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { ntrial := ss.Config.Item("NTrial").AsInt() sitems := ss.Stats.ValuesFunc(nil) for _, st := range sitems { - lt := logd.NewOfType(st.Tensor.Metadata().GetName(), st.Tensor.DataType(), ntrial) + nm := st.Tensor.Metadata().GetName() + lt := logd.NewOfType(nm, st.Tensor.DataType(), ntrial) lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source tensor.SetCalcFunc(lt.Tensor, func() error { trl := ss.Stats.Item("Trial").AsInt() if st.Tensor.IsString() { - lt.SetString1D(st.StringRowCell(0, 0), trl) + lt.SetStringRowCell(st.StringRowCell(0, 0), trl, 0) } else { - lt.SetFloat1D(st.FloatRowCell(0, 0), trl) + lt.SetFloatRowCell(st.FloatRowCell(0, 0), trl, 0) } return nil }) @@ -85,7 +86,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a logd, _ := dir.Mkdir(level) sitems := ss.Stats.ValuesFunc(nil) nctr := ss.Config.Item("N" + level).AsInt() - stout := tensor.NewFloat64Scalar(0) + stout := tensor.NewFloat64Scalar(0) // tmp stat output for _, st := range sitems { if st.Tensor.IsString() { continue From 53e7b3e8d6cd88b576de85a8c55790cb11ef7e92 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 17:23:25 -0700 Subject: [PATCH 042/311] splits Groups working -- fits well in the datafs paradigm --- tensor/examples/datafs-sim/sim.go | 23 +++ tensor/indexed.go | 6 + tensor/stats/split/group.go | 177 +++++++----------- tensor/stats/split/random.go | 10 +- tensor/stats/split/random_test.go | 10 +- .../split/{agg_test.go => split_test.go} | 36 ++-- tensor/stats/split/splits.go | 12 +- tensor/stats/split/{agg.go => stats.go} | 28 ++- tensor/table/columns.go | 4 +- tensor/table/indexes.go | 10 +- tensor/table/table.go | 35 +++- 11 files changed, 189 insertions(+), 162 deletions(-) rename tensor/stats/split/{agg_test.go => split_test.go} (56%) rename tensor/stats/split/{agg.go => stats.go} (70%) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 114dd77c2c..70cf1526a4 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -14,7 +14,9 @@ import ( "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" "cogentcore.org/core/tensor/datafs" + "cogentcore.org/core/tensor/stats/split" "cogentcore.org/core/tensor/stats/stats" + "cogentcore.org/core/tensor/table" ) type Sim struct { @@ -151,6 +153,8 @@ func (ss *Sim) EpochDone() { } func main() { + testGroup() + return ss := &Sim{} ss.ConfigAll() ss.Run() @@ -158,3 +162,22 @@ func main() { databrowser.NewBrowserWindow(ss.Root, "Root") core.Wait() } + +func testGroup() { + dt := table.NewTable().SetNumRows(4) + dt.AddStringColumn("Name") + dt.AddFloat32Column("Value") + for i := range dt.NumRows() { + gp := "A" + if i >= 2 { + gp = "B" + } + dt.Column("Name").SetStringRowCell(gp, i, 0) + dt.Column("Value").SetFloatRowCell(float64(i), i, 0) + } + dir, _ := datafs.NewDir("Group") + split.TableGroups(dir, dt, "Name") + + databrowser.NewBrowserWindow(dir, "Group") + core.Wait() +} diff --git a/tensor/indexed.go b/tensor/indexed.go index 14141a3ad9..417bac04cf 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -217,6 +217,12 @@ const ( // Descending specifies a descending sort direction for tensor Sort routines Descending = false + + // Stable specifies using stable, original order-preserving sort, which is slower. + Stable = true + + // Unstable specifies using faster but unstable sorting. + Unstable = false ) // SortFunc sorts the row-wise indexes using given compare function. diff --git a/tensor/stats/split/group.go b/tensor/stats/split/group.go index 18a5d0c16e..452f8389c8 100644 --- a/tensor/stats/split/group.go +++ b/tensor/stats/split/group.go @@ -7,130 +7,95 @@ package split //go:generate core generate import ( - "log" - "slices" + "strconv" - "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" ) // All returns a single "split" with all of the rows in given view // useful for leveraging the aggregation management functions in splits -func All(ix *table.Table) *table.Splits { - spl := &table.Splits{} - spl.Levels = []string{"All"} - spl.New(ix.Table, []string{"All"}, ix.Indexes...) - return spl -} - -// GroupByIndex returns a new Splits set based on the groups of values -// across the given set of column indexes. -// Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupByIndex(ix *table.Table, colIndexes []int) *table.Splits { - nc := len(colIndexes) - if nc == 0 || ix.Table == nil { - return nil - } - if ix.Table.ColumnNames == nil { - log.Println("split.GroupBy: Table does not have any column names -- will not work") - return nil - } - spl := &table.Splits{} - spl.Levels = make([]string, nc) - for i, ci := range colIndexes { - spl.Levels[i] = ix.Table.ColumnNames[ci] - } - srt := ix.Clone() - srt.SortStableColumns(colIndexes, true) // important for consistency - lstValues := make([]string, nc) - curValues := make([]string, nc) - var curIx *table.Table - for _, rw := range srt.Indexes { - diff := false - for i, ci := range colIndexes { - cl := ix.Table.Columns[ci] - cv := cl.String1D(rw) - curValues[i] = cv - if cv != lstValues[i] { - diff = true - } - } - if diff || curIx == nil { - curIx = spl.New(ix.Table, curValues, rw) - copy(lstValues, curValues) - } else { - curIx.AddIndex(rw) - } - } - if spl.Len() == 0 { // prevent crashing from subsequent ops: add an empty split - spl.New(ix.Table, curValues) // no rows added here - } - return spl -} +// func All(ix *table.Table) *table.Splits { +// spl := &table.Splits{} +// spl.Levels = []string{"All"} +// spl.New(ix.Table, []string{"All"}, ix.Indexes...) +// return spl +// } -// GroupBy returns a new Splits set based on the groups of values -// across the given set of column names. -// Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupBy(ix *table.Table, columns ...string) *table.Splits { - return GroupByIndex(ix, errors.Log1(ix.Table.ColumnIndexesByNames(columns...))) +// TableGroups does [Groups] on given columns from table. +func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { + dv := table.NewView(dt) + // important for consistency across columns, to do full outer product sort first. + dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) + Groups(dir, dv.ColumnList(columns...)...) } -// GroupByFunc returns a new Splits set based on the given function -// which returns value(s) to group on for each row of the table. -// The function should always return the same number of values -- if -// it doesn't behavior is undefined. +// Groups generates indexes for each unique value in each of the given tensors. +// One can then use the resulting indexes for the [tensor.Indexed] indexes to +// perform computations restricted to grouped subsets of data, as in the +// [Stats] function. +// It creates subdirectories in given [datafs] for each tensor +// passed in here, using the metadata Name property for names (index if empty). +// Within each subdirectory there are int value tensors for each unique 1D +// row-wise value of elements in the input tensor, named as the string +// representation of the value, where the int tensor contains a list of +// row-wise indexes corresponding to the source rows having that value. +// Note that these indexes are directly in terms of the underlying [Tensor] data +// rows, indirected through any existing indexes on the inputs, so that +// the results can be used directly as indexes into the corresponding tensor data. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func GroupByFunc(ix *table.Table, fun func(row int) []string) *table.Splits { - if ix.Table == nil { - return nil - } +func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { - // save function values - funvals := make(map[int][]string, ix.Len()) - nv := 0 // number of valeus - for _, rw := range ix.Indexes { - sv := fun(rw) - if nv == 0 { - nv = len(sv) + makeIdxs := func(dir *datafs.Data, srt *tensor.Indexed, val string, start, r int) { + n := r - start + it := datafs.NewValue[int](dir, val, n) + for j := range n { + it.SetIntRowCell(srt.Indexes[start+j], j, 0) // key to indirect through sort indexes } - funvals[rw] = slices.Clone(sv) } - srt := ix.Clone() - srt.SortStable(func(et *table.Table, i, j int) bool { // sort based on given function values - fvi := funvals[i] - fvj := funvals[j] - for fi := 0; fi < nv; fi++ { - if fvi[fi] < fvj[fi] { - return true - } else if fvi[fi] > fvj[fi] { - return false - } + for i, tsr := range tsrs { + nr := tsr.NumRows() + if nr == 0 { + continue } - return false - }) - - // now do our usual grouping operation - spl := &table.Splits{} - lstValues := make([]string, nv) - var curIx *table.Table - for _, rw := range srt.Indexes { - curValues := funvals[rw] - diff := (curIx == nil) - if !diff { - for fi := 0; fi < nv; fi++ { - if lstValues[fi] != curValues[fi] { - diff = true - break + nm := tsr.Tensor.Metadata().GetName() + if nm == "" { + nm = strconv.Itoa(i) + } + td, _ := dir.Mkdir(nm) + srt := tsr.CloneIndexes() + srt.SortStable(tensor.Ascending) + start := 0 + if tsr.Tensor.IsString() { + lastVal := srt.StringRowCell(0, 0) + for r := range nr { + v := srt.StringRowCell(r, 0) + if v != lastVal { + makeIdxs(td, srt, lastVal, start, r) + start = r + lastVal = v } } - } - if diff { - curIx = spl.New(ix.Table, curValues, rw) - copy(lstValues, curValues) + if start != nr-1 { + makeIdxs(td, srt, lastVal, start, nr) + } } else { - curIx.AddIndex(rw) + lastVal := srt.FloatRowCell(0, 0) + for r := range nr { + v := srt.FloatRowCell(r, 0) + if v != lastVal { + makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, r) + start = r + lastVal = v + } + } + if start != nr-1 { + makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, nr) + } } } - return spl } + +// todo: make an outer-product function? diff --git a/tensor/stats/split/random.go b/tensor/stats/split/random.go index 3123c47aa4..b34fe4db72 100644 --- a/tensor/stats/split/random.go +++ b/tensor/stats/split/random.go @@ -4,14 +4,7 @@ package split -import ( - "fmt" - "math" - - "cogentcore.org/core/tensor/table" - "gonum.org/v1/gonum/floats" -) - +/* // Permuted generates permuted random splits of table rows, using given list of probabilities, // which will be normalized to sum to 1 (error returned if sum = 0) // names are optional names for each split (e.g., Train, Test) which will be @@ -61,3 +54,4 @@ func Permuted(ix *table.Table, probs []float64, names []string) (*table.Splits, } return spl, nil } +*/ diff --git a/tensor/stats/split/random_test.go b/tensor/stats/split/random_test.go index 9055caf8c9..331d50f46f 100644 --- a/tensor/stats/split/random_test.go +++ b/tensor/stats/split/random_test.go @@ -4,14 +4,7 @@ package split -import ( - "testing" - - "cogentcore.org/core/tensor/table" - - "github.com/stretchr/testify/assert" -) - +/* func TestPermuted(t *testing.T) { dt := table.NewTable().SetNumRows(25) dt.AddStringColumn("Name") @@ -41,3 +34,4 @@ func TestPermuted(t *testing.T) { assert.Equal(t, 13, len(spl.Splits[1].Indexes)) assert.Equal(t, 6, len(spl.Splits[2].Indexes)) } +*/ diff --git a/tensor/stats/split/agg_test.go b/tensor/stats/split/split_test.go similarity index 56% rename from tensor/stats/split/agg_test.go rename to tensor/stats/split/split_test.go index 9cb9d34b5d..403a03fd91 100644 --- a/tensor/stats/split/agg_test.go +++ b/tensor/stats/split/split_test.go @@ -7,37 +7,40 @@ package split import ( "testing" - "cogentcore.org/core/tensor/stats/stats" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" - "github.com/stretchr/testify/assert" ) -func TestAgg(t *testing.T) { +func TestGroup(t *testing.T) { dt := table.NewTable().SetNumRows(4) - dt.AddStringColumn("Group") + dt.AddStringColumn("Name") dt.AddFloat32Column("Value") - for i := 0; i < dt.Rows; i++ { + for i := range dt.NumRows() { gp := "A" if i >= 2 { gp = "B" } - dt.SetString("Group", i, gp) - dt.SetFloat("Value", i, float64(i)) + dt.Column("Name").SetStringRowCell(gp, i, 0) + dt.Column("Value").SetFloatRowCell(float64(i), i, 0) } - ix := table.NewIndexed(dt) - spl := GroupBy(ix, "Group") - assert.Equal(t, 2, len(spl.Splits)) + dir, _ := datafs.NewDir("Group") + TableGroups(dir, dt, "Name") - AggColumn(spl, "Value", stats.Mean) + ixs := dir.FlatValuesFunc(nil) + assert.Equal(t, []int{0, 1}, ixs[0].Tensor.(*tensor.Int).Values) + assert.Equal(t, []int{2, 3}, ixs[1].Tensor.(*tensor.Int).Values) - st := spl.AggsToTable(table.ColumnNameOnly) - assert.Equal(t, 0.5, st.Float("Value", 0)) - assert.Equal(t, 2.5, st.Float("Value", 1)) - assert.Equal(t, "A", st.StringValue("Group", 0)) - assert.Equal(t, "B", st.StringValue("Group", 1)) + // AggColumn(spl, "Value", stats.Mean) + // st := spl.AggsToTable(table.ColumnNameOnly) + // assert.Equal(t, 0.5, st.Float("Value", 0)) + // assert.Equal(t, 2.5, st.Float("Value", 1)) + // assert.Equal(t, "A", st.StringValue("Group", 0)) + // assert.Equal(t, "B", st.StringValue("Group", 1)) } +/* func TestAggEmpty(t *testing.T) { dt := table.NewTable().SetNumRows(4) dt.AddStringColumn("Group") @@ -64,3 +67,4 @@ func TestAggEmpty(t *testing.T) { t.Error("AggsToTable should not be nil!") } } +*/ diff --git a/tensor/stats/split/splits.go b/tensor/stats/split/splits.go index 56f5743c16..ea53ed4deb 100644 --- a/tensor/stats/split/splits.go +++ b/tensor/stats/split/splits.go @@ -4,13 +4,7 @@ package split -import ( - "fmt" - "slices" - "sort" - "strings" -) - +/* // SplitAgg contains aggregation results for splits type SplitAgg struct { @@ -56,7 +50,7 @@ type SplitAgg struct { type Splits struct { // the list of index views for each split - Splits []*Table + Splits []*table.Table // levels of indexes used to organize the splits -- each split contains the full outer product across these index levels. for example, if the split was generated by grouping over column values, then these are the column names in order of grouping. the splits are not automatically sorted hierarchically by these levels but e.g., the GroupBy method produces that result -- use the Sort methods to explicitly sort. Levels []string @@ -501,3 +495,5 @@ func (spl *Splits) Swap(i, j int) { ag.Aggs[i], ag.Aggs[j] = ag.Aggs[j], ag.Aggs[i] } } + +*/ diff --git a/tensor/stats/split/agg.go b/tensor/stats/split/stats.go similarity index 70% rename from tensor/stats/split/agg.go rename to tensor/stats/split/stats.go index a27de3436b..9f13b9f493 100644 --- a/tensor/stats/split/agg.go +++ b/tensor/stats/split/stats.go @@ -5,12 +5,33 @@ package split import ( - "fmt" - - "cogentcore.org/core/tensor/stats/stats" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" ) +func TableStats(dir *datafs.Data, dt *table.Table, columns ...string) { + dv := table.NewView(dt) + // important for consistency across columns, to do full outer product sort first. + dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) + Groups(dir, dv.ColumnList(columns...)...) +} + +// Stats computes the given stats function on the unique grouped values of the +// first tensor passed (the Group tensor), for each of the additional +// tensors passed (the Value tensors). +// It creates a subdirectory in given directory with the name of each value tensor, +// (if it does not yet exist), and then creates a subdirectory within that +// for the statistic name. Within that statistic directory, it creates +// a String tensor with the unique values of the Group tensor, and a +// Float64 tensor with the statistics results for each such unique group value. +// It calls the Groups function on the Group tensor first. +func Stats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { + +} + +/* + // AggIndex performs aggregation using given standard statistic (e.g., Mean) across // all splits, and returns the SplitAgg container of the results, which are also // stored in the Splits. Column is specified by index. @@ -85,3 +106,4 @@ func DescColumn(spl *table.Splits, column string) error { DescIndex(spl, colIndex) return nil } +*/ diff --git a/tensor/table/columns.go b/tensor/table/columns.go index 64b6550537..9dfc8d2add 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -37,7 +37,8 @@ func (cl *Columns) SetNumRows(rows int) *Columns { //types:add // AddColumn adds the given tensor as a column, // returning an error and not adding if the name is not unique. -// Automatically adjusts the shape to fit the current number of rows. +// Automatically adjusts the shape to fit the current number of rows, +// and calls the metadata SetName with column name. func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { err := cl.Add(name, tsr) if err != nil { @@ -45,6 +46,7 @@ func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { } rows := max(1, cl.Rows) tsr.SetNumRows(rows) + tsr.Metadata().SetName(name) return nil } diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 0b2ab87b6d..03e029c932 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -136,15 +136,7 @@ func (dt *Table) SortStableFunc(cmp func(dt *Table, i, j int) int) { // and optionally using a stable sort. // Uses first cell of higher dimensional data. func (dt *Table) SortColumns(ascending, stable bool, columns ...string) { //types:add - nc := len(columns) - cis := make([]int, 0, nc) - for _, cn := range columns { - ci := dt.ColumnIndex(cn) - if ci >= 0 { - cis = append(cis, ci) - } - } - dt.SortColumnIndexes(ascending, stable, cis...) + dt.SortColumnIndexes(ascending, stable, dt.ColumnIndexList(columns...)...) } // SortColumnIndexes sorts the indexes into our Table according to values in diff --git a/tensor/table/table.go b/tensor/table/table.go index a276ee6d1f..ae27afb406 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -42,21 +42,25 @@ type Table struct { //types:add } // NewTable returns a new Table with its own (empty) set of Columns. -// Can pass an optional name which sets metadata. +// Can pass an optional name which calls metadata SetName. func NewTable(name ...string) *Table { dt := &Table{} dt.Columns = NewColumns() if len(name) > 0 { - dt.Meta.Set("name", name[0]) + dt.Meta.SetName(name[0]) } return dt } // NewView returns a new Table with its own Indexed view into the // same underlying set of Column tensor data as the source table. -// Indexes are nil in the new Table, resulting in default full sequential view. +// Indexes are copied from the existing table -- use Sequential +// to reset to full sequential view. func NewView(src *Table) *Table { dt := &Table{Columns: src.Columns} + if src.Indexes != nil { + dt.Indexes = slices.Clone(src.Indexes) + } dt.Meta.Copy(src.Meta) return dt } @@ -105,6 +109,19 @@ func (dt *Table) ColumnByIndex(idx int) *tensor.Indexed { return tensor.NewIndexed(cl, dt.Indexes) } +// ColumnList returns a list of tensors with given column names, +// as [tensor.Indexed] with the shared [Table.Indexes] from this table. +func (dt *Table) ColumnList(names ...string) []*tensor.Indexed { + list := make([]*tensor.Indexed, 0, len(names)) + for _, nm := range names { + cl := dt.Column(nm) + if cl != nil { + list = append(list, cl) + } + } + return list +} + // ColumnName returns the name of given column. func (dt *Table) ColumnName(i int) string { return dt.Columns.Keys[i] @@ -115,6 +132,18 @@ func (dt *Table) ColumnIndex(name string) int { return dt.Columns.IndexByKey(name) } +// ColumnIndexList returns a list of indexes to columns of given names. +func (dt *Table) ColumnIndexList(names ...string) []int { + list := make([]int, 0, len(names)) + for _, nm := range names { + ci := dt.ColumnIndex(nm) + if ci >= 0 { + list = append(list, ci) + } + } + return list +} + // AddColumn adds a new column to the table, of given type and column name // (which must be unique). If no cellSizes are specified, it holds scalar values, // otherwise the cells are n-dimensional tensors of given size. From 49e2a7c0e3eda671e975ed8e320980338377eb2e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 15 Sep 2024 22:47:26 -0700 Subject: [PATCH 043/311] group stats working --- tensor/datafs/dir.go | 21 +++- tensor/datafs/stats.go | 115 --------------------- tensor/examples/datafs-sim/sim.go | 9 +- tensor/stats/split/group.go | 21 ++-- tensor/stats/split/stats.go | 26 ----- tensor/stats/stats/group.go | 164 ++++++++++++++++++++++++++++++ tensor/stats/stats/group_test.go | 72 +++++++++++++ 7 files changed, 273 insertions(+), 155 deletions(-) delete mode 100644 tensor/datafs/stats.go create mode 100644 tensor/stats/stats/group.go create mode 100644 tensor/stats/stats/group_test.go diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index b99bd302dd..d2c641f366 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -43,8 +43,7 @@ func NewDir(name string, parent ...*Data) (*Data, error) { // Item returns data item in given directory by name. // This is for fast access and direct usage of known -// items, and it will panic if item is not found or -// this data is not a directory. +// items, and it will panic if this data is not a directory. func (d *Data) Item(name string) *Data { return d.Dir.ValueByKey(name) } @@ -306,7 +305,10 @@ func (d *Data) Add(it *Data) error { } // Mkdir creates a new directory with the specified name. -// The only error is if this item is not a directory. +// Returns an error if this item is not a directory, +// or if the name is already used within this directory. +// See [Data.RecycleDir] for a version ensures a directory +// exists whether it needs to be made or already does. func (d *Data) Mkdir(name string) (*Data, error) { if err := d.mustDir("Mkdir", name); err != nil { return nil, err @@ -314,6 +316,19 @@ func (d *Data) Mkdir(name string) (*Data, error) { return NewDir(name, d) } +// RecycleDir creates a new directory with the specified name +// if it doesn't already exist, otherwise returns the existing one. +// It only returns an error is if this item is not a directory. +func (d *Data) RecycleDir(name string) (*Data, error) { + if err := d.mustDir("RecycleDir", name); err != nil { + return nil, err + } + if cd := d.Dir.ValueByKey(name); cd != nil { + return cd, nil + } + return NewDir(name, d) +} + // GetDirTable gets the DirTable as a [table.Table] for this directory item, // with columns as the Tensor values elements in the directory // and any subdirectories, from FlatItemsFunc using given filter function. diff --git a/tensor/datafs/stats.go b/tensor/datafs/stats.go deleted file mode 100644 index 4c2a87be88..0000000000 --- a/tensor/datafs/stats.go +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build not - -package datafs - -import ( - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/stats" - "cogentcore.org/core/tensor/table" -) - -/* - -// StdStats are the standard descriptive stats computed in StdStatsData, -// for one-dimensional tensors. For higher-dimensional cases, the last 3 -// quartile-based ones are excluded because they are not compatible. -var StdStats = []Stats{stats.Count, stats.Mean, stats.Std, stats.Sem, stats.Min, stats.Max, stats.Q1, stats.Median, stats.Q3} - -// StdStatsData adds standard descriptive statistics for given tensor -// to the given [datafs] directory. -// This is an easy way to provide a comprehensive description of data. -// Stats are in StdStats list: Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3 -func StdStatsData(dir *datafs.Data, tsr *tensor.Indexed) { - /* - st := ix.Table - col := st.Columns[colIndex] - stats := DescStats - if col.NumDims() > 1 { // nd cannot do qiles - stats = DescStatsND - } - nAgg := len(stats) - dt := table.NewTable().SetNumRows(nAgg) - dt.AddStringColumn("Stat") - dt.AddFloat64TensorColumn(st.ColumnNames[colIndex], col.Shape().Sizes[1:], col.Shape().Names[1:]...) - dtnm := dt.Columns[0] - dtst := dt.Columns[1] - _, csz := col.RowCellSize() - for i, styp := range DescStatsND { - // ag := StatIndex(ix, colIndex, styp) - ag := 0.0 - si := i * csz - for j := 0; j < csz; j++ { - dtst.SetFloat1D(si+j, ag[j]) - } - dtnm.SetString1D(i, styp.String()) - } - if col.NumDims() == 1 { - sq := len(DescStatsND) - qs := []float64{.25, .5, .75} - qvs := QuantilesIndex(ix, colIndex, qs) - for i, qv := range qvs { - dtst.SetFloat1D(sq+i, qv) - dtnm.SetString1D(sq+i, DescStats[sq+i].String()) - } - } - */ - return ix.Table // dt -} - -// DescAll returns a table of standard descriptive stats for -// all numeric columns in given table, operating over all non-Null, non-NaN elements -// in each column. -func DescAll(ix *table.Table) *table.Table { - /* - st := ix.Table - nAgg := len(DescStats) - dt := table.NewTable().SetNumRows(nAgg) - dt.AddStringColumn("Stat") - for ci := range st.Columns { - col := st.Columns[ci] - if col.IsString() { - continue - } - dt.AddFloat64TensorColumn(st.ColumnNames[ci], col.Shape().Sizes[1:], col.Shape().Names[1:]...) - } - dtnm := dt.Columns[0] - dtci := 1 - qs := []float64{.25, .5, .75} - sq := len(DescStatsND) - for ci := range st.Columns { - col := st.Columns[ci] - if col.IsString() { - continue - } - _, csz := col.RowCellSize() - dtst := dt.Columns[dtci] - for i, styp := range DescStatsND { - ag := StatIndex(ix, ci, styp) - ag := 0.0 - si := i * csz - for j := 0; j < csz; j++ { - dtst.SetFloat1D(si+j, ag[j]) - } - if dtci == 1 { - dtnm.SetString1D(i, styp.String()) - } - } - if col.NumDims() == 1 { - qvs := QuantilesIndex(ix, ci, qs) - for i, qv := range qvs { - dtst.SetFloat1D(sq+i, qv) - dtnm.SetString1D(sq+i, DescStats[sq+i].String()) - } - } - dtci++ - } - */ - return ix.Table // dt -} - -*/ - diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 70cf1526a4..6b6b270a26 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -14,7 +14,6 @@ import ( "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" "cogentcore.org/core/tensor/datafs" - "cogentcore.org/core/tensor/stats/split" "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/table" ) @@ -153,8 +152,8 @@ func (ss *Sim) EpochDone() { } func main() { - testGroup() - return + // testGroup() + // return ss := &Sim{} ss.ConfigAll() ss.Run() @@ -176,7 +175,9 @@ func testGroup() { dt.Column("Value").SetFloatRowCell(float64(i), i, 0) } dir, _ := datafs.NewDir("Group") - split.TableGroups(dir, dt, "Name") + stats.TableGroups(dir, dt, "Name") + + stats.TableGroupStats(dir, stats.Mean.FuncName(), dt, "Value") databrowser.NewBrowserWindow(dir, "Group") core.Wait() diff --git a/tensor/stats/split/group.go b/tensor/stats/split/group.go index 452f8389c8..7bd5c53cc6 100644 --- a/tensor/stats/split/group.go +++ b/tensor/stats/split/group.go @@ -9,6 +9,7 @@ package split import ( "strconv" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" @@ -34,19 +35,25 @@ func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { // Groups generates indexes for each unique value in each of the given tensors. // One can then use the resulting indexes for the [tensor.Indexed] indexes to // perform computations restricted to grouped subsets of data, as in the -// [Stats] function. -// It creates subdirectories in given [datafs] for each tensor -// passed in here, using the metadata Name property for names (index if empty). -// Within each subdirectory there are int value tensors for each unique 1D +// [Stats] function. See [GroupCombined] for function that makes a combined +// "Combined" Group that has a unique group for each _combination_ of +// the groups created by this function. +// It creates subdirectories in a "Groups" directory within given [datafs], +// for each tensor passed in here, using the metadata Name property for +// names (index if empty). +// Within each subdirectory there are int tensors for each unique 1D // row-wise value of elements in the input tensor, named as the string // representation of the value, where the int tensor contains a list of // row-wise indexes corresponding to the source rows having that value. // Note that these indexes are directly in terms of the underlying [Tensor] data // rows, indirected through any existing indexes on the inputs, so that -// the results can be used directly as indexes into the corresponding tensor data. +// the results can be used directly as Indexes into the corresponding tensor data. // Uses a stable sort on columns, so ordering of other dimensions is preserved. func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { - + gd, err := dir.RecycleDir("Groups") + if errors.Log(err) != nil { + return + } makeIdxs := func(dir *datafs.Data, srt *tensor.Indexed, val string, start, r int) { n := r - start it := datafs.NewValue[int](dir, val, n) @@ -64,7 +71,7 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { if nm == "" { nm = strconv.Itoa(i) } - td, _ := dir.Mkdir(nm) + td, _ := dir.Mkdir(gd) srt := tsr.CloneIndexes() srt.SortStable(tensor.Ascending) start := 0 diff --git a/tensor/stats/split/stats.go b/tensor/stats/split/stats.go index 9f13b9f493..cbe5eb338e 100644 --- a/tensor/stats/split/stats.go +++ b/tensor/stats/split/stats.go @@ -4,32 +4,6 @@ package split -import ( - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/datafs" - "cogentcore.org/core/tensor/table" -) - -func TableStats(dir *datafs.Data, dt *table.Table, columns ...string) { - dv := table.NewView(dt) - // important for consistency across columns, to do full outer product sort first. - dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) - Groups(dir, dv.ColumnList(columns...)...) -} - -// Stats computes the given stats function on the unique grouped values of the -// first tensor passed (the Group tensor), for each of the additional -// tensors passed (the Value tensors). -// It creates a subdirectory in given directory with the name of each value tensor, -// (if it does not yet exist), and then creates a subdirectory within that -// for the statistic name. Within that statistic directory, it creates -// a String tensor with the unique values of the Group tensor, and a -// Float64 tensor with the statistics results for each such unique group value. -// It calls the Groups function on the Group tensor first. -func Stats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { - -} - /* // AggIndex performs aggregation using given standard statistic (e.g., Mean) across diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go new file mode 100644 index 0000000000..b7460403da --- /dev/null +++ b/tensor/stats/stats/group.go @@ -0,0 +1,164 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "strconv" + "strings" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" + "cogentcore.org/core/tensor/table" +) + +// All returns a single "split" with all of the rows in given view +// useful for leveraging the aggregation management functions in splits +// func All(ix *table.Table) *table.Splits { +// spl := &table.Splits{} +// spl.Levels = []string{"All"} +// spl.New(ix.Table, []string{"All"}, ix.Indexes...) +// return spl +// } + +// TableGroups does [Groups] on given columns from table. +func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { + dv := table.NewView(dt) + // important for consistency across columns, to do full outer product sort first. + dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) + Groups(dir, dv.ColumnList(columns...)...) +} + +// Groups generates indexes for each unique value in each of the given tensors. +// One can then use the resulting indexes for the [tensor.Indexed] indexes to +// perform computations restricted to grouped subsets of data, as in the +// [Stats] function. See [GroupCombined] for function that makes a combined +// "Combined" Group that has a unique group for each _combination_ of +// the groups created by this function. +// It creates subdirectories in a "Groups" directory within given [datafs], +// for each tensor passed in here, using the metadata Name property for +// names (index if empty). +// Within each subdirectory there are int tensors for each unique 1D +// row-wise value of elements in the input tensor, named as the string +// representation of the value, where the int tensor contains a list of +// row-wise indexes corresponding to the source rows having that value. +// Note that these indexes are directly in terms of the underlying [Tensor] data +// rows, indirected through any existing indexes on the inputs, so that +// the results can be used directly as Indexes into the corresponding tensor data. +// Uses a stable sort on columns, so ordering of other dimensions is preserved. +func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { + gd, err := dir.RecycleDir("Groups") + if errors.Log(err) != nil { + return + } + makeIdxs := func(dir *datafs.Data, srt *tensor.Indexed, val string, start, r int) { + n := r - start + it := datafs.NewValue[int](dir, val, n) + for j := range n { + it.SetIntRowCell(srt.Indexes[start+j], j, 0) // key to indirect through sort indexes + } + } + + for i, tsr := range tsrs { + nr := tsr.NumRows() + if nr == 0 { + continue + } + nm := tsr.Tensor.Metadata().GetName() + if nm == "" { + nm = strconv.Itoa(i) + } + td, _ := gd.Mkdir(nm) + srt := tsr.CloneIndexes() + srt.SortStable(tensor.Ascending) + start := 0 + if tsr.Tensor.IsString() { + lastVal := srt.StringRowCell(0, 0) + for r := range nr { + v := srt.StringRowCell(r, 0) + if v != lastVal { + makeIdxs(td, srt, lastVal, start, r) + start = r + lastVal = v + } + } + if start != nr-1 { + makeIdxs(td, srt, lastVal, start, nr) + } + } else { + lastVal := srt.FloatRowCell(0, 0) + for r := range nr { + v := srt.FloatRowCell(r, 0) + if v != lastVal { + makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, r) + start = r + lastVal = v + } + } + if start != nr-1 { + makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, nr) + } + } + } +} + +// todo: make an outer-product function? + +func TableGroupStats(dir *datafs.Data, stat string, dt *table.Table, columns ...string) { + GroupStats(dir, stat, dt.ColumnList(columns...)...) +} + +// GroupStats computes the given stats function on the unique grouped indexes +// in the "Groups" directory found within the given directory, applied to each +// of the value tensors passed here. +// It creates a "Stats" subdirectory in given directory, with +// subdirectories with the name of each value tensor (if it does not +// yet exist), and then creates a subdirectory within that +// for the statistic name. Within that statistic directory, it creates +// a String "Group" tensor with the unique values of the Group tensor, +// and a aligned Float64 tensor with the statistics results for each such unique group value. +func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { + gd, err := dir.RecycleDir("Groups") + if errors.Log(err) != nil { + return + } + sd, err := dir.RecycleDir("Stats") + if errors.Log(err) != nil { + return + } + stnm := stat + spl := strings.Split(stat, ".") + if len(spl) == 2 { + stnm = spl[1] + } + stout := tensor.NewFloat64Scalar(0) + groups := gd.ItemsFunc(nil) + for _, gp := range groups { + ggd, _ := gd.RecycleDir(gp.Name()) + vals := ggd.ValuesFunc(nil) + nv := len(vals) + if nv == 0 { + continue + } + sgd, _ := sd.RecycleDir(gp.Name()) + gv := sgd.Item("Group") + if gv == nil { + gtsr := datafs.NewValue[string](sgd, "Group", nv) + for i, v := range vals { + gtsr.SetStringRowCell(v.Tensor.Metadata().GetName(), i, 0) + } + } + for _, tsr := range tsrs { + vd, _ := sgd.RecycleDir(tsr.Tensor.Metadata().GetName()) + sv := datafs.NewValue[float64](vd, stnm, nv) + for i, v := range vals { + idx := v.Tensor.(*tensor.Int).Values + sg := tensor.NewIndexed(tsr.Tensor, idx) + tensor.Call(stat, sg, stout) + sv.SetFloatRowCell(stout.Float1D(0), i, 0) + } + } + } +} diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go new file mode 100644 index 0000000000..6a7bf809d3 --- /dev/null +++ b/tensor/stats/stats/group_test.go @@ -0,0 +1,72 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "testing" + + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" + "cogentcore.org/core/tensor/table" + "github.com/stretchr/testify/assert" +) + +func TestGroup(t *testing.T) { + dt := table.NewTable().SetNumRows(4) + dt.AddStringColumn("Name") + dt.AddFloat32Column("Value") + for i := range dt.NumRows() { + gp := "A" + if i >= 2 { + gp = "B" + } + dt.Column("Name").SetStringRowCell(gp, i, 0) + dt.Column("Value").SetFloatRowCell(float64(i), i, 0) + } + dir, _ := datafs.NewDir("Group") + TableGroups(dir, dt, "Name") + + ixs := dir.FlatValuesFunc(nil) + assert.Equal(t, []int{0, 1}, ixs[0].Tensor.(*tensor.Int).Values) + assert.Equal(t, []int{2, 3}, ixs[1].Tensor.(*tensor.Int).Values) + + TableGroupStats(dir, Mean.FuncName(), dt, "Value") + + // AggColumn(spl, "Value", stats.Mean) + // st := spl.AggsToTable(table.ColumnNameOnly) + // assert.Equal(t, 0.5, st.Float("Value", 0)) + // assert.Equal(t, 2.5, st.Float("Value", 1)) + // assert.Equal(t, "A", st.StringValue("Group", 0)) + // assert.Equal(t, "B", st.StringValue("Group", 1)) +} + +/* +func TestAggEmpty(t *testing.T) { + dt := table.NewTable().SetNumRows(4) + dt.AddStringColumn("Group") + dt.AddFloat32Column("Value") + for i := 0; i < dt.Rows; i++ { + gp := "A" + if i >= 2 { + gp = "B" + } + dt.SetString("Group", i, gp) + dt.SetFloat("Value", i, float64(i)) + } + ix := table.NewIndexed(dt) + ix.Filter(func(et *table.Table, row int) bool { + return false // exclude all + }) + spl := GroupBy(ix, "Group") + assert.Equal(t, 1, len(spl.Splits)) + + AggColumn(spl, "Value", stats.Mean) + + st := spl.AggsToTable(table.ColumnNameOnly) + if st == nil { + t.Error("AggsToTable should not be nil!") + } +} +*/ From 5c04ee706f333a97e93abee13125f1c0941c296e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 10:41:03 -0700 Subject: [PATCH 044/311] removed splits, descriptive stats in stats, updated the planets example. basic data all looking good now. --- tensor/README.md | 13 +- tensor/examples/datafs-sim/sim.go | 7 +- tensor/examples/dataproc/dataproc.go | 140 ----- .../examples/{dataproc => planets}/README.md | 0 .../{dataproc => planets}/planets.csv | 0 tensor/examples/planets/planets.go | 102 ++++ tensor/stats/README.md | 12 +- tensor/stats/metric/README.md | 15 +- tensor/stats/metric/matrix.go | 10 +- tensor/stats/metric/metric_test.go | 2 +- tensor/stats/split/README.md | 5 - tensor/stats/split/doc.go | 10 - tensor/stats/split/group.go | 108 ---- tensor/stats/split/random.go | 57 -- tensor/stats/split/random_test.go | 37 -- tensor/stats/split/split_test.go | 70 --- tensor/stats/split/splits.go | 499 ------------------ tensor/stats/split/stats.go | 83 --- tensor/stats/stats/README.md | 47 +- tensor/stats/stats/describe.go | 63 +++ tensor/stats/stats/group.go | 62 ++- tensor/stats/stats/quantiles.go | 6 +- tensor/stats/stats/stats.go | 13 + 23 files changed, 301 insertions(+), 1060 deletions(-) delete mode 100644 tensor/examples/dataproc/dataproc.go rename tensor/examples/{dataproc => planets}/README.md (100%) rename tensor/examples/{dataproc => planets}/planets.csv (100%) create mode 100644 tensor/examples/planets/planets.go delete mode 100644 tensor/stats/split/README.md delete mode 100644 tensor/stats/split/doc.go delete mode 100644 tensor/stats/split/group.go delete mode 100644 tensor/stats/split/random.go delete mode 100644 tensor/stats/split/random_test.go delete mode 100644 tensor/stats/split/split_test.go delete mode 100644 tensor/stats/split/splits.go delete mode 100644 tensor/stats/split/stats.go create mode 100644 tensor/stats/stats/describe.go diff --git a/tensor/README.md b/tensor/README.md index 0a687c90de..8571924bcd 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -12,7 +12,7 @@ Use the `[Set]FloatRowCell` methods wherever possible, for the most efficient an The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. It is completely up to the compute function how to interpret the index. There is a Threaded version of this for parallelizable functions, and a GPU version. -All tensor package functions are registered using a single name to function map (`Funcs`). +All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensors for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. * [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. @@ -29,11 +29,12 @@ All tensor package functions are registered using a single name to function map * [bitslice](bitslice) is a Go slice of bytes `[]byte` that has methods for setting individual bits, as if it was a slice of bools, while being 8x more memory efficient. This is used for encoding null entries in `etensor`, and as a Tensor of bool / bits there as well, and is generally very useful for binary (boolean) data. * [stats](stats) implements a number of different ways of analyzing tensor and table data, including: - - [split](split) supports splitting a Table into any number of indexed sub-views and aggregating over those (i.e., pivot tables), grouping, summarizing data, etc. - - [metric](metric) provides similarity / distance metrics such as `Euclidean`, `Cosine`, or `Correlation` that operate on slices of `[]float64` or `[]float32`. - - TODO: now in metric: [simat](simat) provides similarity / distance matrix computation methods operating on `etensor.Tensor` or `etable.Table` data. The `SimMat` type holds the resulting matrix and labels for the rows and columns, which has a special `SimMatGrid` view in `etview` for visualizing labeled similarity matricies. - - TODO: where? [pca](pca) provides principal-components-analysis (PCA) and covariance matrix computation functions. - - TODO: in metric? [clust](clust) provides standard agglomerative hierarchical clustering including ability to plot results in an eplot. + - [cluster](cluster) implements agglomerative clustering of items based on [metric](metric) distance / similarity matrix data. + - [convolve](convolve) convolves data (e.g., for smoothing). + - [glm](glm) fits a general linear model for one or more dependent variables as a function of one or more independent variables. This encompasses all forms of regression. + - [histogram](histogram) bins data into groups and reports the frequency of elements in the bins. + - [metric](metric) computes similarity / distance metrics for comparing two tensors, and associated distance / similarity matrix functions, including PCA and SVD analysis functions that operate on a covariance matrix. + - [stats](stats) provides a set of standard summary statistics on a range of different data types, including basic slices of floats, to tensor and table data. It also includes the ability to extract Groups of values and generate statistics for each group, as in a "pivot table" in a spreadsheet. # Standard shapes and dimensional terminology diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 6b6b270a26..d018be4b08 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -152,8 +152,8 @@ func (ss *Sim) EpochDone() { } func main() { - // testGroup() - // return + testGroup() + return ss := &Sim{} ss.ConfigAll() ss.Run() @@ -172,12 +172,13 @@ func testGroup() { gp = "B" } dt.Column("Name").SetStringRowCell(gp, i, 0) - dt.Column("Value").SetFloatRowCell(float64(i), i, 0) + dt.Column("Value").SetFloatRowCell(float64(10*(i+1)), i, 0) } dir, _ := datafs.NewDir("Group") stats.TableGroups(dir, dt, "Name") stats.TableGroupStats(dir, stats.Mean.FuncName(), dt, "Value") + stats.TableGroupStats(dir, stats.Sem.FuncName(), dt, "Value") databrowser.NewBrowserWindow(dir, "Group") core.Wait() diff --git a/tensor/examples/dataproc/dataproc.go b/tensor/examples/dataproc/dataproc.go deleted file mode 100644 index 7e146bd0b6..0000000000 --- a/tensor/examples/dataproc/dataproc.go +++ /dev/null @@ -1,140 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "embed" - "fmt" - "math" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/core" - "cogentcore.org/core/events" - "cogentcore.org/core/icons" - "cogentcore.org/core/tensor/stats/split" - "cogentcore.org/core/tensor/stats/stats" - "cogentcore.org/core/tensor/table" - "cogentcore.org/core/tensor/tensorcore" - "cogentcore.org/core/tree" -) - -// Planets is raw data -var Planets *table.Table - -// PlanetsDesc are descriptive stats of all (non-Null) data -var PlanetsDesc *table.Table - -// PlanetsNNDesc are descriptive stats of planets where entire row is non-null -var PlanetsNNDesc *table.Table - -// GpMethodOrbit shows the median of orbital period as a function of method -var GpMethodOrbit *table.Table - -// GpMethodYear shows all stats of year described by orbit -var GpMethodYear *table.Table - -// GpMethodDecade shows number of planets found in each decade by given method -var GpMethodDecade *table.Table - -// GpDecade shows number of planets found in each decade -var GpDecade *table.Table - -//go:embed *.csv -var csv embed.FS - -// AnalyzePlanets analyzes planets.csv data following some of the examples -// given here, using pandas: -// -// https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html -func AnalyzePlanets() { - Planets = table.NewTable("planets") - Planets.OpenFS(csv, "planets.csv", table.Comma) - - PlanetsAll := table.NewIndexed(Planets) // full original data - - PlanetsDesc = stats.DescAll(PlanetsAll) // individually excludes Null values in each col, but not row-wise - PlanetsNNDesc = stats.DescAll(PlanetsAll) // standard descriptive stats for row-wise non-nulls - - byMethod := split.GroupBy(PlanetsAll, "method") - split.AggColumn(byMethod, "orbital_period", stats.Median) - GpMethodOrbit = byMethod.AggsToTable(table.AddAggName) - - byMethod.DeleteAggs() - split.DescColumn(byMethod, "year") // full desc stats of year - - byMethod.Filter(func(idx int) bool { - ag := errors.Log1(byMethod.AggByColumnName("year:Std")) - return ag.Aggs[idx][0] > 0 // exclude results with 0 std - }) - - GpMethodYear = byMethod.AggsToTable(table.AddAggName) - - byMethodDecade := split.GroupByFunc(PlanetsAll, func(row int) []string { - meth := Planets.StringValue("method", row) - yr := Planets.Float("year", row) - decade := math.Floor(yr/10) * 10 - return []string{meth, fmt.Sprintf("%gs", decade)} - }) - byMethodDecade.SetLevels("method", "decade") - - split.AggColumn(byMethodDecade, "number", stats.Sum) - - // uncomment this to switch to decade first, then method - // byMethodDecade.ReorderLevels([]int{1, 0}) - // byMethodDecade.SortLevels() - - decadeOnly := errors.Log1(byMethodDecade.ExtractLevels([]int{1})) - split.AggColumn(decadeOnly, "number", stats.Sum) - GpDecade = decadeOnly.AggsToTable(table.AddAggName) - - GpMethodDecade = byMethodDecade.AggsToTable(table.AddAggName) // here to ensure that decadeOnly didn't mess up.. - - // todo: need unstack -- should be specific to the splits data because we already have the cols and - // groups etc -- the ExtractLevels method provides key starting point. - - // todo: pivot table -- neeeds unstack function. - - // todo: could have a generic unstack-like method that takes a column for the data to turn into columns - // and another that has the data to put in the cells. -} - -func main() { - AnalyzePlanets() - - b := core.NewBody("dataproc") - tv := core.NewTabs(b) - - nt, _ := tv.NewTab("Planets Data") - tbv := tensorcore.NewTable(nt).SetTable(Planets) - b.AddTopBar(func(bar *core.Frame) { - tb := core.NewToolbar(bar) - tb.Maker(tbv.MakeToolbar) - tb.Maker(func(p *tree.Plan) { - tree.Add(p, func(w *core.Button) { - w.SetText("README").SetIcon(icons.FileMarkdown). - SetTooltip("open README help file").OnClick(func(e events.Event) { - core.TheApp.OpenURL("https://github.com/cogentcore/core/blob/main/tensor/examples/dataproc/README.md") - }) - }) - }) - }) - - nt, _ = tv.NewTab("Non-Null Rows Desc") - tensorcore.NewTable(nt).SetTable(PlanetsNNDesc) - nt, _ = tv.NewTab("All Desc") - tensorcore.NewTable(nt).SetTable(PlanetsDesc) - nt, _ = tv.NewTab("By Method Orbit") - tensorcore.NewTable(nt).SetTable(GpMethodOrbit) - nt, _ = tv.NewTab("By Method Year") - tensorcore.NewTable(nt).SetTable(GpMethodYear) - nt, _ = tv.NewTab("By Method Decade") - tensorcore.NewTable(nt).SetTable(GpMethodDecade) - nt, _ = tv.NewTab("By Decade") - tensorcore.NewTable(nt).SetTable(GpDecade) - - tv.SelectTabIndex(0) - - b.RunMainWindow() -} diff --git a/tensor/examples/dataproc/README.md b/tensor/examples/planets/README.md similarity index 100% rename from tensor/examples/dataproc/README.md rename to tensor/examples/planets/README.md diff --git a/tensor/examples/dataproc/planets.csv b/tensor/examples/planets/planets.csv similarity index 100% rename from tensor/examples/dataproc/planets.csv rename to tensor/examples/planets/planets.csv diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go new file mode 100644 index 0000000000..2e2bb3e64e --- /dev/null +++ b/tensor/examples/planets/planets.go @@ -0,0 +1,102 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "embed" + "math" + + "cogentcore.org/core/core" + "cogentcore.org/core/events" + "cogentcore.org/core/icons" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/databrowser" + "cogentcore.org/core/tensor/datafs" + "cogentcore.org/core/tensor/stats/stats" + "cogentcore.org/core/tensor/table" + "cogentcore.org/core/tree" +) + +//go:embed *.csv +var csv embed.FS + +// AnalyzePlanets analyzes planets.csv data following some of the examples +// given here, using pandas: +// +// https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html +func AnalyzePlanets(dir *datafs.Data) { + Planets := table.NewTable("planets") + Planets.OpenFS(csv, "planets.csv", tensor.Comma) + + vals := []string{"number", "orbital_period", "mass", "distance", "year"} + + stats.DescribeTable(dir, Planets, vals...) + + decade := Planets.AddFloat64Column("decade") + year := Planets.Column("year") + for row := range Planets.NumRows() { + yr := year.FloatRowCell(row, 0) + dec := math.Floor(yr/10) * 10 + decade.SetFloatRowCell(dec, row, 0) + } + + stats.TableGroups(dir, Planets, "method", "decade") + stats.TableGroupDescribe(dir, Planets, vals...) + + // byMethod := split.GroupBy(PlanetsAll, "method") + // split.AggColumn(byMethod, "orbital_period", stats.Median) + // GpMethodOrbit = byMethod.AggsToTable(table.AddAggName) + + // byMethod.DeleteAggs() + // split.DescColumn(byMethod, "year") // full desc stats of year + + // byMethod.Filter(func(idx int) bool { + // ag := errors.Log1(byMethod.AggByColumnName("year:Std")) + // return ag.Aggs[idx][0] > 0 // exclude results with 0 std + // }) + + // GpMethodYear = byMethod.AggsToTable(table.AddAggName) + + // split.AggColumn(byMethodDecade, "number", stats.Sum) + + // uncomment this to switch to decade first, then method + // byMethodDecade.ReorderLevels([]int{1, 0}) + // byMethodDecade.SortLevels() + + // decadeOnly := errors.Log1(byMethodDecade.ExtractLevels([]int{1})) + // split.AggColumn(decadeOnly, "number", stats.Sum) + // GpDecade = decadeOnly.AggsToTable(table.AddAggName) + // + // GpMethodDecade = byMethodDecade.AggsToTable(table.AddAggName) // here to ensure that decadeOnly didn't mess up.. + + // todo: need unstack -- should be specific to the splits data because we already have the cols and + // groups etc -- the ExtractLevels method provides key starting point. + + // todo: pivot table -- neeeds unstack function. + + // todo: could have a generic unstack-like method that takes a column for the data to turn into columns + // and another that has the data to put in the cells. +} + +func main() { + dir, _ := datafs.NewDir("Planets") + AnalyzePlanets(dir) + + br := databrowser.NewBrowserWindow(dir, "Planets") + b := br.Parent.(*core.Body) + b.AddTopBar(func(bar *core.Frame) { + tb := core.NewToolbar(bar) + // tb.Maker(tbv.MakeToolbar) + tb.Maker(func(p *tree.Plan) { + tree.Add(p, func(w *core.Button) { + w.SetText("README").SetIcon(icons.FileMarkdown). + SetTooltip("open README help file").OnClick(func(e events.Event) { + core.TheApp.OpenURL("https://github.com/cogentcore/core/blob/main/tensor/examples/planets/README.md") + }) + }) + }) + }) + core.Wait() +} diff --git a/tensor/stats/README.md b/tensor/stats/README.md index 9e40210158..1f8a764e09 100644 --- a/tensor/stats/README.md +++ b/tensor/stats/README.md @@ -1,15 +1,11 @@ # stats -There are several packages here for operating on vector, [tensor](../tensor), and [table](../table) data, for computing standard statistics and performing related computations, such as normalizing the data. +There are several packages here for operating on vector, [tensor](../), and [table](../table) data, for computing standard statistics and performing related computations, such as normalizing the data. -* [clust](clust) implements agglomerative clustering of items based on [simat](simat) similarity matrix data. +* [cluster](cluster) implements agglomerative clustering of items based on [metric](metric) distance / similarity matrix data. * [convolve](convolve) convolves data (e.g., for smoothing). * [glm](glm) fits a general linear model for one or more dependent variables as a function of one or more independent variables. This encompasses all forms of regression. * [histogram](histogram) bins data into groups and reports the frequency of elements in the bins. -* [metric](metric) computes similarity / distance metrics for comparing two vectors -* [norm](norm) normalizes vector data -* [pca](pca) computes principal components analysis (PCA) or singular value decomposition (SVD) on correlation matricies, which is a widely-used way of reducing the dimensionality of high-dimensional data. -* [simat](simat) computes a similarity matrix for the [metric](metric) similarity of two vectors. -* [split](split) provides grouping and aggregation functions operating on `table.Table` data, e.g., like a "pivot table" in a spreadsheet. -* [stats](stats) provides a set of standard summary statistics on a range of different data types, including basic slices of floats, to tensor and table data. +* [metric](metric) computes similarity / distance metrics for comparing two tensors, and associated distance / similarity matrix functions, including PCA and SVD analysis functions that operate on a covariance matrix. +* [stats](stats) provides a set of standard summary statistics on a range of different data types, including basic slices of floats, to tensor and table data. It also includes the ability to extract Groups of values and generate statistics for each group, as in a "pivot table" in a spreadsheet. diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index e2e409df13..783a324954 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -2,7 +2,7 @@ `metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Indexed` standard data representation. -The `Matrix` function returns a distance / similarity matrix computed from the n-dimensional "cells" of row-organized tensor data, and the `SimMat` type provides labels for displaying such matricies. +The `Matrix` function returns a distance / similarity matrix computed from the n-dimensional "cells" of row-organized tensor data, and the `LabeledMatrix` type provides labels for displaying such matricies. ## Metrics @@ -35,3 +35,16 @@ All metric functions skip over NaN's, as a missing value, and use the min of the Metric functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. +## Matrix functions + +* `Matrix` computes a distance / similarity matrix using a metric function, operating on the n-dimensional sub-space patterns on a given tensor (i.e., a row-wise list of patterns). The result is a square rows x rows matrix where each cell is the metric value for the pattern at the given row. The diagonal contains the self-similarity metric. + +* `CrossMatrix` is like `Matrix` except it compares two separate lists of patterns. + +* `CovarianceMatrix` computes the _covariance matrix_ for row-wise lists of patterns, where the result is a square matrix of cells x cells size ("cells" is number of elements in the patterns per row), and each value represents the extent to which value of a given cell covaries across the rows of the tensor with the value of another cell. For example, if the rows represent time, then the covariance matrix represents the extent to which the patterns tend to move in the same way over time. + +* `PCA` and `SVD` operate on the `CovarianceMatrix` to extract the "principal components" of covariance, in terms of the _eigenvectors_ and corresponding _eigenvalues_ of this matrix. The eigenvector (component) with the largest eigenvalue is the "direction" in n-dimensional pattern space along which there is the greatest variance in the patterns across the rows. + +* `ProjectOnMatrixColumn` is a convenient function for projecting data along a vector extracted from a matrix, which allows you to project data along an eigenvector from the PCA or SVD functions. By doing this projection along the strongest 2 eigenvectors (those with the largest eigenvalues), you can visualize high-dimensional data in a 2D plot, which typically reveals important aspects of the structure of the underlying high-dimensional data, which is otherwise hard to see given the difficulty in visualizing high-dimensional spaces. + + diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 3064ad77cc..d88f946d19 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -17,7 +17,7 @@ import ( func init() { tensor.AddFunc("metric.Matrix", Matrix, 1, tensor.StringFirstArg) tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.StringFirstArg) - tensor.AddFunc("metric.CovarMatrix", CovarMatrix, 1, tensor.StringFirstArg) + tensor.AddFunc("metric.CovarianceMatrix", CovarianceMatrix, 1, tensor.StringFirstArg) } // Matrix computes the rows x rows square distance / similarity matrix @@ -88,7 +88,7 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { }, a, b, out) } -// CovarMatrix generates the cells x cells square covariance matrix +// CovarianceMatrix generates the cells x cells square covariance matrix // for all per-row cells of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns (cells). @@ -101,7 +101,7 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { // which is typical in neural network models, and use // Correlation if they are on very different scales, because it effectively rescales). // The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. -func CovarMatrix(funcName string, in, out *tensor.Indexed) { +func CovarianceMatrix(funcName string, in, out *tensor.Indexed) { rows, cells := in.RowCellSize() if rows == 0 || cells == 0 { return @@ -144,7 +144,7 @@ func CovarMatrix(funcName string, in, out *tensor.Indexed) { } } -// PCA performs the eigen decomposition of the given CovarMatrix, +// PCA performs the eigen decomposition of the given CovarianceMatrix, // using principal components analysis (PCA), which is slower than [SVD]. // The eigenvectors are same size as Covar. Each eigenvector is a column // in this 2D square matrix, ordered *lowest* to *highest* across the columns, @@ -174,7 +174,7 @@ func PCA(covar, eigenvecs, vals *tensor.Indexed) { } } -// SVD performs the eigen decomposition of the given CovarMatrix, +// SVD performs the eigen decomposition of the given CovarianceMatrix, // using singular value decomposition (SVD), which is faster than [PCA]. // The eigenvectors are same size as Covar. Each eigenvector is a column // in this 2D square matrix, ordered *highest* to *lowest* across the columns, diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 5c2a7322a1..a6634cfd12 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -109,7 +109,7 @@ func TestPCAIris(t *testing.T) { } data := dt.Column("data") covar := tensor.NewFloat64Indexed() - CovarMatrix(Correlation.FuncName(), data, covar) + CovarianceMatrix(Correlation.FuncName(), data, covar) // fmt.Printf("correl: %s\n", covar.Tensor.String()) vecs := tensor.NewFloat64Indexed() diff --git a/tensor/stats/split/README.md b/tensor/stats/split/README.md deleted file mode 100644 index 6d6cdfde63..0000000000 --- a/tensor/stats/split/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# split - -`split` provides `GroupBy`, `Agg`, `Permute` and other functions that create and populate Splits of `table.Table` data. These are powerful tools for quickly summarizing and analyzing data. - - diff --git a/tensor/stats/split/doc.go b/tensor/stats/split/doc.go deleted file mode 100644 index 5af6a8cb87..0000000000 --- a/tensor/stats/split/doc.go +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -/* -Package split provides GroupBy, Agg, Permute and other functions that -create and populate Splits of table.Table data. These are powerful -tools for quickly summarizing and analyzing data. -*/ -package split diff --git a/tensor/stats/split/group.go b/tensor/stats/split/group.go deleted file mode 100644 index 7bd5c53cc6..0000000000 --- a/tensor/stats/split/group.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package split - -//go:generate core generate - -import ( - "strconv" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/datafs" - "cogentcore.org/core/tensor/table" -) - -// All returns a single "split" with all of the rows in given view -// useful for leveraging the aggregation management functions in splits -// func All(ix *table.Table) *table.Splits { -// spl := &table.Splits{} -// spl.Levels = []string{"All"} -// spl.New(ix.Table, []string{"All"}, ix.Indexes...) -// return spl -// } - -// TableGroups does [Groups] on given columns from table. -func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { - dv := table.NewView(dt) - // important for consistency across columns, to do full outer product sort first. - dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) - Groups(dir, dv.ColumnList(columns...)...) -} - -// Groups generates indexes for each unique value in each of the given tensors. -// One can then use the resulting indexes for the [tensor.Indexed] indexes to -// perform computations restricted to grouped subsets of data, as in the -// [Stats] function. See [GroupCombined] for function that makes a combined -// "Combined" Group that has a unique group for each _combination_ of -// the groups created by this function. -// It creates subdirectories in a "Groups" directory within given [datafs], -// for each tensor passed in here, using the metadata Name property for -// names (index if empty). -// Within each subdirectory there are int tensors for each unique 1D -// row-wise value of elements in the input tensor, named as the string -// representation of the value, where the int tensor contains a list of -// row-wise indexes corresponding to the source rows having that value. -// Note that these indexes are directly in terms of the underlying [Tensor] data -// rows, indirected through any existing indexes on the inputs, so that -// the results can be used directly as Indexes into the corresponding tensor data. -// Uses a stable sort on columns, so ordering of other dimensions is preserved. -func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { - gd, err := dir.RecycleDir("Groups") - if errors.Log(err) != nil { - return - } - makeIdxs := func(dir *datafs.Data, srt *tensor.Indexed, val string, start, r int) { - n := r - start - it := datafs.NewValue[int](dir, val, n) - for j := range n { - it.SetIntRowCell(srt.Indexes[start+j], j, 0) // key to indirect through sort indexes - } - } - - for i, tsr := range tsrs { - nr := tsr.NumRows() - if nr == 0 { - continue - } - nm := tsr.Tensor.Metadata().GetName() - if nm == "" { - nm = strconv.Itoa(i) - } - td, _ := dir.Mkdir(gd) - srt := tsr.CloneIndexes() - srt.SortStable(tensor.Ascending) - start := 0 - if tsr.Tensor.IsString() { - lastVal := srt.StringRowCell(0, 0) - for r := range nr { - v := srt.StringRowCell(r, 0) - if v != lastVal { - makeIdxs(td, srt, lastVal, start, r) - start = r - lastVal = v - } - } - if start != nr-1 { - makeIdxs(td, srt, lastVal, start, nr) - } - } else { - lastVal := srt.FloatRowCell(0, 0) - for r := range nr { - v := srt.FloatRowCell(r, 0) - if v != lastVal { - makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, r) - start = r - lastVal = v - } - } - if start != nr-1 { - makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, nr) - } - } - } -} - -// todo: make an outer-product function? diff --git a/tensor/stats/split/random.go b/tensor/stats/split/random.go deleted file mode 100644 index b34fe4db72..0000000000 --- a/tensor/stats/split/random.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package split - -/* -// Permuted generates permuted random splits of table rows, using given list of probabilities, -// which will be normalized to sum to 1 (error returned if sum = 0) -// names are optional names for each split (e.g., Train, Test) which will be -// used to label the Values of the resulting Splits. -func Permuted(ix *table.Table, probs []float64, names []string) (*table.Splits, error) { - if ix == nil || ix.Len() == 0 { - return nil, fmt.Errorf("split.Random table is nil / empty") - } - np := len(probs) - if len(names) > 0 && len(names) != np { - return nil, fmt.Errorf("split.Random names not same len as probs") - } - sum := floats.Sum(probs) - if sum == 0 { - return nil, fmt.Errorf("split.Random probs sum to 0") - } - nr := ix.Len() - ns := make([]int, np) - cum := 0 - fnr := float64(nr) - for i, p := range probs { - p /= sum - per := int(math.Round(p * fnr)) - if cum+per > nr { - per = nr - cum - if per <= 0 { - break - } - } - ns[i] = per - cum += per - } - spl := &table.Splits{} - perm := ix.Clone() - perm.Permuted() - cum = 0 - spl.SetLevels("permuted") - for i, n := range ns { - nm := "" - if names != nil { - nm = names[i] - } else { - nm = fmt.Sprintf("p=%v", probs[i]/sum) - } - spl.New(ix.Table, []string{nm}, perm.Indexes[cum:cum+n]...) - cum += n - } - return spl, nil -} -*/ diff --git a/tensor/stats/split/random_test.go b/tensor/stats/split/random_test.go deleted file mode 100644 index 331d50f46f..0000000000 --- a/tensor/stats/split/random_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package split - -/* -func TestPermuted(t *testing.T) { - dt := table.NewTable().SetNumRows(25) - dt.AddStringColumn("Name") - dt.AddFloat32TensorColumn("Input", []int{5, 5}, "Y", "X") - dt.AddFloat32TensorColumn("Output", []int{5, 5}, "Y", "X") - ix := table.NewIndexed(dt) - spl, err := Permuted(ix, []float64{.5, .5}, nil) - if err != nil { - t.Error(err) - } - // for i, sp := range spl.Splits { - // fmt.Printf("split: %v name: %v len: %v idxs: %v\n", i, spl.Values[i], len(sp.Indexes), sp.Indexes) - // } - assert.Equal(t, 2, len(spl.Splits)) - assert.Contains(t, []int{12, 13}, len(spl.Splits[0].Indexes)) - assert.Contains(t, []int{12, 13}, len(spl.Splits[1].Indexes)) - - spl, err = Permuted(ix, []float64{.25, .5, .25}, []string{"test", "train", "validate"}) - if err != nil { - t.Error(err) - } - // for i, sp := range spl.Splits { - // fmt.Printf("split: %v name: %v len: %v idxs: %v\n", i, spl.Values[i], len(sp.Indexes), sp.Indexes) - // } - assert.Equal(t, 3, len(spl.Splits)) - assert.Equal(t, 6, len(spl.Splits[0].Indexes)) - assert.Equal(t, 13, len(spl.Splits[1].Indexes)) - assert.Equal(t, 6, len(spl.Splits[2].Indexes)) -} -*/ diff --git a/tensor/stats/split/split_test.go b/tensor/stats/split/split_test.go deleted file mode 100644 index 403a03fd91..0000000000 --- a/tensor/stats/split/split_test.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package split - -import ( - "testing" - - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/datafs" - "cogentcore.org/core/tensor/table" - "github.com/stretchr/testify/assert" -) - -func TestGroup(t *testing.T) { - dt := table.NewTable().SetNumRows(4) - dt.AddStringColumn("Name") - dt.AddFloat32Column("Value") - for i := range dt.NumRows() { - gp := "A" - if i >= 2 { - gp = "B" - } - dt.Column("Name").SetStringRowCell(gp, i, 0) - dt.Column("Value").SetFloatRowCell(float64(i), i, 0) - } - dir, _ := datafs.NewDir("Group") - TableGroups(dir, dt, "Name") - - ixs := dir.FlatValuesFunc(nil) - assert.Equal(t, []int{0, 1}, ixs[0].Tensor.(*tensor.Int).Values) - assert.Equal(t, []int{2, 3}, ixs[1].Tensor.(*tensor.Int).Values) - - // AggColumn(spl, "Value", stats.Mean) - // st := spl.AggsToTable(table.ColumnNameOnly) - // assert.Equal(t, 0.5, st.Float("Value", 0)) - // assert.Equal(t, 2.5, st.Float("Value", 1)) - // assert.Equal(t, "A", st.StringValue("Group", 0)) - // assert.Equal(t, "B", st.StringValue("Group", 1)) -} - -/* -func TestAggEmpty(t *testing.T) { - dt := table.NewTable().SetNumRows(4) - dt.AddStringColumn("Group") - dt.AddFloat32Column("Value") - for i := 0; i < dt.Rows; i++ { - gp := "A" - if i >= 2 { - gp = "B" - } - dt.SetString("Group", i, gp) - dt.SetFloat("Value", i, float64(i)) - } - ix := table.NewIndexed(dt) - ix.Filter(func(et *table.Table, row int) bool { - return false // exclude all - }) - spl := GroupBy(ix, "Group") - assert.Equal(t, 1, len(spl.Splits)) - - AggColumn(spl, "Value", stats.Mean) - - st := spl.AggsToTable(table.ColumnNameOnly) - if st == nil { - t.Error("AggsToTable should not be nil!") - } -} -*/ diff --git a/tensor/stats/split/splits.go b/tensor/stats/split/splits.go deleted file mode 100644 index ea53ed4deb..0000000000 --- a/tensor/stats/split/splits.go +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package split - -/* -// SplitAgg contains aggregation results for splits -type SplitAgg struct { - - // the name of the aggregation operation performed, e.g., Sum, Mean, etc - Name string - - // column index on which the aggregation was performed -- results will have same shape as cells in this column - ColumnIndex int - - // aggregation results -- outer index is length of splits, inner is the length of the cell shape for the column - Aggs [][]float64 -} - -// Splits is a list of indexed views into a given Table, that represent a particular -// way of splitting up the data, e.g., whenever a given column value changes. -// -// It is functionally equivalent to the MultiIndex in python's pandas: it has multiple -// levels of indexes as listed in the Levels field, which then have corresponding -// Values for each split. These index levels can be re-ordered, and new Splits or -// Indexeds's can be created from subsets of the existing levels. The Values are -// stored simply as string values, as this is the most general type and often -// index values are labels etc. -// -// For Splits created by the splits.GroupBy function for example, each index Level is -// the column name that the data was grouped by, and the Values for each split are then -// the values of those columns. However, any arbitrary set of levels and values can -// be used, e.g., as in the splits.GroupByFunc function. -// -// Conceptually, a given Split always contains the full "outer product" of all the -// index levels -- there is one split for each unique combination of values along each -// index level. Thus, removing one level collapses across those values and moves the -// corresponding indexes into the remaining split indexes. -// -// You can Sort and Filter based on the index values directly, to reorganize the splits -// and drop particular index values, etc. -// -// Splits also maintains Aggs aggregate values for each split, which can be computed using -// standard aggregation methods over data columns, using the split.Agg* functions. -// -// The table code contains the structural methods for managing the Splits data. -// See split package for end-user methods to generate different kinds of splits, -// and perform aggregations, etc. -type Splits struct { - - // the list of index views for each split - Splits []*table.Table - - // levels of indexes used to organize the splits -- each split contains the full outer product across these index levels. for example, if the split was generated by grouping over column values, then these are the column names in order of grouping. the splits are not automatically sorted hierarchically by these levels but e.g., the GroupBy method produces that result -- use the Sort methods to explicitly sort. - Levels []string - - // the values of the index levels associated with each split. The outer dimension is the same length as Splits, and the inner dimension is the levels. - Values [][]string - - // aggregate results, one for each aggregation operation performed -- split-level data is contained within each SplitAgg struct -- deleting a split removes these aggs but adding new splits just invalidates all existing aggs (they are automatically deleted). - Aggs []*SplitAgg - - // current Less function used in sorting - lessFunc SplitsLessFunc `copier:"-" display:"-" xml:"-" json:"-"` -} - -// SplitsLessFunc is a function used for sort comparisons that returns -// true if split i is less than split j -type SplitsLessFunc func(spl *Splits, i, j int) bool - -// Len returns number of splits -func (spl *Splits) Len() int { - return len(spl.Splits) -} - -// Table returns the table from the first split (should be same for all) -// returns nil if no splits yet -func (spl *Splits) Table() *Table { - if len(spl.Splits) == 0 { - return nil - } - return spl.Splits[0] -} - -// New adds a new split to the list for given table, and with associated -// values, which are copied before saving into Values list, and any number of rows -// from the table associated with this split (also copied). -// Any existing Aggs are deleted by this. -func (spl *Splits) New(dt *Table, values []string, rows ...int) *Table { - spl.Aggs = nil - ix := NewView(dt) - spl.Splits = append(spl.Splits, ix) - if len(rows) > 0 { - ix.Indexes = append(ix.Indexes, slices.Clone(rows)...) - } - if len(values) > 0 { - spl.Values = append(spl.Values, slices.Clone(values)) - } else { - spl.Values = append(spl.Values, nil) - } - return ix -} - -// ByValue finds split indexes by matching to split values, returns nil if not found. -// values are used in order as far as they go and any remaining values are assumed -// to match, and any empty values will match anything. Can use this to access different -// subgroups within overall set of splits. -func (spl *Splits) ByValue(values []string) []int { - var matches []int - for si, sn := range spl.Values { - sz := min(len(sn), len(values)) - match := true - for j := 0; j < sz; j++ { - if values[j] == "" { - continue - } - if values[j] != sn[j] { - match = false - break - } - } - if match { - matches = append(matches, si) - } - } - return matches -} - -// Delete deletes split at given index -- use this to coordinate deletion -// of Splits, Values, and Aggs values for given split -func (spl *Splits) Delete(idx int) { - spl.Splits = append(spl.Splits[:idx], spl.Splits[idx+1:]...) - spl.Values = append(spl.Values[:idx], spl.Values[idx+1:]...) - for _, ag := range spl.Aggs { - ag.Aggs = append(ag.Aggs[:idx], ag.Aggs[idx+1:]...) - } -} - -// Filter removes any split for which given function returns false -func (spl *Splits) Filter(fun func(idx int) bool) { - sz := len(spl.Splits) - for si := sz - 1; si >= 0; si-- { - if !fun(si) { - spl.Delete(si) - } - } -} - -// Sort sorts the splits according to the given Less function. -func (spl *Splits) Sort(lessFunc func(spl *Splits, i, j int) bool) { - spl.lessFunc = lessFunc - sort.Sort(spl) -} - -// SortLevels sorts the splits according to the current index level ordering of values -// i.e., first index level is outer sort dimension, then within that is the next, etc -func (spl *Splits) SortLevels() { - spl.Sort(func(sl *Splits, i, j int) bool { - vli := sl.Values[i] - vlj := sl.Values[j] - for k := range vli { - if vli[k] < vlj[k] { - return true - } else if vli[k] > vlj[k] { - return false - } // fallthrough - } - return false - }) -} - -// SortOrder sorts the splits according to the given ordering of index levels -// which can be a subset as well -func (spl *Splits) SortOrder(order []int) error { - if len(order) == 0 || len(order) > len(spl.Levels) { - return fmt.Errorf("table.Splits SortOrder: order length == 0 or > Levels") - } - spl.Sort(func(sl *Splits, i, j int) bool { - vli := sl.Values[i] - vlj := sl.Values[j] - for k := range order { - if vli[order[k]] < vlj[order[k]] { - return true - } else if vli[order[k]] > vlj[order[k]] { - return false - } // fallthrough - } - return false - }) - return nil -} - -// ReorderLevels re-orders the index levels according to the given new ordering indexes -// e.g., []int{1,0} will move the current level 0 to level 1, and 1 to level 0 -// no checking is done to ensure these are sensible beyond basic length test -- -// behavior undefined if so. Typically you want to call SortLevels after this. -func (spl *Splits) ReorderLevels(order []int) error { - nlev := len(spl.Levels) - if len(order) != nlev { - return fmt.Errorf("table.Splits ReorderLevels: order length != Levels") - } - old := make([]string, nlev) - copy(old, spl.Levels) - for i := range order { - spl.Levels[order[i]] = old[i] - } - for si := range spl.Values { - copy(old, spl.Values[si]) - for i := range order { - spl.Values[si][order[i]] = old[i] - } - } - return nil -} - -// ExtractLevels returns a new Splits that only has the given levels of indexes, -// in their given order, with the other levels removed and their corresponding indexes -// merged into the appropriate remaining levels. -// Any existing aggregation data is not retained in the new splits. -func (spl *Splits) ExtractLevels(levels []int) (*Splits, error) { - nlv := len(levels) - if nlv == 0 || nlv >= len(spl.Levels) { - return nil, fmt.Errorf("table.Splits ExtractLevels: levels length == 0 or >= Levels") - } - aggs := spl.Aggs - spl.Aggs = nil - ss := spl.Clone() - spl.Aggs = aggs - ss.SortOrder(levels) - // now just do the grouping by levels values - lstValues := make([]string, nlv) - curValues := make([]string, nlv) - var curIx *Table - nsp := len(ss.Splits) - for si := nsp - 1; si >= 0; si-- { - diff := false - for li := range levels { - vl := ss.Values[si][levels[li]] - curValues[li] = vl - if vl != lstValues[li] { - diff = true - } - } - if diff || curIx == nil { - curIx = ss.Splits[si] - copy(lstValues, curValues) - ss.Values[si] = slices.Clone(curValues) - } else { - curIx.Indexes = append(curIx.Indexes, ss.Splits[si].Indexes...) // absorb - ss.Delete(si) - } - } - ss.Levels = make([]string, nlv) - for li := range levels { - ss.Levels[li] = spl.Levels[levels[li]] - } - return ss, nil -} - -// Clone returns a cloned copy of our SplitAgg -func (sa *SplitAgg) Clone() *SplitAgg { - nsa := &SplitAgg{} - nsa.CopyFrom(sa) - return nsa -} - -// CopyFrom copies from other SplitAgg -- we get our own unique copy of everything -func (sa *SplitAgg) CopyFrom(osa *SplitAgg) { - sa.Name = osa.Name - sa.ColumnIndex = osa.ColumnIndex - nags := len(osa.Aggs) - if nags > 0 { - sa.Aggs = make([][]float64, nags) - for si := range osa.Aggs { - sa.Aggs[si] = slices.Clone(osa.Aggs[si]) - } - } -} - -// Clone returns a cloned copy of our splits -func (spl *Splits) Clone() *Splits { - nsp := &Splits{} - nsp.CopyFrom(spl) - return nsp -} - -// CopyFrom copies from other Splits -- we get our own unique copy of everything -func (spl *Splits) CopyFrom(osp *Splits) { - spl.Splits = make([]*Table, len(osp.Splits)) - spl.Values = make([][]string, len(osp.Values)) - for si := range osp.Splits { - spl.Splits[si] = osp.Splits[si].Clone() - spl.Values[si] = slices.Clone(osp.Values[si]) - } - spl.Levels = slices.Clone(osp.Levels) - - nag := len(osp.Aggs) - if nag > 0 { - spl.Aggs = make([]*SplitAgg, nag) - for ai := range osp.Aggs { - spl.Aggs[ai] = osp.Aggs[ai].Clone() - } - } -} - -// AddAgg adds a new set of aggregation results for the Splits -func (spl *Splits) AddAgg(name string, colIndex int) *SplitAgg { - ag := &SplitAgg{Name: name, ColumnIndex: colIndex} - spl.Aggs = append(spl.Aggs, ag) - return ag -} - -// DeleteAggs deletes all existing aggregation data -func (spl *Splits) DeleteAggs() { - spl.Aggs = nil -} - -// AggByName returns Agg results for given name, which does NOT include the -// column name, just the name given to the Agg result -// (e.g., Mean for a standard Mean agg). -// Returns error message if not found. -func (spl *Splits) AggByName(name string) (*SplitAgg, error) { - for _, ag := range spl.Aggs { - if ag.Name == name { - return ag, nil - } - } - return nil, fmt.Errorf("table.Splits AggByName: agg results named: %v not found", name) -} - -// AggByColumnName returns Agg results for given column name, -// optionally including :Name agg name appended, where Name -// is the name given to the Agg result (e.g., Mean for a standard Mean agg). -// Returns error message if not found. -func (spl *Splits) AggByColumnName(name string) (*SplitAgg, error) { - dt := spl.Table() - if dt == nil { - return nil, fmt.Errorf("table.Splits AggByColumnName: table nil") - } - nmsp := strings.Split(name, ":") - colIndex := dt.ColumnIndex(nmsp[0]) - if colIndex < 0 { - return nil, nil - } - for _, ag := range spl.Aggs { - if ag.ColumnIndex != colIndex { - continue - } - if len(nmsp) == 2 && nmsp[1] != ag.Name { - continue - } - return ag, nil - } - return nil, fmt.Errorf("table.Splits AggByColumnName: agg results named: %v not found", name) -} - -// SetLevels sets the Levels index names -- must match actual index dimensionality -// of the Values. This is automatically done by e.g., GroupBy, but must be done -// manually if creating custom indexes. -func (spl *Splits) SetLevels(levels ...string) { - spl.Levels = levels -} - -// use these for arg to ArgsToTable* -const ( - // ColumnNameOnly means resulting agg table just has the original column name, no aggregation name - ColumnNameOnly bool = true - // AddAggName means resulting agg table columns have aggregation name appended - AddAggName = false -) - -// AggsToTable returns a Table containing this Splits' aggregate data. -// Must have Levels and Aggs all created as in the split.Agg* methods. -// if colName == ColumnNameOnly, then the name of the columns for the Table -// is just the corresponding agg column name -- otherwise it also includes -// the name of the aggregation function with a : divider (e.g., Name:Mean) -func (spl *Splits) AggsToTable(colName bool) *Table { - nsp := len(spl.Splits) - if nsp == 0 { - return nil - } - dt := spl.Splits[0] - st := NewTable().SetNumRows(nsp) - for _, cn := range spl.Levels { - oc := dt.Column(cn) - if oc != nil { - st.AddColumnOfType(cn, oc.Tensor.DataType()) - } else { - st.AddStringColumn(cn) - } - } - for _, ag := range spl.Aggs { - col := dt.Columns.Values[ag.ColumnIndex] - an := "" // dt.ColumnNames[ag.ColumnIndex] // todo: need fast access! - if colName == AddAggName { - an += ":" + ag.Name - } - st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:]...) - } - for si := range spl.Splits { - cidx := 0 - for ci := range spl.Levels { - col := st.Columns.Values[cidx] - col.SetString1D(spl.Values[si][ci], si) - cidx++ - } - for _, ag := range spl.Aggs { - col := st.Columns.Values[cidx] - _, csz := col.RowCellSize() - sti := si * csz - av := ag.Aggs[si] - for j, a := range av { - col.SetFloat1D(a, sti+j) - } - cidx++ - } - } - return st -} - -// AggsToTableCopy returns a Table containing this Splits' aggregate data -// and a copy of the first row of data for each split for all non-agg cols, -// which is useful for recording other data that goes along with aggregated values. -// Must have Levels and Aggs all created as in the split.Agg* methods. -// if colName == ColumnNameOnly, then the name of the columns for the Table -// is just the corresponding agg column name -- otherwise it also includes -// the name of the aggregation function with a : divider (e.g., Name:Mean) -func (spl *Splits) AggsToTableCopy(colName bool) *Table { - nsp := len(spl.Splits) - if nsp == 0 { - return nil - } - dt := spl.Splits[0] - st := NewTable().SetNumRows(nsp) - exmap := make(map[string]struct{}) - for _, cn := range spl.Levels { - st.AddStringColumn(cn) - exmap[cn] = struct{}{} - } - for _, ag := range spl.Aggs { - col := dt.Columns.Values[ag.ColumnIndex] - an := "" // dt.ColumnNames[ag.ColumnIndex] - exmap[an] = struct{}{} - if colName == AddAggName { - an += ":" + ag.Name - } - st.AddFloat64TensorColumn(an, col.Shape().Sizes[1:]...) - } - var cpcol []string - for _, cn := range dt.Columns.Keys { - if _, ok := exmap[cn]; !ok { - cpcol = append(cpcol, cn) - col := dt.Column(cn) - st.AddColumn(cn, col.Tensor.Clone()) - } - } - for si, sidx := range spl.Splits { - cidx := 0 - for ci := range spl.Levels { - col := st.Columns.Values[cidx] - col.SetString1D(spl.Values[si][ci], si) - cidx++ - } - for _, ag := range spl.Aggs { - col := st.Columns.Values[cidx] - _, csz := col.RowCellSize() - sti := si * csz - av := ag.Aggs[si] - for j, a := range av { - col.SetFloat1D(a, sti+j) - } - cidx++ - } - if len(sidx.Indexes) > 0 { - stidx := sidx.Indexes[0] - for _, cn := range cpcol { - st.CopyCell(cn, si, dt, cn, stidx) - } - } - } - return st -} - -// Less calls the LessFunc for sorting -func (spl *Splits) Less(i, j int) bool { - return spl.lessFunc(spl, i, j) -} - -// Swap switches the indexes for i and j -func (spl *Splits) Swap(i, j int) { - spl.Splits[i], spl.Splits[j] = spl.Splits[j], spl.Splits[i] - spl.Values[i], spl.Values[j] = spl.Values[j], spl.Values[i] - for _, ag := range spl.Aggs { - ag.Aggs[i], ag.Aggs[j] = ag.Aggs[j], ag.Aggs[i] - } -} - -*/ diff --git a/tensor/stats/split/stats.go b/tensor/stats/split/stats.go deleted file mode 100644 index cbe5eb338e..0000000000 --- a/tensor/stats/split/stats.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package split - -/* - -// AggIndex performs aggregation using given standard statistic (e.g., Mean) across -// all splits, and returns the SplitAgg container of the results, which are also -// stored in the Splits. Column is specified by index. -func AggIndex(spl *table.Splits, colIndex int, stat stats.Stats) *table.SplitAgg { - ag := spl.AddAgg(stat.String(), colIndex) - for _, sp := range spl.Splits { - agv := stats.StatIndex(sp, colIndex, stat) - ag.Aggs = append(ag.Aggs, agv) - } - return ag -} - -// AggColumn performs aggregation using given standard statistic (e.g., Mean) across -// all splits, and returns the SplitAgg container of the results, which are also -// stored in the Splits. Column is specified by name; returns error for bad column name. -func AggColumn(spl *table.Splits, column string, stat stats.Stats) (*table.SplitAgg, error) { - dt := spl.Table() - if dt == nil { - return nil, fmt.Errorf("split.AggTry: No splits to aggregate over") - } - colIndex, err := dt.ColumnByIndex(column) - if err != nil { - return nil, err - } - return AggIndex(spl, colIndex, stat), nil -} - -// AggAllNumericColumns performs aggregation using given standard aggregation function across -// all splits, for all number-valued columns in the table. -func AggAllNumericColumns(spl *table.Splits, stat stats.Stats) { - dt := spl.Table() - for ci, cl := range dt.Columns { - if cl.IsString() { - continue - } - AggIndex(spl, ci, stat) - } -} - -/////////////////////////////////////////////////// -// Desc - -// DescIndex performs aggregation using standard statistics across -// all splits, and stores results in the Splits. Column is specified by index. -func DescIndex(spl *table.Splits, colIndex int) { - dt := spl.Table() - if dt == nil { - return - } - col := dt.Columns[colIndex] - sts := stats.DescStats - if col.NumDims() > 1 { // nd cannot do qiles - sts = stats.DescStatsND - } - for _, st := range sts { - AggIndex(spl, colIndex, st) - } -} - -// DescColumn performs aggregation using standard statistics across -// all splits, and stores results in the Splits. -// Column is specified by name; returns error for bad column name. -func DescColumn(spl *table.Splits, column string) error { - dt := spl.Table() - if dt == nil { - return fmt.Errorf("split.DescTry: No splits to aggregate over") - } - colIndex, err := dt.ColumnByIndex(column) - if err != nil { - return err - } - DescIndex(spl, colIndex) - return nil -} -*/ diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index c216ec52c3..da5262e2a2 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -7,7 +7,10 @@ type StatsFunc func(in, out *tensor.Indexed) For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outermost row dimension set to 1. -There is a `StatsFuncs` map of named stats funcs, which is initialized with the standard Stats per below, and any additional user-defined functions can be added to. +All stats are registered in the `tensor.Funcs` global list, and can be called using the `FuncName` method, e.g.,: +```Go +tensor.Call(Mean.FuncName(), in, out) +``` ## Stats @@ -39,12 +42,50 @@ Here is the general info associated with these function calls: `StatsFunc` is the function signature for a stats function, where the output has the same shape as the input but with the outermost row dimension size of 1, and contains the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. -Critically, the stat is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ stat for that cell across the different rows. To compute a stat on the `tensor.SubSpace` cells themselves, must call on a [tensor.New1DViewOf] the sub space. +Critically, the stat is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ stat for that cell across the different rows. To compute a stat on the `tensor.SubSpace` cells themselves, must call on a `tensor.New1DViewOf` the `RowTensor`. All stats functions skip over NaN's, as a missing value. Stats functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. - + +## Groups + +The `Groups` function (and `TableGroups` convenience function for `table.Table` columns) creates lists of indexes for each unique value in a 1D tensor, and `GroupStats` calls a stats function on those groups, thereby creating a "pivot table" that summarizes data in terms of the groups present within it. The data is stored in a [datafs](../datafs) data filesystem, which can be visualized and further manipulated. + +For example, with this data: +``` +Person Score Time +Alia 40 8 +Alia 30 12 +Ben 20 10 +Ben 10 12 +``` +The `Groups` function called on the `Person` column would create the following `datafs` structure: +``` +Groups + Person + Alia: [0,1] // int tensor + Ben: [2,3] + // other groups here if passed +``` +Then the `GroupStats` function operating on this `datafs` directory, using the `Score` and `Time` data and the `Mean` stat, followed by a second call with the `Sem` stat, would produce: +``` +Stats + Person + Person: [Alia,Ben] // string tensor of group values of Person + Score + Mean: [35, 15] // float64 tensor of means + Sem: [5, 5] + Time + Mean: [10, 11] + Sem: [1, 0.5] + // other groups here.. +``` + +The `Person` directory can be turned directly into a `table.Table` and plotted or otherwise used, in the `datafs` system and associated `databrowser`. + +See the [examples/planets](../examples/planets) example for an interactive exploration of data on exoplanets using the `Groups` functions. + ## Vectorize functions See [vecfuncs.go](vecfuncs.go) for corresponding `tensor.Vectorize` functions that are used in performing the computations. These cannot be parallelized directly due to shared writing to output accumulators, and other ordering constraints. If needed, special atomic-locking or other such techniques would be required. diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go new file mode 100644 index 0000000000..13a5bfdae0 --- /dev/null +++ b/tensor/stats/stats/describe.go @@ -0,0 +1,63 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package stats + +import ( + "strconv" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" + "cogentcore.org/core/tensor/table" +) + +// DescriptiveStats are the standard descriptive stats used in Describe function. +// Cannot apply the final 3 sort-based stats to higher-dimensional data. +var DescriptiveStats = []Stats{Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3} + +// Describe adds standard descriptive statistics for given tensor +// to the given [datafs] directory, adding a directory for each tensor +// and result tensor stats for each result. +// This is an easy way to provide a comprehensive description of data. +// The [DescriptiveStats] list is: [Count], [Mean], [Std], [Sem], +// [Min], [Max], [Q1], [Median], [Q3] +func Describe(dir *datafs.Data, tsrs ...*tensor.Indexed) { + dd, err := dir.RecycleDir("Describe") + if errors.Log(err) != nil { + return + } + for i, tsr := range tsrs { + nr := tsr.NumRows() + if nr == 0 { + continue + } + nm := tsr.Tensor.Metadata().GetName() + if nm == "" { + nm = strconv.Itoa(i) + } + td, _ := dd.Mkdir(nm) + for _, st := range DescriptiveStats { + stnm := st.String() + sv := datafs.NewValue[float64](td, stnm, 1) + tensor.Call(st.FuncName(), tsr, sv) + } + } +} + +// DescribeTable runs [Describe] on given columns in table. +func DescribeTable(dir *datafs.Data, dt *table.Table, columns ...string) { + Describe(dir, dt.ColumnList(columns...)...) +} + +// DescribeTableAll runs [Describe] on all numeric columns in given table. +func DescribeTableAll(dir *datafs.Data, dt *table.Table) { + var cols []string + for i, cl := range dt.Columns.Values { + if !cl.IsString() { + cols = append(cols, dt.ColumnName(i)) + } + } + Describe(dir, dt.ColumnList(cols...)...) +} diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index b7460403da..cb50eeb281 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -23,20 +23,12 @@ import ( // return spl // } -// TableGroups does [Groups] on given columns from table. -func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { - dv := table.NewView(dt) - // important for consistency across columns, to do full outer product sort first. - dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) - Groups(dir, dv.ColumnList(columns...)...) -} - // Groups generates indexes for each unique value in each of the given tensors. // One can then use the resulting indexes for the [tensor.Indexed] indexes to // perform computations restricted to grouped subsets of data, as in the -// [Stats] function. See [GroupCombined] for function that makes a combined +// [GroupStats] function. See [GroupCombined] for function that makes a // "Combined" Group that has a unique group for each _combination_ of -// the groups created by this function. +// the separate, independent groups created by this function. // It creates subdirectories in a "Groups" directory within given [datafs], // for each tensor passed in here, using the metadata Name property for // names (index if empty). @@ -104,21 +96,26 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { } } -// todo: make an outer-product function? - -func TableGroupStats(dir *datafs.Data, stat string, dt *table.Table, columns ...string) { - GroupStats(dir, stat, dt.ColumnList(columns...)...) +// TableGroups runs [Groups] on the given columns from given [table.Table]. +func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { + dv := table.NewView(dt) + // important for consistency across columns, to do full outer product sort first. + dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) + Groups(dir, dv.ColumnList(columns...)...) } +// todo: GroupCombined + // GroupStats computes the given stats function on the unique grouped indexes -// in the "Groups" directory found within the given directory, applied to each -// of the value tensors passed here. +// produced by the [Groups] function, in the given [datafs] directory, +// applied to each of the tensors passed here. // It creates a "Stats" subdirectory in given directory, with // subdirectories with the name of each value tensor (if it does not // yet exist), and then creates a subdirectory within that // for the statistic name. Within that statistic directory, it creates -// a String "Group" tensor with the unique values of the Group tensor, -// and a aligned Float64 tensor with the statistics results for each such unique group value. +// a String tensor with the unique values of each source [Groups] tensor, +// and a aligned Float64 tensor with the statistics results for each such +// unique group value. See the README.md file for a diagram of the results. func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { gd, err := dir.RecycleDir("Groups") if errors.Log(err) != nil { @@ -128,7 +125,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { if errors.Log(err) != nil { return } - stnm := stat + stnm := StripPackage(stat) spl := strings.Split(stat, ".") if len(spl) == 2 { stnm = spl[1] @@ -136,16 +133,17 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { stout := tensor.NewFloat64Scalar(0) groups := gd.ItemsFunc(nil) for _, gp := range groups { - ggd, _ := gd.RecycleDir(gp.Name()) + gpnm := gp.Name() + ggd, _ := gd.RecycleDir(gpnm) vals := ggd.ValuesFunc(nil) nv := len(vals) if nv == 0 { continue } - sgd, _ := sd.RecycleDir(gp.Name()) - gv := sgd.Item("Group") + sgd, _ := sd.RecycleDir(gpnm) + gv := sgd.Item(gpnm) if gv == nil { - gtsr := datafs.NewValue[string](sgd, "Group", nv) + gtsr := datafs.NewValue[string](sgd, gpnm, nv) for i, v := range vals { gtsr.SetStringRowCell(v.Tensor.Metadata().GetName(), i, 0) } @@ -162,3 +160,21 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { } } } + +// TableGroupStats runs [GroupStats] on the given columns from given [table.Table]. +func TableGroupStats(dir *datafs.Data, stat string, dt *table.Table, columns ...string) { + GroupStats(dir, stat, dt.ColumnList(columns...)...) +} + +// GroupDescribe runs standard descriptive statistics on given tensor data +// using [GroupStats] function, with [DescriptiveStats] list of stats. +func GroupDescribe(dir *datafs.Data, tsrs ...*tensor.Indexed) { + for _, st := range DescriptiveStats { + GroupStats(dir, st.FuncName(), tsrs...) + } +} + +// TableGroupDescribe runs [GroupDescribe] on the given columns from given [table.Table]. +func TableGroupDescribe(dir *datafs.Data, dt *table.Table, columns ...string) { + GroupDescribe(dir, dt.ColumnList(columns...)...) +} diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index e05e3eb346..24728e0b92 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -29,9 +29,13 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { sin := in.Clone() sin.ExcludeMissing() sin.Sort(tensor.Ascending) + out.Tensor.SetShapeFrom(qs.Tensor) sz := len(sin.Indexes) - 1 // length of our own index list + if sz <= 0 { + out.Tensor.SetZeros() + return nil + } fsz := float64(sz) - out.Tensor.SetShapeFrom(qs.Tensor) nq := qs.Tensor.Len() for i := range nq { q := qs.Float1D(i) diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index f7bd6fc319..d4eeca3e1d 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -5,6 +5,8 @@ package stats import ( + "strings" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -110,6 +112,17 @@ func (s Stats) FuncName() string { return "stats." + s.String() } +// StripPackage removes any package name from given string, +// used for naming based on FuncName() which could be custom +// or have a package prefix. +func StripPackage(name string) string { + spl := strings.Split(name, ".") + if len(spl) > 1 { + return spl[len(spl)-1] + } + return name +} + // Stat calls a standard Stats enum function on given tensors. // Output results are in the out tensor. func Stat(stat Stats, in, out *tensor.Indexed) { From e9fa59e32eeb824f4d1442431e22af864676ad1d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 13:46:53 -0700 Subject: [PATCH 045/311] starts on copy, list in datafs --- tensor/README.md | 4 +- tensor/cmd/tablecat/tablecat.go | 8 +- tensor/databrowser/filetree.go | 8 +- tensor/datafs/copy.go | 98 ++++++++++++++++++++ tensor/datafs/data.go | 12 +-- tensor/datafs/dir.go | 26 ++++++ tensor/datafs/fs.go | 46 +-------- tensor/datafs/list.go | 61 ++++++++++++ tensor/examples/datafs-sim/sim.go | 18 ++-- tensor/examples/planets/planets.go | 2 +- tensor/indexed.go | 144 ++++++++++++++++++++--------- tensor/io.go | 15 ++- tensor/stats/README.md | 2 +- tensor/stats/cluster/plot.go | 22 ++--- tensor/stats/stats/group.go | 14 +-- tensor/stats/stats/group_test.go | 4 +- tensor/stats/stats/quantiles.go | 8 +- tensor/table/README.md | 18 +++- tensor/table/io.go | 3 +- tensor/table/slicetable.go | 4 +- tensor/table/table_test.go | 20 ++-- tensor/tensor.go | 50 +++++----- tensor/tensorcore/table.go | 2 +- tensor/tensorcore/tensoreditor.go | 3 + tensor/tmath/ops.go | 16 ++-- 25 files changed, 422 insertions(+), 186 deletions(-) create mode 100644 tensor/datafs/copy.go create mode 100644 tensor/datafs/list.go diff --git a/tensor/README.md b/tensor/README.md index 8571924bcd..c75b4aa780 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -4,11 +4,11 @@ Tensor and related sub-packages provide a simple yet powerful framework for repr The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. The indexes provide a specific view onto the underlying [Tensor] data, applying to the outermost _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. -The `float64` ("Float") and `string` types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. +The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. The [cosl](../cosl) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. -Use the `[Set]FloatRowCell` methods wherever possible, for the most efficient and natural indirection through the indexes. The 1D methods on underlying tensor data do not indirect through the indexes and must be called directly on the [Tensor]. +Use the `[Set]FloatRow[Cell]` methods wherever possible, for the most efficient and natural indirection through the `tensor.Indexed` indexes. The 1D methods on underlying tensor data do not indirect through the indexes. The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. It is completely up to the compute function how to interpret the index. There is a Threaded version of this for parallelizable functions, and a GPU version. diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index 96eb32fcb8..9dad1cf3b6 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -10,12 +10,12 @@ import ( "fmt" "os" "sort" - "strconv" "cogentcore.org/core/core" - "cogentcore.org/core/tensor/stats/split" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/table" + "github.com/emer/etable/v2/split" ) var ( @@ -127,7 +127,7 @@ func AvgCat(files []string) { return } avgdt := stats.MeanTables(dts) - avgdt.SetMetaData("precision", strconv.Itoa(LogPrec)) + tensor.SetPrecision(avgdt, LogPrec) avgdt.SaveCSV(core.Filename(Output), table.Tab, table.Headers) } @@ -159,7 +159,7 @@ func AvgByColumn(files []string, column string) { split.AggIndex(spl, ci, stats.Mean) } avgdt := spl.AggsToTable(table.ColumnNameOnly) - avgdt.SetMetaData("precision", strconv.Itoa(LogPrec)) + tensor.SetPrecision(avgdt, LogPrec) avgdt.SaveCSV(core.Filename(Output), table.Tab, table.Headers) } } diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index 14a8de0c79..71d28f7c63 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -6,7 +6,6 @@ package databrowser import ( "image" - "log" "strings" "cogentcore.org/core/base/errors" @@ -98,6 +97,11 @@ func (fn *FileNode) OpenFile() error { } df := fsx.DirAndFile(string(fn.Filepath)) switch { + case fn.IsDir(): + d := DataFS(ofn) + dt := d.GetDirTable(nil) + br.NewTabTensorTable(df, dt) + br.Update() case fn.Info.Cat == fileinfo.Data: switch fn.Info.Known { case fileinfo.Tensor: @@ -176,7 +180,7 @@ func (fn *FileNode) EditFiles() { //types:add // EditFile pulls up this file in a texteditor func (fn *FileNode) EditFile() { if fn.IsDir() { - log.Printf("FileNode Edit -- cannot view (edit) directories!\n") + fn.OpenFile() return } br := ParentBrowser(fn.This) diff --git a/tensor/datafs/copy.go b/tensor/datafs/copy.go new file mode 100644 index 0000000000..9450e21d8e --- /dev/null +++ b/tensor/datafs/copy.go @@ -0,0 +1,98 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafs + +import ( + "errors" + "fmt" + "io/fs" + "time" +) + +const ( + // Preserve is used for Overwrite flag, indicating to not overwrite and preserve existing. + Preserve = false + + // Overwrite is used for Overwrite flag, indicating to overwrite existing. + Overwrite = true +) + +// CopyFromValue copies value from given source data node, cloning it. +func (d *Data) CopyFromValue(frd *Data) { + d.modTime = time.Now() + d.Data = frd.Data.Clone() +} + +// Clone returns a copy of this data item, recursively cloning directory items +// if it is a directory. +func (d *Data) Clone() *Data { + if !d.IsDir() { + cp, _ := newData(nil, d.name) + cp.Data = d.Data.Clone() + return cp + } + items := d.ItemsFunc(nil) + cp, _ := NewDir(d.name) + for _, it := range items { + cp.Add(it.Clone()) + } + return cp +} + +// Copy copies item(s) from given paths to given path or directory. +// if there are multiple from items, then to must be a directory. +// must be called on a directory node. +func (d *Data) Copy(overwrite bool, to string, from ...string) error { + if err := d.mustDir("Copy", to); err != nil { + return err + } + switch { + case to == "": + return &fs.PathError{Op: "Copy", Path: to, Err: errors.New("to location is empty")} + case len(from) == 0: + return &fs.PathError{Op: "Copy", Path: to, Err: errors.New("no from sources specified")} + } + tod, _ := d.ItemAtPath(to) + var errs []error + if len(from) > 1 && tod != nil && !tod.IsDir() { + return &fs.PathError{Op: "Copy", Path: to, Err: errors.New("multiple source items requires destination to be a directory")} + } + targd := d + targf := to + if tod != nil && tod.IsDir() { + targd = tod + targf = "" + } + for _, fr := range from { + opstr := fr + " -> " + to + frd, err := d.ItemAtPath(fr) + if err != nil { + errs = append(errs, err) + continue + } + if trg, ok := targd.Dir.ValueByKeyTry(frd.name); ok { // target exists + switch { + case trg.IsDir() && frd.IsDir(): + // todo: copy all items from frd into trg + case trg.IsDir(): // frd is not + errs = append(errs, &fs.PathError{Op: "Copy", Path: opstr, Err: errors.New("cannot copy from Value onto directory of same name")}) + case frd.IsDir(): // trg is not + errs = append(errs, &fs.PathError{Op: "Copy", Path: opstr, Err: errors.New("cannot copy from Directory onto Value of same name")}) + default: // both nodes + if overwrite { // todo: interactive!? + trg.CopyFromValue(frd) + } + } + continue + } + nw := frd.Clone() + if targf != "" { + nw.name = targf + } + fmt.Println("adding new:", nw.name, nw.String()) + targd.Add(nw) + } + return errors.Join(errs...) +} diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 7f1fed6f16..1c89261d79 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -137,7 +137,7 @@ func (d *Data) AsString() string { if d.Data == nil { return "" } - return d.Data.StringRowCell(0, 0) + return d.Data.StringRow(0) } // SetString sets scalar data value from given string. @@ -145,7 +145,7 @@ func (d *Data) SetString(v string) { if d.Data == nil { return } - d.Data.SetStringRowCell(v, 0, 0) + d.Data.SetStringRow(v, 0) } // AsFloat64 returns data as a scalar float64 (first element of tensor). @@ -153,7 +153,7 @@ func (d *Data) AsFloat64() float64 { if d.Data == nil { return 0 } - return d.Data.FloatRowCell(0, 0) + return d.Data.FloatRow(0) } // SetFloat64 sets scalar data value from given float64. @@ -161,7 +161,7 @@ func (d *Data) SetFloat64(v float64) { if d.Data == nil { return } - d.Data.SetFloatRowCell(v, 0, 0) + d.Data.SetFloatRow(v, 0) } // AsFloat32 returns data as a scalar float32 (first element of tensor). @@ -179,7 +179,7 @@ func (d *Data) AsInt() int { if d.Data == nil { return 0 } - return d.Data.IntRowCell(0, 0) + return d.Data.IntRow(0) } // SetInt sets scalar data value from given int. @@ -187,5 +187,5 @@ func (d *Data) SetInt(v int) { if d.Data == nil { return } - d.Data.SetIntRowCell(v, 0, 0) + d.Data.SetIntRow(v, 0) } diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index d2c641f366..0642a23d88 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -232,6 +232,8 @@ func (d *Data) FlatValuesAlphaFunc(fun func(item *Data) bool) []*tensor.Indexed return its } +// todo: these must handle going up the tree using .. + // DirAtPath returns directory at given relative path // from this starting dir. func (d *Data) DirAtPath(dir string) (*Data, error) { @@ -244,6 +246,30 @@ func (d *Data) DirAtPath(dir string) (*Data, error) { return sdf.(*Data), nil } +// ItemAtPath returns item at given relative path +// from this starting dir. +func (d *Data) ItemAtPath(name string) (*Data, error) { + if err := d.mustDir("ItemAtPath", name); err != nil { + return nil, err + } + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: "ItemAtPath", Path: name, Err: errors.New("invalid path")} + } + dir, file := path.Split(name) + sd, err := d.DirAtPath(dir) + if err != nil { + return nil, err + } + itm, ok := sd.Dir.ValueByKeyTry(file) + if !ok { + if dir == "" && (file == d.name || file == ".") { + return d, nil + } + return nil, &fs.PathError{Op: "ItemAtPath", Path: name, Err: errors.New("file not found")} + } + return itm, nil +} + // Path returns the full path to this data item func (d *Data) Path() string { pt := d.name diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index 95d04107d4..b4b1b75567 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -8,7 +8,6 @@ import ( "bytes" "errors" "io/fs" - "path" "slices" "time" @@ -19,21 +18,10 @@ import ( // Open opens the given data Value within this datafs filesystem. func (d *Data) Open(name string) (fs.File, error) { - if !fs.ValidPath(name) { - return nil, &fs.PathError{Op: "Open", Path: name, Err: errors.New("invalid name")} - } - dir, file := path.Split(name) - sd, err := d.DirAtPath(dir) + itm, err := d.ItemAtPath(name) if err != nil { return nil, err } - itm, ok := sd.Dir.ValueByKeyTry(file) - if !ok { - if dir == "" && (file == d.name || file == ".") { - return &DirFile{File: File{Reader: *bytes.NewReader(d.Bytes()), Data: d}}, nil - } - return nil, &fs.PathError{Op: "Open", Path: name, Err: errors.New("file not found")} - } if itm.IsDir() { return &DirFile{File: File{Reader: *bytes.NewReader(itm.Bytes()), Data: itm}}, nil } @@ -43,22 +31,7 @@ func (d *Data) Open(name string) (fs.File, error) { // Stat returns a FileInfo describing the file. // If there is an error, it should be of type *PathError. func (d *Data) Stat(name string) (fs.FileInfo, error) { - if !fs.ValidPath(name) { - return nil, &fs.PathError{Op: "Open", Path: name, Err: errors.New("invalid name")} - } - dir, file := path.Split(name) - sd, err := d.DirAtPath(dir) - if err != nil { - return nil, err - } - itm, ok := sd.Dir.ValueByKeyTry(file) - if !ok { - if dir == "" && (file == d.name || file == ".") { - return d, nil - } - return nil, &fs.PathError{Op: "Stat", Path: name, Err: errors.New("file not found")} - } - return itm, nil + return d.ItemAtPath(name) } // Sub returns a data FS corresponding to the subtree rooted at dir. @@ -121,23 +94,12 @@ func (d *Data) ReadDir(dir string) ([]fs.DirEntry, error) { // The caller is permitted to modify the returned byte slice. // This method should return a copy of the underlying data. func (d *Data) ReadFile(name string) ([]byte, error) { - if err := d.mustDir("ReadFile", name); err != nil { - return nil, err - } - if !fs.ValidPath(name) { - return nil, &fs.PathError{Op: "readFile", Path: name, Err: errors.New("invalid name")} - } - dir, file := path.Split(name) - sd, err := d.DirAtPath(dir) + itm, err := d.ItemAtPath(name) if err != nil { return nil, err } - itm, ok := sd.Dir.ValueByKeyTry(file) - if !ok { - return nil, &fs.PathError{Op: "readFile", Path: name, Err: errors.New("file not found")} - } if itm.IsDir() { - return nil, &fs.PathError{Op: "readFile", Path: name, Err: errors.New("Value is a directory")} + return nil, &fs.PathError{Op: "ReadFile", Path: name, Err: errors.New("Item is a directory")} } return slices.Clone(itm.Bytes()), nil } diff --git a/tensor/datafs/list.go b/tensor/datafs/list.go new file mode 100644 index 0000000000..c806829309 --- /dev/null +++ b/tensor/datafs/list.go @@ -0,0 +1,61 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafs + +import "strings" + +const ( + Short = false + Long = true + + DirOnly = false + Recursive = true +) + +// todo: list options string + +func (d *Data) String() string { + if !d.IsDir() { + return d.Data.Tensor.Label() + } + return d.List(Short, DirOnly) +} + +func (d *Data) List(long, recursive bool) string { + if long { + return d.ListLong(recursive, 0) + } + return d.ListShort(recursive, 0) +} + +func (d *Data) ListShort(recursive bool, indent int) string { + var b strings.Builder + items := d.ItemsFunc(nil) + // todo: dirs first, then items + for _, it := range items { + if it.IsDir() { + if recursive { + b.WriteString(d.ListShort(recursive, indent+1)) + } else { + b.WriteString(d.name + " ") + } + } else { + b.WriteString(d.name + " ") + } + } + return b.String() +} + +func (d *Data) ListLong(recursive bool, indent int) string { + var b strings.Builder + items := d.ItemsFunc(nil) + for _, it := range items { + if it.IsDir() { + } else { + b.WriteString(d.String() + "\n") + } + } + return b.String() +} diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index d018be4b08..470bf69182 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -9,6 +9,7 @@ import ( "reflect" "strconv" + "cogentcore.org/core/base/errors" "cogentcore.org/core/core" "cogentcore.org/core/plot/plotcore" "cogentcore.org/core/tensor" @@ -72,13 +73,14 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { tensor.SetCalcFunc(lt.Tensor, func() error { trl := ss.Stats.Item("Trial").AsInt() if st.Tensor.IsString() { - lt.SetStringRowCell(st.StringRowCell(0, 0), trl, 0) + lt.SetStringRow(st.StringRow(0), trl) } else { - lt.SetFloatRowCell(st.FloatRowCell(0, 0), trl, 0) + lt.SetFloatRow(st.FloatRow(0), trl) } return nil }) } + errors.Log(dir.Copy(datafs.Preserve, "AllTrials", "Trial")) return logd } @@ -102,7 +104,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a tensor.SetCalcFunc(lt.Tensor, func() error { stats.Stat(ag, src, stout) ctr := ss.Stats.Item(level).AsInt() - lt.SetFloatRowCell(stout.FloatRowCell(0, 0), ctr, 0) + lt.SetFloatRow(stout.FloatRow(0), ctr) return nil }) } @@ -110,9 +112,9 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a lt := logd.NewOfType(nm, st.Tensor.DataType(), nctr) lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) tensor.SetCalcFunc(lt.Tensor, func() error { - v := st.FloatRowCell(0, 0) + v := st.FloatRow(0) ctr := ss.Stats.Item(level).AsInt() - lt.SetFloatRowCell(v, ctr, 0) + lt.SetFloatRow(v, ctr) return nil }) } @@ -152,8 +154,6 @@ func (ss *Sim) EpochDone() { } func main() { - testGroup() - return ss := &Sim{} ss.ConfigAll() ss.Run() @@ -171,8 +171,8 @@ func testGroup() { if i >= 2 { gp = "B" } - dt.Column("Name").SetStringRowCell(gp, i, 0) - dt.Column("Value").SetFloatRowCell(float64(10*(i+1)), i, 0) + dt.Column("Name").SetStringRow(gp, i) + dt.Column("Value").SetFloatRow(float64(10*(i+1)), i) } dir, _ := datafs.NewDir("Group") stats.TableGroups(dir, dt, "Name") diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go index 2e2bb3e64e..6b85ccda22 100644 --- a/tensor/examples/planets/planets.go +++ b/tensor/examples/planets/planets.go @@ -37,7 +37,7 @@ func AnalyzePlanets(dir *datafs.Data) { decade := Planets.AddFloat64Column("decade") year := Planets.Column("year") for row := range Planets.NumRows() { - yr := year.FloatRowCell(row, 0) + yr := year.FloatRow(row) dec := math.Floor(yr/10) * 10 decade.SetFloatRowCell(dec, row, 0) } diff --git a/tensor/indexed.go b/tensor/indexed.go index 417bac04cf..bcb97e716d 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -450,118 +450,154 @@ func (ix *Indexed) Swap(i, j int) { /////////////////////////////////////////////// // Indexed access -///////////////////// Strings +///////////////////// Floats -// StringValue returns the value of given index as a string. +// Float returns the value of given index as a float64. // The first index value is indirected through the indexes. -func (ix *Indexed) StringValue(i ...int) string { +func (ix *Indexed) Float(i ...int) float64 { if ix.Indexes == nil { - return ix.Tensor.StringValue(i...) + return ix.Tensor.Float(i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.StringValue(ic...) + return ix.Tensor.Float(ic...) } -// SetString sets the value of given index as a string +// SetFloat sets the value of given index as a float64 // The first index value is indirected through the [Indexed.Indexes]. -func (ix *Indexed) SetString(val string, i ...int) { +func (ix *Indexed) SetFloat(val float64, i ...int) { if ix.Indexes == nil { - ix.Tensor.SetString(val, i...) + ix.Tensor.SetFloat(val, i...) + return } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetString(val, ic...) + ix.Tensor.SetFloat(val, ic...) } -// StringRowCell returns the value at given row and cell, +// FloatRow returns the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for FloatRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (ix *Indexed) FloatRow(row int) float64 { + return ix.Tensor.FloatRowCell(ix.Index(row), 0) +} + +// SetFloatRow sets the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (ix *Indexed) SetFloatRow(val float64, row int) { + ix.Tensor.SetFloatRowCell(val, ix.Index(row), 0) +} + +// FloatRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) StringRowCell(row, cell int) string { - return ix.Tensor.StringRowCell(ix.Index(row), cell) +func (ix *Indexed) FloatRowCell(row, cell int) float64 { + return ix.Tensor.FloatRowCell(ix.Index(row), cell) } -// SetStringRowCell sets the value at given row and cell, +// SetFloatRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetStringRowCell(val string, row, cell int) { - ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) +func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { + ix.Tensor.SetFloatRowCell(val, ix.Index(row), cell) } -// String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. +// Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. // This is just a convenience pass-through to the Tensor, and does _not_ use // the [Indexed.Indexes]. -func (ix *Indexed) String1D(i int) string { - return ix.Tensor.String1D(i) +func (ix *Indexed) Float1D(i int) float64 { + return ix.Tensor.Float1D(i) } -// SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. +// SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. // This is just a convenience pass-through to the Tensor, and does _not_ use // the [Indexed.Indexes]. -func (ix *Indexed) SetString1D(val string, i int) { - ix.Tensor.SetString1D(val, i) +func (ix *Indexed) SetFloat1D(val float64, i int) { + ix.Tensor.SetFloat1D(val, i) } -///////////////////// Floats +///////////////////// Strings -// Float returns the value of given index as a float64. +// StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. -func (ix *Indexed) Float(i ...int) float64 { +func (ix *Indexed) StringValue(i ...int) string { if ix.Indexes == nil { - return ix.Tensor.Float(i...) + return ix.Tensor.StringValue(i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.Float(ic...) + return ix.Tensor.StringValue(ic...) } -// SetFloat sets the value of given index as a float64 +// SetString sets the value of given index as a string // The first index value is indirected through the [Indexed.Indexes]. -func (ix *Indexed) SetFloat(val float64, i ...int) { +func (ix *Indexed) SetString(val string, i ...int) { if ix.Indexes == nil { - ix.Tensor.SetFloat(val, i...) - return + ix.Tensor.SetString(val, i...) } ic := slices.Clone(i) ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetFloat(val, ic...) + ix.Tensor.SetString(val, ic...) } -// FloatRowCell returns the value at given row and cell, +// StringRow returns the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for StringRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (ix *Indexed) StringRow(row int) string { + return ix.Tensor.StringRowCell(ix.Index(row), 0) +} + +// SetStringRow sets the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for SetStringRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (ix *Indexed) SetStringRow(val string, row int) { + ix.Tensor.SetStringRowCell(val, ix.Index(row), 0) +} + +// StringRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) FloatRowCell(row, cell int) float64 { - return ix.Tensor.FloatRowCell(ix.Index(row), cell) +func (ix *Indexed) StringRowCell(row, cell int) string { + return ix.Tensor.StringRowCell(ix.Index(row), cell) } -// SetFloatRowCell sets the value at given row and cell, +// SetStringRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { - ix.Tensor.SetFloatRowCell(val, ix.Index(row), cell) +func (ix *Indexed) SetStringRowCell(val string, row, cell int) { + ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) } -// Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. +// String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. // This is just a convenience pass-through to the Tensor, and does _not_ use // the [Indexed.Indexes]. -func (ix *Indexed) Float1D(i int) float64 { - return ix.Tensor.Float1D(i) +func (ix *Indexed) String1D(i int) string { + return ix.Tensor.String1D(i) } -// SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. +// SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. // This is just a convenience pass-through to the Tensor, and does _not_ use // the [Indexed.Indexes]. -func (ix *Indexed) SetFloat1D(val float64, i int) { - ix.Tensor.SetFloat1D(val, i) +func (ix *Indexed) SetString1D(val string, i int) { + ix.Tensor.SetString1D(val, i) } ///////////////////// Ints -// Int returns the value of given index as a float64. +// Int returns the value of given index as an int. // The first index value is indirected through the indexes. func (ix *Indexed) Int(i ...int) int { if ix.Indexes == nil { @@ -572,7 +608,7 @@ func (ix *Indexed) Int(i ...int) int { return ix.Tensor.Int(ic...) } -// SetInt sets the value of given index as a int +// SetInt sets the value of given index as an int // The first index value is indirected through the [Indexed.Indexes]. func (ix *Indexed) SetInt(val int, i ...int) { if ix.Indexes == nil { @@ -584,6 +620,24 @@ func (ix *Indexed) SetInt(val int, i ...int) { ix.Tensor.SetInt(val, ic...) } +// IntRow returns the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for IntRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (ix *Indexed) IntRow(row int) int { + return ix.Tensor.IntRowCell(ix.Index(row), 0) +} + +// SetIntRow sets the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for SetIntRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (ix *Indexed) SetIntRow(val int, row int) { + ix.Tensor.SetIntRowCell(val, ix.Index(row), 0) +} + // IntRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. diff --git a/tensor/io.go b/tensor/io.go index be3ac42f18..02cc175a17 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -44,6 +44,19 @@ func (dl Delims) Rune() rune { return '\t' } +// SetPrecision sets the "precision" metadata value that determines +// the precision to use in writing floating point numbers to files. +func SetPrecision(md metadata.Data, prec int) { + md.Set("precision", prec) +} + +// GetPrecision gets the "precision" metadata value that determines +// the precision to use in writing floating point numbers to files. +// returns an error if not set. +func GetPrecision(md metadata.Data) (int, error) { + return metadata.Get[int](md, "precision") +} + // SaveCSV writes a tensor to a comma-separated-values (CSV) file // (where comma = any delimiter, specified in the delim arg). // Outer-most dims are rows in the file, and inner-most is column -- @@ -83,7 +96,7 @@ func OpenCSV(tsr Tensor, filename core.Filename, delim Delims) error { // Reading just grabs all values and doesn't care about shape. func WriteCSV(tsr Tensor, w io.Writer, delim Delims) error { prec := -1 - if ps, err := metadata.Get[int](*tsr.Metadata(), "precision"); err == nil { + if ps, err := GetPrecision(*tsr.Metadata()); err == nil { prec = ps } cw := csv.NewWriter(w) diff --git a/tensor/stats/README.md b/tensor/stats/README.md index 1f8a764e09..3c84a69d38 100644 --- a/tensor/stats/README.md +++ b/tensor/stats/README.md @@ -1,6 +1,6 @@ # stats -There are several packages here for operating on vector, [tensor](../), and [table](../table) data, for computing standard statistics and performing related computations, such as normalizing the data. +There are several packages here for operating on [tensor](../), and [table](../table) data, for computing standard statistics and performing related computations, such as normalizing the data. * [cluster](cluster) implements agglomerative clustering of items based on [metric](metric) distance / similarity matrix data. * [convolve](convolve) convolves data (e.g., for smoothing). diff --git a/tensor/stats/cluster/plot.go b/tensor/stats/cluster/plot.go index e269b1ea9d..e253b2236e 100644 --- a/tensor/stats/cluster/plot.go +++ b/tensor/stats/cluster/plot.go @@ -33,29 +33,29 @@ func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { lbl := pt.ColumnByIndex(2) if nn.IsLeaf() { pt.SetNumRows(row + 1) - xc.SetFloatRowCell(nn.ParDist, row, 0) - yc.SetFloatRowCell(nn.Y, row, 0) + xc.SetFloatRow(nn.ParDist, row) + yc.SetFloatRow(nn.Y, row) if labels.Len() > nn.Index { - lbl.SetStringRowCell(labels.StringValue(nn.Index), row, 0) + lbl.SetStringRow(labels.StringValue(nn.Index), row) } } else { for _, kn := range nn.Kids { pt.SetNumRows(row + 2) - xc.SetFloatRowCell(nn.ParDist, row, 0) - yc.SetFloatRowCell(kn.Y, row, 0) + xc.SetFloatRow(nn.ParDist, row) + yc.SetFloatRow(kn.Y, row) row++ - xc.SetFloatRowCell(nn.ParDist+nn.Dist, row, 0) - yc.SetFloatRowCell(kn.Y, row, 0) + xc.SetFloatRow(nn.ParDist+nn.Dist, row) + yc.SetFloatRow(kn.Y, row) kn.Plot(pt, dmat, labels) row = pt.NumRows() pt.SetNumRows(row + 1) - xc.SetFloatRowCell(nn.ParDist, row, 0) - yc.SetFloatRowCell(kn.Y, row, 0) + xc.SetFloatRow(nn.ParDist, row) + yc.SetFloatRow(kn.Y, row) row++ } pt.SetNumRows(row + 1) - xc.SetFloatRowCell(nn.ParDist, row, 0) - yc.SetFloatRowCell(nn.Y, row, 0) + xc.SetFloatRow(nn.ParDist, row) + yc.SetFloatRow(nn.Y, row) } } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index cb50eeb281..1dd164b61e 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -49,7 +49,7 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { n := r - start it := datafs.NewValue[int](dir, val, n) for j := range n { - it.SetIntRowCell(srt.Indexes[start+j], j, 0) // key to indirect through sort indexes + it.SetIntRow(srt.Indexes[start+j], j) // key to indirect through sort indexes } } @@ -67,9 +67,9 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { srt.SortStable(tensor.Ascending) start := 0 if tsr.Tensor.IsString() { - lastVal := srt.StringRowCell(0, 0) + lastVal := srt.StringRow(0) for r := range nr { - v := srt.StringRowCell(r, 0) + v := srt.StringRow(r) if v != lastVal { makeIdxs(td, srt, lastVal, start, r) start = r @@ -80,9 +80,9 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { makeIdxs(td, srt, lastVal, start, nr) } } else { - lastVal := srt.FloatRowCell(0, 0) + lastVal := srt.FloatRow(0) for r := range nr { - v := srt.FloatRowCell(r, 0) + v := srt.FloatRow(r) if v != lastVal { makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, r) start = r @@ -145,7 +145,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { if gv == nil { gtsr := datafs.NewValue[string](sgd, gpnm, nv) for i, v := range vals { - gtsr.SetStringRowCell(v.Tensor.Metadata().GetName(), i, 0) + gtsr.SetStringRow(v.Tensor.Metadata().GetName(), i) } } for _, tsr := range tsrs { @@ -155,7 +155,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { idx := v.Tensor.(*tensor.Int).Values sg := tensor.NewIndexed(tsr.Tensor, idx) tensor.Call(stat, sg, stout) - sv.SetFloatRowCell(stout.Float1D(0), i, 0) + sv.SetFloatRow(stout.Float1D(0), i) } } } diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index 6a7bf809d3..3696f0ebec 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -22,8 +22,8 @@ func TestGroup(t *testing.T) { if i >= 2 { gp = "B" } - dt.Column("Name").SetStringRowCell(gp, i, 0) - dt.Column("Value").SetFloatRowCell(float64(i), i, 0) + dt.Column("Name").SetStringRow(gp, i) + dt.Column("Value").SetFloatRow(float64(i), i) } dir, _ := datafs.NewDir("Group") TableGroups(dir, dt, "Name") diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 24728e0b92..3ab83217f7 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -44,13 +44,13 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { lwi := math.Floor(qi) lwii := int(lwi) if lwii >= sz { - val = sin.FloatRowCell(sz, 0) + val = sin.FloatRow(sz) } else if lwii < 0 { - val = sin.FloatRowCell(0, 0) + val = sin.FloatRow(0) } else { phi := qi - lwi - lwv := sin.FloatRowCell(lwii, 0) - hiv := sin.FloatRowCell(lwii+1, 0) + lwv := sin.FloatRow(lwii) + hiv := sin.FloatRow(lwii + 1) val = (1-phi)*lwv + phi*hiv } out.SetFloat1D(val, i) diff --git a/tensor/table/README.md b/tensor/table/README.md index 952a7ee9fe..39909e0c4f 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -28,16 +28,30 @@ It is very low-cost to create a new View of an existing Table, via `NewView`, as Column data access: +```Go +// FloatRow is a method on the `tensor.Indexed` returned from the `Column` method. +// This is the best method to use in general for generic 1D data access, +// as it works on any data from 1D on up (although it only samples the first value +// from higher dimensional data) . +val := dt.Column("Values").FloatRow(3) +``` + +```Go +dt.Column("Name").SetStringRow(4) +``` + +To access higher-dimensional "cell" level data using a simple 1D index into the cell patterns: + ```Go // FloatRowCell is a method on the `tensor.Indexed` returned from the `Column` method. // This is the best method to use in general for generic 1D data access, // as it works on any data from 1D on up (although it only samples the first value // from higher dimensional data) . -val := dt.Column("Values").FloatRowCell(3, 0) +val := dt.Column("Values").FloatRowCell(3, 2) ``` ```Go -dt.Column("Name").SetStringRowCell(4, 0) +dt.Column("Name").SetStringRowCell("Alia", 4, 1) ``` todo: more diff --git a/tensor/table/io.go b/tensor/table/io.go index 143e159220..8271ecb47b 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -18,7 +18,6 @@ import ( "strings" "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/metadata" "cogentcore.org/core/core" "cogentcore.org/core/tensor" ) @@ -376,7 +375,7 @@ func (dt *Table) WriteCSVRow(w io.Writer, row int, delim tensor.Delims) error { // WriteCSVRowWriter uses csv.Writer to write one row func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { prec := -1 - if ps, err := metadata.Get[int](dt.Meta, "precision"); err == nil { + if ps, err := tensor.GetPrecision(dt.Meta); err == nil { prec = ps } var rec []string diff --git a/tensor/table/slicetable.go b/tensor/table/slicetable.go index e25827817c..2adfcd1c1e 100644 --- a/tensor/table/slicetable.go +++ b/tensor/table/slicetable.go @@ -54,10 +54,10 @@ func UpdateSliceTable(st any, dt *Table) { val := npv.Index(ri).Field(i).Interface() cl := dt.Column(f.Name) if kind == reflect.String { - cl.SetStringRowCell(val.(string), ri, 0) + cl.SetStringRow(val.(string), ri) } else { fv, _ := reflectx.ToFloat(val) - cl.SetFloatRowCell(fv, ri, 0) + cl.SetFloatRow(fv, ri) } } } diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index 2320c9867f..ff7e8d3d2b 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -45,9 +45,9 @@ func NewTestTable() *Table { dt.AddIntColumn("Int") dt.SetNumRows(3) for i := range dt.NumRows() { - dt.Column("Str").SetStringRowCell(strconv.Itoa(i), i, 0) - dt.Column("Flt64").SetFloatRowCell(float64(i), i, 0) - dt.Column("Int").SetFloatRowCell(float64(i), i, 0) + dt.Column("Str").SetStringRow(strconv.Itoa(i), i) + dt.Column("Flt64").SetFloatRow(float64(i), i) + dt.Column("Int").SetFloatRow(float64(i), i) } return dt } @@ -61,16 +61,16 @@ func TestAppendRowsEtc(t *testing.T) { for j := range 3 { for i := range st.NumRows() { sr := j*3 + i - ss := st.Column("Str").StringRowCell(i, 0) - ds := dt.Column("Str").StringRowCell(sr, 0) + ss := st.Column("Str").StringRow(i) + ds := dt.Column("Str").StringRow(sr) assert.Equal(t, ss, ds) - sf := st.Column("Flt64").FloatRowCell(i, 0) - df := dt.Column("Flt64").FloatRowCell(sr, 0) + sf := st.Column("Flt64").FloatRow(i) + df := dt.Column("Flt64").FloatRow(sr) assert.Equal(t, sf, df) - sf = st.Column("Int").FloatRowCell(i, 0) - df = dt.Column("Int").FloatRowCell(sr, 0) + sf = st.Column("Int").FloatRow(i) + df = dt.Column("Int").FloatRow(sr) assert.Equal(t, sf, df) } } @@ -88,7 +88,7 @@ func TestAppendRowsEtc(t *testing.T) { dt.Sequential() dt.Filter(func(dt *Table, row int) bool { - return dt.Column("Flt64").FloatRowCell(row, 0) > 1 + return dt.Column("Flt64").FloatRow(row) > 1 }) assert.Equal(t, []int{2, 5, 8, 11}, dt.Indexes) } diff --git a/tensor/tensor.go b/tensor/tensor.go index 3a9442c109..98114fb1cc 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -98,16 +98,16 @@ type Tensor interface { // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. SetFloat1D(val float64, i int) - // FloatRowCell returns the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. This is useful for lists of - // patterns, and the [table.Table] container. [Indexed] tensors index along the row, - // and use this interface extensively. + // FloatRowCell returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Indexed] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. FloatRowCell(row, cell int) float64 - // SetFloatRowCell sets the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. This is useful for lists of - // patterns, and the [table.Table] container. [Indexed] tensors index along the row, - // and use this interface extensively. + // SetFloatRowCell sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Indexed] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. SetFloatRowCell(val float64, row, cell int) ///////////////////// Strings @@ -125,14 +125,16 @@ type Tensor interface { // SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. SetString1D(val string, i int) - // StringRowCell returns the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. [Indexed] tensors index along the row, - // and use this interface extensively. + // StringRowCell returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Indexed] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. StringRowCell(row, cell int) string - // SetStringRowCell sets the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. [Indexed] tensors index along the row, - // and use this interface extensively. + // SetStringRowCell sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Indexed] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. SetStringRowCell(val string, row, cell int) ///////////////////// Ints @@ -149,16 +151,16 @@ type Tensor interface { // SetInt1D sets the value of given 1-dimensional index (0-Len()-1) as a int. SetInt1D(val int, i int) - // IntRowCell returns the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. This is useful for lists of - // patterns, and the [table.Table] container. [Indexed] tensors index along the row, - // and use this interface extensively. + // IntRowCell returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Indexed] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. IntRowCell(row, cell int) int - // SetIntRowCell sets the value at given row and cell, where row is outermost dim, - // and cell is 1D index into remaining inner dims. This is useful for lists of - // patterns, and the [table.Table] container. [Indexed] tensors index along the row, - // and use this interface extensively. + // SetIntRowCell sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Indexed] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. SetIntRowCell(val int, row, cell int) ///////////////////// SubSpaces @@ -202,12 +204,12 @@ type Tensor interface { // CopyFrom copies all values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and - // otherwise it goes through the appropriate standard type (Float or String). + // otherwise it goes through the appropriate standard type (Float, Int, String). CopyFrom(from Tensor) // AppendFrom appends all values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and - // otherwise it goes through the appropriate standard type (Float or String). + // otherwise it goes through the appropriate standard type (Float, Int, String). AppendFrom(from Tensor) error // SetShapeFrom sets our shape from given source tensor, calling diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 9ed8888153..3c1b2087d9 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -624,7 +624,7 @@ func (tb *Table) SelectedColumnStrings(colName string) []string { var s []string col := dt.Column(colName) for _, i := range jis { - v := col.StringRowCell(i, 0) + v := col.StringRow(i) s = append(s, v) } return s diff --git a/tensor/tensorcore/tensoreditor.go b/tensor/tensorcore/tensoreditor.go index 0e024e4c7f..aac5b70a15 100644 --- a/tensor/tensorcore/tensoreditor.go +++ b/tensor/tensorcore/tensoreditor.go @@ -70,6 +70,9 @@ func (tb *TensorEditor) Init() { tb.UpdateMaxWidths() tb.Updater(func() { + if tb.Tensor.NumDims() == 1 { + tb.Layout.TopZero = true + } tb.UpdateStartIndex() tb.UpdateMaxWidths() }) diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 212b81d519..091aa52f00 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -16,11 +16,11 @@ import ( // for overlapping cells. func Add(a, b, out *tensor.Indexed) { if b.Len() == 1 { - AddScalar(b.FloatRowCell(0, 0), a, out) + AddScalar(b.FloatRow(0), a, out) return } if a.Len() == 1 { - AddScalar(a.FloatRowCell(0, 0), b, out) + AddScalar(a.FloatRow(0), b, out) return } arows, acells := a.Tensor.RowCellSize() @@ -76,11 +76,11 @@ func AddSubSpace(a, sub, out *tensor.Indexed) { // for overlapping cells. func Sub(a, b, out *tensor.Indexed) { if b.Len() == 1 { - SubScalar(1, b.FloatRowCell(0, 0), a, out) + SubScalar(1, b.FloatRow(0), a, out) return } if a.Len() == 1 { - SubScalar(-1, a.FloatRowCell(0, 0), b, out) + SubScalar(-1, a.FloatRow(0), b, out) return } arows, acells := a.Tensor.RowCellSize() @@ -138,11 +138,11 @@ func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { // for overlapping cells. func Mul(a, b, out *tensor.Indexed) { if b.Len() == 1 { - MulScalar(b.FloatRowCell(0, 0), a, out) + MulScalar(b.FloatRow(0), a, out) return } if a.Len() == 1 { - MulScalar(a.FloatRowCell(0, 0), b, out) + MulScalar(a.FloatRow(0), b, out) return } arows, acells := a.Tensor.RowCellSize() @@ -198,11 +198,11 @@ func MulSubSpace(a, sub, out *tensor.Indexed) { // for overlapping cells. func Div(a, b, out *tensor.Indexed) { if b.Len() == 1 { - DivScalar(b.FloatRowCell(0, 0), a, out) + DivScalar(b.FloatRow(0), a, out) return } if a.Len() == 1 { - DivScalarInv(a.FloatRowCell(0, 0), b, out) + DivScalarInv(a.FloatRow(0), b, out) return } arows, acells := a.Tensor.RowCellSize() From 22d09e9ffe60cb0854c8cebf77215673f0cedbc9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 15:17:25 -0700 Subject: [PATCH 046/311] sim has data grouping on alltrials --- tensor/datafs/copy.go | 29 ++- tensor/datafs/list.go | 9 +- tensor/examples/datafs-sim/sim.go | 72 +++--- .../cogentcore_org-core-base-fileinfo.go | 4 + .../symbols/cogentcore_org-core-base-fsx.go | 2 + .../cogentcore_org-core-base-reflectx.go | 1 + yaegicore/symbols/cogentcore_org-core-core.go | 12 + .../symbols/cogentcore_org-core-filetree.go | 4 + .../symbols/cogentcore_org-core-math32.go | 1 + .../cogentcore_org-core-plot-plotcore.go | 29 ++- .../symbols/cogentcore_org-core-plot-plots.go | 2 +- .../cogentcore_org-core-tensor-databrowser.go | 25 ++ .../cogentcore_org-core-tensor-datafs.go | 26 ++ ...cogentcore_org-core-tensor-stats-metric.go | 70 +++++ .../cogentcore_org-core-tensor-stats-stats.go | 92 +++++++ .../cogentcore_org-core-tensor-table.go | 32 +-- .../symbols/cogentcore_org-core-tensor.go | 242 ++++++++++++++++++ yaegicore/symbols/make | 2 +- 18 files changed, 563 insertions(+), 91 deletions(-) create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-datafs.go create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor.go diff --git a/tensor/datafs/copy.go b/tensor/datafs/copy.go index 9450e21d8e..b043916ef2 100644 --- a/tensor/datafs/copy.go +++ b/tensor/datafs/copy.go @@ -6,7 +6,6 @@ package datafs import ( "errors" - "fmt" "io/fs" "time" ) @@ -54,6 +53,7 @@ func (d *Data) Copy(overwrite bool, to string, from ...string) error { case len(from) == 0: return &fs.PathError{Op: "Copy", Path: to, Err: errors.New("no from sources specified")} } + // todo: check for to conflict first here.. tod, _ := d.ItemAtPath(to) var errs []error if len(from) > 1 && tod != nil && !tod.IsDir() { @@ -72,26 +72,27 @@ func (d *Data) Copy(overwrite bool, to string, from ...string) error { errs = append(errs, err) continue } - if trg, ok := targd.Dir.ValueByKeyTry(frd.name); ok { // target exists - switch { - case trg.IsDir() && frd.IsDir(): - // todo: copy all items from frd into trg - case trg.IsDir(): // frd is not - errs = append(errs, &fs.PathError{Op: "Copy", Path: opstr, Err: errors.New("cannot copy from Value onto directory of same name")}) - case frd.IsDir(): // trg is not - errs = append(errs, &fs.PathError{Op: "Copy", Path: opstr, Err: errors.New("cannot copy from Directory onto Value of same name")}) - default: // both nodes - if overwrite { // todo: interactive!? - trg.CopyFromValue(frd) + if targf == "" { + if trg, ok := targd.Dir.ValueByKeyTry(frd.name); ok { // target exists + switch { + case trg.IsDir() && frd.IsDir(): + // todo: copy all items from frd into trg + case trg.IsDir(): // frd is not + errs = append(errs, &fs.PathError{Op: "Copy", Path: opstr, Err: errors.New("cannot copy from Value onto directory of same name")}) + case frd.IsDir(): // trg is not + errs = append(errs, &fs.PathError{Op: "Copy", Path: opstr, Err: errors.New("cannot copy from Directory onto Value of same name")}) + default: // both nodes + if overwrite { // todo: interactive!? + trg.CopyFromValue(frd) + } } + continue } - continue } nw := frd.Clone() if targf != "" { nw.name = targf } - fmt.Println("adding new:", nw.name, nw.String()) targd.Add(nw) } return errors.Join(errs...) diff --git a/tensor/datafs/list.go b/tensor/datafs/list.go index c806829309..9000d0306c 100644 --- a/tensor/datafs/list.go +++ b/tensor/datafs/list.go @@ -33,16 +33,15 @@ func (d *Data) List(long, recursive bool) string { func (d *Data) ListShort(recursive bool, indent int) string { var b strings.Builder items := d.ItemsFunc(nil) - // todo: dirs first, then items for _, it := range items { if it.IsDir() { if recursive { - b.WriteString(d.ListShort(recursive, indent+1)) + b.WriteString("\n" + it.ListShort(recursive, indent+1)) } else { - b.WriteString(d.name + " ") + b.WriteString(it.name + "/ ") } } else { - b.WriteString(d.name + " ") + b.WriteString(it.name + " ") } } return b.String() @@ -54,7 +53,7 @@ func (d *Data) ListLong(recursive bool, indent int) string { for _, it := range items { if it.IsDir() { } else { - b.WriteString(d.String() + "\n") + b.WriteString(it.String() + "\n") } } return b.String() diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 470bf69182..8a5cb41124 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -9,14 +9,12 @@ import ( "reflect" "strconv" - "cogentcore.org/core/base/errors" "cogentcore.org/core/core" "cogentcore.org/core/plot/plotcore" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/stats/stats" - "cogentcore.org/core/tensor/table" ) type Sim struct { @@ -80,7 +78,28 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { return nil }) } - errors.Log(dir.Copy(datafs.Preserve, "AllTrials", "Trial")) + alllogd, _ := dir.Mkdir("AllTrials") + for _, st := range sitems { + nm := st.Tensor.Metadata().GetName() + lt := alllogd.NewOfType(nm, st.Tensor.DataType()) + lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source + tensor.SetCalcFunc(lt.Tensor, func() error { + // todo: helper for below + row := 0 + if lt.Tensor.NumDims() == 0 { + lt.Tensor.SetShape(1) + } else { + row = lt.Tensor.DimSize(0) + lt.Tensor.SetShape(row + 1) + } + if st.Tensor.IsString() { + lt.SetStringRow(st.StringRow(0), row) + } else { + lt.SetFloatRow(st.FloatRow(0), row) + } + return nil + }) + } return logd } @@ -97,6 +116,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a nm := st.Tensor.Metadata().GetName() src := from.Value(nm) if st.Tensor.DataType() >= reflect.Float32 { + // todo: pct correct etc dd, _ := logd.Mkdir(nm) for _, ag := range aggs { // key advantage of dir structure: multiple stats per item lt := dd.NewOfType(ag.String(), st.Tensor.DataType(), nctr) @@ -123,16 +143,27 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a } func (ss *Sim) Run() { + nrun := ss.Config.Item("NRun").AsInt() nepc := ss.Config.Item("NEpoch").AsInt() ntrl := ss.Config.Item("NTrial").AsInt() - for epc := range nepc { - ss.Stats.Item("Epoch").SetInt(epc) - for trl := range ntrl { - ss.Stats.Item("Trial").SetInt(trl) - ss.RunTrial(trl) + for run := range nrun { + ss.Stats.Item("Run").SetInt(run) + for epc := range nepc { + ss.Stats.Item("Epoch").SetInt(epc) + for trl := range ntrl { + ss.Stats.Item("Trial").SetInt(trl) + ss.RunTrial(trl) + } + ss.EpochDone() } - ss.EpochDone() } + alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) + dir, _ := ss.Logs.Mkdir("Stats") + stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") + sts := []string{"SSE", "AvgSSE", "TrlErr"} + stats.TableGroupStats(dir, stats.Mean.FuncName(), alldt, sts...) + stats.TableGroupStats(dir, stats.Sem.FuncName(), alldt, sts...) + } func (ss *Sim) RunTrial(trl int) { @@ -147,6 +178,7 @@ func (ss *Sim) RunTrial(trl int) { } ss.Stats.Item("TrlErr").SetFloat32(trlErr) ss.Logs.Item("Trial").CalcAll() + ss.Logs.Item("AllTrials").CalcAll() } func (ss *Sim) EpochDone() { @@ -161,25 +193,3 @@ func main() { databrowser.NewBrowserWindow(ss.Root, "Root") core.Wait() } - -func testGroup() { - dt := table.NewTable().SetNumRows(4) - dt.AddStringColumn("Name") - dt.AddFloat32Column("Value") - for i := range dt.NumRows() { - gp := "A" - if i >= 2 { - gp = "B" - } - dt.Column("Name").SetStringRow(gp, i) - dt.Column("Value").SetFloatRow(float64(10*(i+1)), i) - } - dir, _ := datafs.NewDir("Group") - stats.TableGroups(dir, dt, "Name") - - stats.TableGroupStats(dir, stats.Mean.FuncName(), dt, "Value") - stats.TableGroupStats(dir, stats.Sem.FuncName(), dt, "Value") - - databrowser.NewBrowserWindow(dir, "Group") - core.Wait() -} diff --git a/yaegicore/symbols/cogentcore_org-core-base-fileinfo.go b/yaegicore/symbols/cogentcore_org-core-base-fileinfo.go index 44645fc85c..0f3a5a442f 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-fileinfo.go +++ b/yaegicore/symbols/cogentcore_org-core-base-fileinfo.go @@ -126,6 +126,7 @@ func init() { "Multipart": reflect.ValueOf(fileinfo.Multipart), "NewFileInfo": reflect.ValueOf(fileinfo.NewFileInfo), "NewFileInfoType": reflect.ValueOf(fileinfo.NewFileInfoType), + "Number": reflect.ValueOf(fileinfo.Number), "OCaml": reflect.ValueOf(fileinfo.OCaml), "Obj": reflect.ValueOf(fileinfo.Obj), "ObjC": reflect.ValueOf(fileinfo.ObjC), @@ -157,10 +158,13 @@ func init() { "Shar": reflect.ValueOf(fileinfo.Shar), "Sheet": reflect.ValueOf(fileinfo.Sheet), "StandardMimes": reflect.ValueOf(&fileinfo.StandardMimes).Elem(), + "String": reflect.ValueOf(fileinfo.String), "Svg": reflect.ValueOf(fileinfo.Svg), + "Table": reflect.ValueOf(fileinfo.Table), "Tar": reflect.ValueOf(fileinfo.Tar), "Tcl": reflect.ValueOf(fileinfo.Tcl), "TeX": reflect.ValueOf(fileinfo.TeX), + "Tensor": reflect.ValueOf(fileinfo.Tensor), "Texinfo": reflect.ValueOf(fileinfo.Texinfo), "Text": reflect.ValueOf(fileinfo.Text), "TextPlain": reflect.ValueOf(constant.MakeFromLiteral("\"text/plain\"", token.STRING, 0)), diff --git a/yaegicore/symbols/cogentcore_org-core-base-fsx.go b/yaegicore/symbols/cogentcore_org-core-base-fsx.go index 52fbcd16dc..4083d45fd0 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-fsx.go +++ b/yaegicore/symbols/cogentcore_org-core-base-fsx.go @@ -13,6 +13,7 @@ func init() { "DirAndFile": reflect.ValueOf(fsx.DirAndFile), "DirFS": reflect.ValueOf(fsx.DirFS), "Dirs": reflect.ValueOf(fsx.Dirs), + "ExtSplit": reflect.ValueOf(fsx.ExtSplit), "FileExists": reflect.ValueOf(fsx.FileExists), "FileExistsFS": reflect.ValueOf(fsx.FileExistsFS), "Filenames": reflect.ValueOf(fsx.Filenames), @@ -22,6 +23,7 @@ func init() { "HasFile": reflect.ValueOf(fsx.HasFile), "LatestMod": reflect.ValueOf(fsx.LatestMod), "RelativeFilePath": reflect.ValueOf(fsx.RelativeFilePath), + "SplitRootPathFS": reflect.ValueOf(fsx.SplitRootPathFS), "Sub": reflect.ValueOf(fsx.Sub), } } diff --git a/yaegicore/symbols/cogentcore_org-core-base-reflectx.go b/yaegicore/symbols/cogentcore_org-core-base-reflectx.go index be574d4356..aef279b5af 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-reflectx.go +++ b/yaegicore/symbols/cogentcore_org-core-base-reflectx.go @@ -17,6 +17,7 @@ func init() { "CopySliceRobust": reflect.ValueOf(reflectx.CopySliceRobust), "FormatDefault": reflect.ValueOf(reflectx.FormatDefault), "KindIsBasic": reflect.ValueOf(reflectx.KindIsBasic), + "KindIsNumber": reflect.ValueOf(reflectx.KindIsNumber), "LongTypeName": reflect.ValueOf(reflectx.LongTypeName), "MapAdd": reflect.ValueOf(reflectx.MapAdd), "MapDelete": reflect.ValueOf(reflectx.MapDelete), diff --git a/yaegicore/symbols/cogentcore_org-core-core.go b/yaegicore/symbols/cogentcore_org-core-core.go index ceae6f8906..0eef56d453 100644 --- a/yaegicore/symbols/cogentcore_org-core-core.go +++ b/yaegicore/symbols/cogentcore_org-core-core.go @@ -271,6 +271,7 @@ func init() { "ListGrid": reflect.ValueOf((*core.ListGrid)(nil)), "ListStyler": reflect.ValueOf((*core.ListStyler)(nil)), "Lister": reflect.ValueOf((*core.Lister)(nil)), + "MenuSearcher": reflect.ValueOf((*core.MenuSearcher)(nil)), "Meter": reflect.ValueOf((*core.Meter)(nil)), "MeterTypes": reflect.ValueOf((*core.MeterTypes)(nil)), "OnBinder": reflect.ValueOf((*core.OnBinder)(nil)), @@ -331,6 +332,7 @@ func init() { "_ButtonEmbedder": reflect.ValueOf((*_cogentcore_org_core_core_ButtonEmbedder)(nil)), "_Layouter": reflect.ValueOf((*_cogentcore_org_core_core_Layouter)(nil)), "_Lister": reflect.ValueOf((*_cogentcore_org_core_core_Lister)(nil)), + "_MenuSearcher": reflect.ValueOf((*_cogentcore_org_core_core_MenuSearcher)(nil)), "_OnBinder": reflect.ValueOf((*_cogentcore_org_core_core_OnBinder)(nil)), "_Settings": reflect.ValueOf((*_cogentcore_org_core_core_Settings)(nil)), "_SettingsOpener": reflect.ValueOf((*_cogentcore_org_core_core_SettingsOpener)(nil)), @@ -513,6 +515,16 @@ func (W _cogentcore_org_core_core_Lister) StyleValue(w core.Widget, s *styles.St func (W _cogentcore_org_core_core_Lister) UpdateMaxWidths() { W.WUpdateMaxWidths() } func (W _cogentcore_org_core_core_Lister) UpdateSliceSize() int { return W.WUpdateSliceSize() } +// _cogentcore_org_core_core_MenuSearcher is an interface wrapper for MenuSearcher type +type _cogentcore_org_core_core_MenuSearcher struct { + IValue interface{} + WMenuSearch func(items *[]core.ChooserItem) +} + +func (W _cogentcore_org_core_core_MenuSearcher) MenuSearch(items *[]core.ChooserItem) { + W.WMenuSearch(items) +} + // _cogentcore_org_core_core_OnBinder is an interface wrapper for OnBinder type type _cogentcore_org_core_core_OnBinder struct { IValue interface{} diff --git a/yaegicore/symbols/cogentcore_org-core-filetree.go b/yaegicore/symbols/cogentcore_org-core-filetree.go index b845d24ab4..442548dcc5 100644 --- a/yaegicore/symbols/cogentcore_org-core-filetree.go +++ b/yaegicore/symbols/cogentcore_org-core-filetree.go @@ -66,12 +66,14 @@ type _cogentcore_org_core_filetree_Filer struct { WDestroy func() WDragDrop func(e events.Event) WDropDeleteSource func(e events.Event) + WGetFileInfo func() error WInit func() WMimeData func(md *mimedata.Mimes) WNodeWalkDown func(fun func(n tree.Node) bool) WOnAdd func() WOnClose func() WOnOpen func() + WOpenFile func() error WPaste func() WPlanName func() string WPosition func() @@ -105,6 +107,7 @@ func (W _cogentcore_org_core_filetree_Filer) Cut() { func (W _cogentcore_org_core_filetree_Filer) Destroy() { W.WDestroy() } func (W _cogentcore_org_core_filetree_Filer) DragDrop(e events.Event) { W.WDragDrop(e) } func (W _cogentcore_org_core_filetree_Filer) DropDeleteSource(e events.Event) { W.WDropDeleteSource(e) } +func (W _cogentcore_org_core_filetree_Filer) GetFileInfo() error { return W.WGetFileInfo() } func (W _cogentcore_org_core_filetree_Filer) Init() { W.WInit() } func (W _cogentcore_org_core_filetree_Filer) MimeData(md *mimedata.Mimes) { W.WMimeData(md) } func (W _cogentcore_org_core_filetree_Filer) NodeWalkDown(fun func(n tree.Node) bool) { @@ -113,6 +116,7 @@ func (W _cogentcore_org_core_filetree_Filer) NodeWalkDown(fun func(n tree.Node) func (W _cogentcore_org_core_filetree_Filer) OnAdd() { W.WOnAdd() } func (W _cogentcore_org_core_filetree_Filer) OnClose() { W.WOnClose() } func (W _cogentcore_org_core_filetree_Filer) OnOpen() { W.WOnOpen() } +func (W _cogentcore_org_core_filetree_Filer) OpenFile() error { return W.WOpenFile() } func (W _cogentcore_org_core_filetree_Filer) Paste() { W.WPaste() } func (W _cogentcore_org_core_filetree_Filer) PlanName() string { return W.WPlanName() } func (W _cogentcore_org_core_filetree_Filer) Position() { W.WPosition() } diff --git a/yaegicore/symbols/cogentcore_org-core-math32.go b/yaegicore/symbols/cogentcore_org-core-math32.go index df1c0a89e8..3d9fabd2c2 100644 --- a/yaegicore/symbols/cogentcore_org-core-math32.go +++ b/yaegicore/symbols/cogentcore_org-core-math32.go @@ -30,6 +30,7 @@ func init() { "Cbrt": reflect.ValueOf(math32.Cbrt), "Ceil": reflect.ValueOf(math32.Ceil), "Clamp": reflect.ValueOf(math32.Clamp), + "Clamp64": reflect.ValueOf(math32.Clamp64), "ClampInt": reflect.ValueOf(math32.ClampInt), "ContainsPoint": reflect.ValueOf(math32.ContainsPoint), "CopyFloat32s": reflect.ValueOf(math32.CopyFloat32s), diff --git a/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go b/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go index 41c461e613..61f3260cac 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go +++ b/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go @@ -10,19 +10,22 @@ import ( func init() { Symbols["cogentcore.org/core/plot/plotcore/plotcore"] = map[string]reflect.Value{ // function, constant and variable definitions - "Bar": reflect.ValueOf(plotcore.Bar), - "FixMax": reflect.ValueOf(plotcore.FixMax), - "FixMin": reflect.ValueOf(plotcore.FixMin), - "FloatMax": reflect.ValueOf(plotcore.FloatMax), - "FloatMin": reflect.ValueOf(plotcore.FloatMin), - "NewPlot": reflect.ValueOf(plotcore.NewPlot), - "NewPlotEditor": reflect.ValueOf(plotcore.NewPlotEditor), - "NewSubPlot": reflect.ValueOf(plotcore.NewSubPlot), - "Off": reflect.ValueOf(plotcore.Off), - "On": reflect.ValueOf(plotcore.On), - "PlotTypesN": reflect.ValueOf(plotcore.PlotTypesN), - "PlotTypesValues": reflect.ValueOf(plotcore.PlotTypesValues), - "XY": reflect.ValueOf(plotcore.XY), + "Bar": reflect.ValueOf(plotcore.Bar), + "FixMax": reflect.ValueOf(plotcore.FixMax), + "FixMin": reflect.ValueOf(plotcore.FixMin), + "FloatMax": reflect.ValueOf(plotcore.FloatMax), + "FloatMin": reflect.ValueOf(plotcore.FloatMin), + "NewPlot": reflect.ValueOf(plotcore.NewPlot), + "NewPlotEditor": reflect.ValueOf(plotcore.NewPlotEditor), + "NewSubPlot": reflect.ValueOf(plotcore.NewSubPlot), + "Off": reflect.ValueOf(plotcore.Off), + "On": reflect.ValueOf(plotcore.On), + "PlotColumnOptions": reflect.ValueOf(plotcore.PlotColumnOptions), + "PlotColumnZeroOne": reflect.ValueOf(plotcore.PlotColumnZeroOne), + "PlotTypesN": reflect.ValueOf(plotcore.PlotTypesN), + "PlotTypesValues": reflect.ValueOf(plotcore.PlotTypesValues), + "SetPlotColumnOptions": reflect.ValueOf(plotcore.SetPlotColumnOptions), + "XY": reflect.ValueOf(plotcore.XY), // type definitions "ColumnOptions": reflect.ValueOf((*plotcore.ColumnOptions)(nil)), diff --git a/yaegicore/symbols/cogentcore_org-core-plot-plots.go b/yaegicore/symbols/cogentcore_org-core-plot-plots.go index 956dce326e..956a47e0d0 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot-plots.go +++ b/yaegicore/symbols/cogentcore_org-core-plot-plots.go @@ -44,7 +44,7 @@ func init() { "Square": reflect.ValueOf(plots.Square), "StepKindN": reflect.ValueOf(plots.StepKindN), "StepKindValues": reflect.ValueOf(plots.StepKindValues), - "TableColumnIndex": reflect.ValueOf(plots.TableColumnIndex), + "TableColumnByIndex": reflect.ValueOf(plots.TableColumnByIndex), "Triangle": reflect.ValueOf(plots.Triangle), // type definitions diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go b/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go new file mode 100644 index 0000000000..51e341d16a --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go @@ -0,0 +1,25 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/databrowser'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/databrowser" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/databrowser/databrowser"] = map[string]reflect.Value{ + // function, constant and variable definitions + "DataFS": reflect.ValueOf(databrowser.DataFS), + "IsTableFile": reflect.ValueOf(databrowser.IsTableFile), + "NewBrowser": reflect.ValueOf(databrowser.NewBrowser), + "NewBrowserWindow": reflect.ValueOf(databrowser.NewBrowserWindow), + "NewDiffBrowserDirs": reflect.ValueOf(databrowser.NewDiffBrowserDirs), + "NewFileNode": reflect.ValueOf(databrowser.NewFileNode), + "ParentBrowser": reflect.ValueOf(databrowser.ParentBrowser), + + // type definitions + "Browser": reflect.ValueOf((*databrowser.Browser)(nil)), + "FileNode": reflect.ValueOf((*databrowser.FileNode)(nil)), + } +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go b/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go new file mode 100644 index 0000000000..bec8384506 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go @@ -0,0 +1,26 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/datafs'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/datafs" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/datafs/datafs"] = map[string]reflect.Value{ + // function, constant and variable definitions + "DirOnly": reflect.ValueOf(datafs.DirOnly), + "Long": reflect.ValueOf(datafs.Long), + "NewDir": reflect.ValueOf(datafs.NewDir), + "Overwrite": reflect.ValueOf(datafs.Overwrite), + "Preserve": reflect.ValueOf(datafs.Preserve), + "Recursive": reflect.ValueOf(datafs.Recursive), + "Short": reflect.ValueOf(datafs.Short), + + // type definitions + "Data": reflect.ValueOf((*datafs.Data)(nil)), + "DirFile": reflect.ValueOf((*datafs.DirFile)(nil)), + "File": reflect.ValueOf((*datafs.File)(nil)), + } +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go new file mode 100644 index 0000000000..0c7bcf73e9 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -0,0 +1,70 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/stats/metric'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/stats/metric" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/stats/metric/metric"] = map[string]reflect.Value{ + // function, constant and variable definitions + "Abs": reflect.ValueOf(metric.Abs), + "AbsFunc": reflect.ValueOf(metric.AbsFunc), + "ClosestRow": reflect.ValueOf(metric.ClosestRow), + "Correlation": reflect.ValueOf(metric.Correlation), + "CorrelationFunc": reflect.ValueOf(metric.CorrelationFunc), + "CorrelationFuncOut64": reflect.ValueOf(metric.CorrelationFuncOut64), + "Cosine": reflect.ValueOf(metric.Cosine), + "CosineFunc": reflect.ValueOf(metric.CosineFunc), + "CosineFuncOut64": reflect.ValueOf(metric.CosineFuncOut64), + "Covariance": reflect.ValueOf(metric.Covariance), + "CovarianceFunc": reflect.ValueOf(metric.CovarianceFunc), + "CovarianceMatrix": reflect.ValueOf(metric.CovarianceMatrix), + "CrossEntropy": reflect.ValueOf(metric.CrossEntropy), + "CrossEntropyFunc": reflect.ValueOf(metric.CrossEntropyFunc), + "CrossMatrix": reflect.ValueOf(metric.CrossMatrix), + "Euclidean": reflect.ValueOf(metric.Euclidean), + "EuclideanBinTol": reflect.ValueOf(metric.EuclideanBinTol), + "EuclideanBinTolFunc": reflect.ValueOf(metric.EuclideanBinTolFunc), + "EuclideanFunc": reflect.ValueOf(metric.EuclideanFunc), + "Hamming": reflect.ValueOf(metric.Hamming), + "HammingFunc": reflect.ValueOf(metric.HammingFunc), + "InnerProduct": reflect.ValueOf(metric.InnerProduct), + "InnerProductFunc": reflect.ValueOf(metric.InnerProductFunc), + "InvCorrelation": reflect.ValueOf(metric.InvCorrelation), + "InvCorrelationFunc": reflect.ValueOf(metric.InvCorrelationFunc), + "InvCosine": reflect.ValueOf(metric.InvCosine), + "InvCosineFunc": reflect.ValueOf(metric.InvCosineFunc), + "Matrix": reflect.ValueOf(metric.Matrix), + "Metric": reflect.ValueOf(metric.Metric), + "MetricOut": reflect.ValueOf(metric.MetricOut), + "MetricsN": reflect.ValueOf(metric.MetricsN), + "MetricsValues": reflect.ValueOf(metric.MetricsValues), + "NFunc": reflect.ValueOf(metric.NFunc), + "OutShape": reflect.ValueOf(metric.OutShape), + "PCA": reflect.ValueOf(metric.PCA), + "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), + "SVD": reflect.ValueOf(metric.SVD), + "SumSquares": reflect.ValueOf(metric.SumSquares), + "SumSquaresBinTol": reflect.ValueOf(metric.SumSquaresBinTol), + "SumSquaresBinTolFunc": reflect.ValueOf(metric.SumSquaresBinTolFunc), + "SumSquaresBinTolScaleFuncOut64": reflect.ValueOf(metric.SumSquaresBinTolScaleFuncOut64), + "SumSquaresFunc": reflect.ValueOf(metric.SumSquaresFunc), + "SumSquaresFuncOut64": reflect.ValueOf(metric.SumSquaresFuncOut64), + "SumSquaresScaleFuncOut64": reflect.ValueOf(metric.SumSquaresScaleFuncOut64), + "TriangularLIndicies": reflect.ValueOf(metric.TriangularLIndicies), + "TriangularN": reflect.ValueOf(metric.TriangularN), + "Vec2in3outFunc": reflect.ValueOf(metric.Vec2in3outFunc), + "Vec2inFunc": reflect.ValueOf(metric.Vec2inFunc), + "Vec3outFunc": reflect.ValueOf(metric.Vec3outFunc), + "VecFunc": reflect.ValueOf(metric.VecFunc), + "VecSSFunc": reflect.ValueOf(metric.VecSSFunc), + "Vectorize3Out64": reflect.ValueOf(metric.Vectorize3Out64), + + // type definitions + "MetricFunc": reflect.ValueOf((*metric.MetricFunc)(nil)), + "Metrics": reflect.ValueOf((*metric.Metrics)(nil)), + } +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go new file mode 100644 index 0000000000..a010efe98c --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -0,0 +1,92 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/stats/stats'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/stats/stats" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/stats/stats/stats"] = map[string]reflect.Value{ + // function, constant and variable definitions + "Count": reflect.ValueOf(stats.Count), + "CountFunc": reflect.ValueOf(stats.CountFunc), + "CountFuncOut64": reflect.ValueOf(stats.CountFuncOut64), + "Describe": reflect.ValueOf(stats.Describe), + "DescribeTable": reflect.ValueOf(stats.DescribeTable), + "DescribeTableAll": reflect.ValueOf(stats.DescribeTableAll), + "DescriptiveStats": reflect.ValueOf(&stats.DescriptiveStats).Elem(), + "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), + "GroupStats": reflect.ValueOf(stats.GroupStats), + "Groups": reflect.ValueOf(stats.Groups), + "L1Norm": reflect.ValueOf(stats.L1Norm), + "L2Norm": reflect.ValueOf(stats.L2Norm), + "L2NormFunc": reflect.ValueOf(stats.L2NormFunc), + "L2NormFuncOut64": reflect.ValueOf(stats.L2NormFuncOut64), + "Max": reflect.ValueOf(stats.Max), + "MaxAbs": reflect.ValueOf(stats.MaxAbs), + "MaxAbsFunc": reflect.ValueOf(stats.MaxAbsFunc), + "MaxFunc": reflect.ValueOf(stats.MaxFunc), + "Mean": reflect.ValueOf(stats.Mean), + "MeanFunc": reflect.ValueOf(stats.MeanFunc), + "MeanFuncOut64": reflect.ValueOf(stats.MeanFuncOut64), + "Median": reflect.ValueOf(stats.Median), + "MedianFunc": reflect.ValueOf(stats.MedianFunc), + "Min": reflect.ValueOf(stats.Min), + "MinAbs": reflect.ValueOf(stats.MinAbs), + "MinAbsFunc": reflect.ValueOf(stats.MinAbsFunc), + "MinFunc": reflect.ValueOf(stats.MinFunc), + "NFunc": reflect.ValueOf(stats.NFunc), + "OutShape": reflect.ValueOf(stats.OutShape), + "Prod": reflect.ValueOf(stats.Prod), + "ProdFunc": reflect.ValueOf(stats.ProdFunc), + "Q1": reflect.ValueOf(stats.Q1), + "Q1Func": reflect.ValueOf(stats.Q1Func), + "Q3": reflect.ValueOf(stats.Q3), + "Q3Func": reflect.ValueOf(stats.Q3Func), + "QuantilesFunc": reflect.ValueOf(stats.QuantilesFunc), + "Sem": reflect.ValueOf(stats.Sem), + "SemFunc": reflect.ValueOf(stats.SemFunc), + "SemPop": reflect.ValueOf(stats.SemPop), + "SemPopFunc": reflect.ValueOf(stats.SemPopFunc), + "Stat": reflect.ValueOf(stats.Stat), + "StatOut": reflect.ValueOf(stats.StatOut), + "StatsN": reflect.ValueOf(stats.StatsN), + "StatsValues": reflect.ValueOf(stats.StatsValues), + "Std": reflect.ValueOf(stats.Std), + "StdFunc": reflect.ValueOf(stats.StdFunc), + "StdFuncOut64": reflect.ValueOf(stats.StdFuncOut64), + "StdPop": reflect.ValueOf(stats.StdPop), + "StdPopFunc": reflect.ValueOf(stats.StdPopFunc), + "StripPackage": reflect.ValueOf(stats.StripPackage), + "Sum": reflect.ValueOf(stats.Sum), + "SumAbs": reflect.ValueOf(stats.SumAbs), + "SumAbsFunc": reflect.ValueOf(stats.SumAbsFunc), + "SumFunc": reflect.ValueOf(stats.SumFunc), + "SumFuncOut64": reflect.ValueOf(stats.SumFuncOut64), + "SumSq": reflect.ValueOf(stats.SumSq), + "SumSqDevFuncOut64": reflect.ValueOf(stats.SumSqDevFuncOut64), + "SumSqFunc": reflect.ValueOf(stats.SumSqFunc), + "SumSqFuncOut64": reflect.ValueOf(stats.SumSqFuncOut64), + "SumSqScaleFuncOut64": reflect.ValueOf(stats.SumSqScaleFuncOut64), + "TableGroupDescribe": reflect.ValueOf(stats.TableGroupDescribe), + "TableGroupStats": reflect.ValueOf(stats.TableGroupStats), + "TableGroups": reflect.ValueOf(stats.TableGroups), + "Var": reflect.ValueOf(stats.Var), + "VarFunc": reflect.ValueOf(stats.VarFunc), + "VarFuncOut64": reflect.ValueOf(stats.VarFuncOut64), + "VarPop": reflect.ValueOf(stats.VarPop), + "VarPopFunc": reflect.ValueOf(stats.VarPopFunc), + "VarPopFuncOut64": reflect.ValueOf(stats.VarPopFuncOut64), + "Vec2inFunc": reflect.ValueOf(stats.Vec2inFunc), + "Vec2outFunc": reflect.ValueOf(stats.Vec2outFunc), + "VecFunc": reflect.ValueOf(stats.VecFunc), + "Vectorize2Out64": reflect.ValueOf(stats.Vectorize2Out64), + "VectorizeOut64": reflect.ValueOf(stats.VectorizeOut64), + + // type definitions + "Stats": reflect.ValueOf((*stats.Stats)(nil)), + "StatsFunc": reflect.ValueOf((*stats.StatsFunc)(nil)), + } +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-table.go b/yaegicore/symbols/cogentcore_org-core-tensor-table.go index 211744754a..47c808865c 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-table.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-table.go @@ -3,53 +3,33 @@ package symbols import ( - "reflect" - - "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" + "reflect" ) func init() { Symbols["cogentcore.org/core/tensor/table/table"] = map[string]reflect.Value{ // function, constant and variable definitions - "AddAggName": reflect.ValueOf(table.AddAggName), - "Ascending": reflect.ValueOf(table.Ascending), - "ColumnNameOnly": reflect.ValueOf(table.ColumnNameOnly), - "Comma": reflect.ValueOf(table.Comma), "ConfigFromDataValues": reflect.ValueOf(table.ConfigFromDataValues), "ConfigFromHeaders": reflect.ValueOf(table.ConfigFromHeaders), "ConfigFromTableHeaders": reflect.ValueOf(table.ConfigFromTableHeaders), - "Contains": reflect.ValueOf(table.Contains), - "DelimsN": reflect.ValueOf(tensor.DelimsN), - "DelimsValues": reflect.ValueOf(tensor.DelimsValues), - "Descending": reflect.ValueOf(table.Descending), - "Detect": reflect.ValueOf(table.Detect), "DetectTableHeaders": reflect.ValueOf(table.DetectTableHeaders), - "Equals": reflect.ValueOf(table.Equals), "Headers": reflect.ValueOf(table.Headers), - "IgnoreCase": reflect.ValueOf(table.IgnoreCase), "InferDataType": reflect.ValueOf(table.InferDataType), - "NewIndexed": reflect.ValueOf(table.NewIndexed), + "NewColumns": reflect.ValueOf(table.NewColumns), "NewSliceTable": reflect.ValueOf(table.NewSliceTable), "NewTable": reflect.ValueOf(table.NewTable), + "NewView": reflect.ValueOf(table.NewView), "NoHeaders": reflect.ValueOf(table.NoHeaders), "ShapeFromString": reflect.ValueOf(table.ShapeFromString), - "Space": reflect.ValueOf(table.Space), - "Tab": reflect.ValueOf(table.Tab), "TableColumnType": reflect.ValueOf(table.TableColumnType), "TableHeaderChar": reflect.ValueOf(table.TableHeaderChar), "TableHeaderToType": reflect.ValueOf(&table.TableHeaderToType).Elem(), "UpdateSliceTable": reflect.ValueOf(table.UpdateSliceTable), - "UseCase": reflect.ValueOf(table.UseCase), // type definitions - "Delims": reflect.ValueOf((*tensor.Delims)(nil)), - "Filterer": reflect.ValueOf((*table.Filterer)(nil)), - "Indexed": reflect.ValueOf((*table.Table)(nil)), - "LessFunc": reflect.ValueOf((*table.LessFunc)(nil)), - "SplitAgg": reflect.ValueOf((*table.SplitAgg)(nil)), - "Splits": reflect.ValueOf((*table.Splits)(nil)), - "SplitsLessFunc": reflect.ValueOf((*table.SplitsLessFunc)(nil)), - "Table": reflect.ValueOf((*table.Table)(nil)), + "Columns": reflect.ValueOf((*table.Columns)(nil)), + "FilterFunc": reflect.ValueOf((*table.FilterFunc)(nil)), + "Table": reflect.ValueOf((*table.Table)(nil)), } } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go new file mode 100644 index 0000000000..fcb1d8e00d --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -0,0 +1,242 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/base/metadata" + "cogentcore.org/core/tensor" + "gonum.org/v1/gonum/mat" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/tensor"] = map[string]reflect.Value{ + // function, constant and variable definitions + "AddFunc": reflect.ValueOf(tensor.AddFunc), + "AddShapes": reflect.ValueOf(tensor.AddShapes), + "AsFloat32": reflect.ValueOf(tensor.AsFloat32), + "AsFloat64": reflect.ValueOf(tensor.AsFloat64), + "Ascending": reflect.ValueOf(tensor.Ascending), + "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), + "BoolToInt": reflect.ValueOf(tensor.BoolToInt), + "Calc": reflect.ValueOf(tensor.Calc), + "Call": reflect.ValueOf(tensor.Call), + "CallOut": reflect.ValueOf(tensor.CallOut), + "CallString": reflect.ValueOf(tensor.CallString), + "ColMajorStrides": reflect.ValueOf(tensor.ColMajorStrides), + "Comma": reflect.ValueOf(tensor.Comma), + "Contains": reflect.ValueOf(tensor.Contains), + "CopyDense": reflect.ValueOf(tensor.CopyDense), + "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), + "DelimsN": reflect.ValueOf(tensor.DelimsN), + "DelimsValues": reflect.ValueOf(tensor.DelimsValues), + "Descending": reflect.ValueOf(tensor.Descending), + "Detect": reflect.ValueOf(tensor.Detect), + "Equals": reflect.ValueOf(tensor.Equals), + "Exclude": reflect.ValueOf(tensor.Exclude), + "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), + "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), + "GetPrecision": reflect.ValueOf(tensor.GetPrecision), + "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), + "Include": reflect.ValueOf(tensor.Include), + "IntToBool": reflect.ValueOf(tensor.IntToBool), + "NFirstLen": reflect.ValueOf(tensor.NFirstLen), + "NFirstRows": reflect.ValueOf(tensor.NFirstRows), + "NMinLen": reflect.ValueOf(tensor.NMinLen), + "New1DViewOf": reflect.ValueOf(tensor.New1DViewOf), + "NewBits": reflect.ValueOf(tensor.NewBits), + "NewBitsShape": reflect.ValueOf(tensor.NewBitsShape), + "NewByte": reflect.ValueOf(tensor.NewByte), + "NewFloat32": reflect.ValueOf(tensor.NewFloat32), + "NewFloat64": reflect.ValueOf(tensor.NewFloat64), + "NewFloat64Indexed": reflect.ValueOf(tensor.NewFloat64Indexed), + "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), + "NewFunc": reflect.ValueOf(tensor.NewFunc), + "NewIndexed": reflect.ValueOf(tensor.NewIndexed), + "NewInt": reflect.ValueOf(tensor.NewInt), + "NewInt32": reflect.ValueOf(tensor.NewInt32), + "NewOfType": reflect.ValueOf(tensor.NewOfType), + "NewShape": reflect.ValueOf(tensor.NewShape), + "NewString": reflect.ValueOf(tensor.NewString), + "NewStringFromSlice": reflect.ValueOf(tensor.NewStringFromSlice), + "NewStringShape": reflect.ValueOf(tensor.NewStringShape), + "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), + "OddColumn": reflect.ValueOf(tensor.OddColumn), + "OddRow": reflect.ValueOf(tensor.OddRow), + "OpenCSV": reflect.ValueOf(tensor.OpenCSV), + "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), + "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), + "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), + "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), + "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), + "Projection2DString": reflect.ValueOf(tensor.Projection2DString), + "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), + "ReadCSV": reflect.ValueOf(tensor.ReadCSV), + "RowCellSplit": reflect.ValueOf(tensor.RowCellSplit), + "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), + "SaveCSV": reflect.ValueOf(tensor.SaveCSV), + "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), + "SetPrecision": reflect.ValueOf(tensor.SetPrecision), + "Slice": reflect.ValueOf(tensor.Slice), + "SliceSet": reflect.ValueOf(tensor.SliceSet), + "SliceSize": reflect.ValueOf(tensor.SliceSize), + "Space": reflect.ValueOf(tensor.Space), + "Stable": reflect.ValueOf(tensor.Stable), + "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), + "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), + "Tab": reflect.ValueOf(tensor.Tab), + "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), + "Unstable": reflect.ValueOf(tensor.Unstable), + "UseCase": reflect.ValueOf(tensor.UseCase), + "Vectorize": reflect.ValueOf(tensor.Vectorize), + "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), + "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), + "WriteCSV": reflect.ValueOf(tensor.WriteCSV), + + // type definitions + "Bits": reflect.ValueOf((*tensor.Bits)(nil)), + "Delims": reflect.ValueOf((*tensor.Delims)(nil)), + "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), + "Func": reflect.ValueOf((*tensor.Func)(nil)), + "Indexed": reflect.ValueOf((*tensor.Indexed)(nil)), + "Range": reflect.ValueOf((*tensor.Range)(nil)), + "Shape": reflect.ValueOf((*tensor.Shape)(nil)), + "String": reflect.ValueOf((*tensor.String)(nil)), + "Tensor": reflect.ValueOf((*tensor.Tensor)(nil)), + + // interface wrapper definitions + "_Tensor": reflect.ValueOf((*_cogentcore_org_core_tensor_Tensor)(nil)), + } +} + +// _cogentcore_org_core_tensor_Tensor is an interface wrapper for Tensor type +type _cogentcore_org_core_tensor_Tensor struct { + IValue interface{} + WAppendFrom func(from tensor.Tensor) error + WAt func(i int, j int) float64 + WBytes func() []byte + WClone func() tensor.Tensor + WCopyCellsFrom func(from tensor.Tensor, to int, start int, n int) + WCopyFrom func(from tensor.Tensor) + WDataType func() reflect.Kind + WDimSize func(dim int) int + WDims func() (r int, c int) + WFloat func(i ...int) float64 + WFloat1D func(i int) float64 + WFloatRowCell func(row int, cell int) float64 + WInt func(i ...int) int + WInt1D func(i int) int + WIntRowCell func(row int, cell int) int + WIsString func() bool + WLabel func() string + WLen func() int + WMetadata func() *metadata.Data + WNumDims func() int + WRange func() (min float64, max float64, minIndex int, maxIndex int) + WRowCellSize func() (rows int, cells int) + WRowTensor func(row int) tensor.Tensor + WSetFloat func(val float64, i ...int) + WSetFloat1D func(val float64, i int) + WSetFloatRowCell func(val float64, row int, cell int) + WSetInt func(val int, i ...int) + WSetInt1D func(val int, i int) + WSetIntRowCell func(val int, row int, cell int) + WSetNames func(names ...string) + WSetNumRows func(rows int) + WSetRowTensor func(val tensor.Tensor, row int) + WSetShape func(sizes ...int) + WSetShapeFrom func(from tensor.Tensor) + WSetString func(val string, i ...int) + WSetString1D func(val string, i int) + WSetStringRowCell func(val string, row int, cell int) + WSetZeros func() + WShape func() *tensor.Shape + WSizeof func() int64 + WString func() string + WString1D func(i int) string + WStringRowCell func(row int, cell int) string + WStringValue func(i ...int) string + WSubSpace func(offs ...int) tensor.Tensor + WT func() mat.Matrix + WView func() tensor.Tensor +} + +func (W _cogentcore_org_core_tensor_Tensor) AppendFrom(from tensor.Tensor) error { + return W.WAppendFrom(from) +} +func (W _cogentcore_org_core_tensor_Tensor) At(i int, j int) float64 { return W.WAt(i, j) } +func (W _cogentcore_org_core_tensor_Tensor) Bytes() []byte { return W.WBytes() } +func (W _cogentcore_org_core_tensor_Tensor) Clone() tensor.Tensor { return W.WClone() } +func (W _cogentcore_org_core_tensor_Tensor) CopyCellsFrom(from tensor.Tensor, to int, start int, n int) { + W.WCopyCellsFrom(from, to, start, n) +} +func (W _cogentcore_org_core_tensor_Tensor) CopyFrom(from tensor.Tensor) { W.WCopyFrom(from) } +func (W _cogentcore_org_core_tensor_Tensor) DataType() reflect.Kind { return W.WDataType() } +func (W _cogentcore_org_core_tensor_Tensor) DimSize(dim int) int { return W.WDimSize(dim) } +func (W _cogentcore_org_core_tensor_Tensor) Dims() (r int, c int) { return W.WDims() } +func (W _cogentcore_org_core_tensor_Tensor) Float(i ...int) float64 { return W.WFloat(i...) } +func (W _cogentcore_org_core_tensor_Tensor) Float1D(i int) float64 { return W.WFloat1D(i) } +func (W _cogentcore_org_core_tensor_Tensor) FloatRowCell(row int, cell int) float64 { + return W.WFloatRowCell(row, cell) +} +func (W _cogentcore_org_core_tensor_Tensor) Int(i ...int) int { return W.WInt(i...) } +func (W _cogentcore_org_core_tensor_Tensor) Int1D(i int) int { return W.WInt1D(i) } +func (W _cogentcore_org_core_tensor_Tensor) IntRowCell(row int, cell int) int { + return W.WIntRowCell(row, cell) +} +func (W _cogentcore_org_core_tensor_Tensor) IsString() bool { return W.WIsString() } +func (W _cogentcore_org_core_tensor_Tensor) Label() string { return W.WLabel() } +func (W _cogentcore_org_core_tensor_Tensor) Len() int { return W.WLen() } +func (W _cogentcore_org_core_tensor_Tensor) Metadata() *metadata.Data { return W.WMetadata() } +func (W _cogentcore_org_core_tensor_Tensor) NumDims() int { return W.WNumDims() } +func (W _cogentcore_org_core_tensor_Tensor) Range() (min float64, max float64, minIndex int, maxIndex int) { + return W.WRange() +} +func (W _cogentcore_org_core_tensor_Tensor) RowCellSize() (rows int, cells int) { + return W.WRowCellSize() +} +func (W _cogentcore_org_core_tensor_Tensor) RowTensor(row int) tensor.Tensor { + return W.WRowTensor(row) +} +func (W _cogentcore_org_core_tensor_Tensor) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } +func (W _cogentcore_org_core_tensor_Tensor) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } +func (W _cogentcore_org_core_tensor_Tensor) SetFloatRowCell(val float64, row int, cell int) { + W.WSetFloatRowCell(val, row, cell) +} +func (W _cogentcore_org_core_tensor_Tensor) SetInt(val int, i ...int) { W.WSetInt(val, i...) } +func (W _cogentcore_org_core_tensor_Tensor) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } +func (W _cogentcore_org_core_tensor_Tensor) SetIntRowCell(val int, row int, cell int) { + W.WSetIntRowCell(val, row, cell) +} +func (W _cogentcore_org_core_tensor_Tensor) SetNames(names ...string) { W.WSetNames(names...) } +func (W _cogentcore_org_core_tensor_Tensor) SetNumRows(rows int) { W.WSetNumRows(rows) } +func (W _cogentcore_org_core_tensor_Tensor) SetRowTensor(val tensor.Tensor, row int) { + W.WSetRowTensor(val, row) +} +func (W _cogentcore_org_core_tensor_Tensor) SetShape(sizes ...int) { W.WSetShape(sizes...) } +func (W _cogentcore_org_core_tensor_Tensor) SetShapeFrom(from tensor.Tensor) { W.WSetShapeFrom(from) } +func (W _cogentcore_org_core_tensor_Tensor) SetString(val string, i ...int) { W.WSetString(val, i...) } +func (W _cogentcore_org_core_tensor_Tensor) SetString1D(val string, i int) { W.WSetString1D(val, i) } +func (W _cogentcore_org_core_tensor_Tensor) SetStringRowCell(val string, row int, cell int) { + W.WSetStringRowCell(val, row, cell) +} +func (W _cogentcore_org_core_tensor_Tensor) SetZeros() { W.WSetZeros() } +func (W _cogentcore_org_core_tensor_Tensor) Shape() *tensor.Shape { return W.WShape() } +func (W _cogentcore_org_core_tensor_Tensor) Sizeof() int64 { return W.WSizeof() } +func (W _cogentcore_org_core_tensor_Tensor) String() string { + if W.WString == nil { + return "" + } + return W.WString() +} +func (W _cogentcore_org_core_tensor_Tensor) String1D(i int) string { return W.WString1D(i) } +func (W _cogentcore_org_core_tensor_Tensor) StringRowCell(row int, cell int) string { + return W.WStringRowCell(row, cell) +} +func (W _cogentcore_org_core_tensor_Tensor) StringValue(i ...int) string { return W.WStringValue(i...) } +func (W _cogentcore_org_core_tensor_Tensor) SubSpace(offs ...int) tensor.Tensor { + return W.WSubSpace(offs...) +} +func (W _cogentcore_org_core_tensor_Tensor) T() mat.Matrix { return W.WT() } +func (W _cogentcore_org_core_tensor_Tensor) View() tensor.Tensor { return W.WView() } diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 67bc088dec..8fa61ca4c8 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -8,4 +8,4 @@ command extract { } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor htmlcore pages paint math32 plot plot/plots plot/plotcore tensor/table base/errors base/fsx base/reflectx base/labels base/fileinfo +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo From f07ab1b825d4abea33514caf1fc03c4fc882fef2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 23:06:29 -0700 Subject: [PATCH 047/311] all building and tests passing --- tensor/cmd/tablecat/tablecat.go | 39 ++++++++++++----------- tensor/examples/grids/grids.go | 11 +++---- tensor/stats/convolve/table.go | 9 ++---- tensor/stats/glm/glm.go | 56 +++++++++++++-------------------- tensor/stats/stats/group.go | 30 ++++++++++++------ tensor/tensorcore/table.go | 2 +- tensor/tensorcore/typegen.go | 10 +++--- tensor/tensormpi/table.go | 16 +++++----- 8 files changed, 84 insertions(+), 89 deletions(-) diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index 9dad1cf3b6..afabc8f4ad 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -13,9 +13,9 @@ import ( "cogentcore.org/core/core" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/table" - "github.com/emer/etable/v2/split" ) var ( @@ -111,12 +111,12 @@ func AvgCat(files []string) { dts := make([]*table.Table, 0, len(files)) for _, fn := range files { dt := table.NewTable() - err := dt.OpenCSV(core.Filename(fn), table.Tab) + err := dt.OpenCSV(core.Filename(fn), tensor.Tab) if err != nil { fmt.Println("Error opening file: ", err) continue } - if dt.Rows == 0 { + if dt.NumRows() == 0 { fmt.Printf("File %v empty\n", fn) continue } @@ -126,9 +126,10 @@ func AvgCat(files []string) { fmt.Println("No files or files are empty, exiting") return } - avgdt := stats.MeanTables(dts) - tensor.SetPrecision(avgdt, LogPrec) - avgdt.SaveCSV(core.Filename(Output), table.Tab, table.Headers) + // todo: need meantables + // avgdt := stats.MeanTables(dts) + // tensor.SetPrecision(avgdt, LogPrec) + // avgdt.SaveCSV(core.Filename(Output), tensor.Tab, table.Headers) } // AvgByColumn computes average by given column for given files @@ -136,30 +137,32 @@ func AvgCat(files []string) { func AvgByColumn(files []string, column string) { for _, fn := range files { dt := table.NewTable() - err := dt.OpenCSV(core.Filename(fn), table.Tab) + err := dt.OpenCSV(core.Filename(fn), tensor.Tab) if err != nil { fmt.Println("Error opening file: ", err) continue } - if dt.Rows == 0 { + if dt.NumRows() == 0 { fmt.Printf("File %v empty\n", fn) continue } - ix := table.NewIndexed(dt) - var spl *table.Splits + dir, _ := datafs.NewDir("Groups") if column == "" { - spl = split.All(ix) + stats.GroupAll(dir, dt.ColumnByIndex(0)) } else { - spl = split.GroupBy(ix, column) + stats.TableGroups(dir, dt, column) } - for ci, cl := range dt.Columns { - if cl.IsString() || dt.ColumnNames[ci] == column { + var cols []string + for ci, cl := range dt.Columns.Values { + if cl.IsString() || dt.Columns.Keys[ci] == column { continue } - split.AggIndex(spl, ci, stats.Mean) + cols = append(cols, dt.Columns.Keys[ci]) } - avgdt := spl.AggsToTable(table.ColumnNameOnly) - tensor.SetPrecision(avgdt, LogPrec) - avgdt.SaveCSV(core.Filename(Output), table.Tab, table.Headers) + stats.TableGroupStats(dir, stats.Mean.FuncName(), dt, cols...) + std := dir.Item("Stats") + avgdt := std.GetDirTable(nil) // todo: has stat name slash + tensor.SetPrecision(avgdt.Meta, LogPrec) + avgdt.SaveCSV(core.Filename(Output), tensor.Tab, table.Headers) } } diff --git a/tensor/examples/grids/grids.go b/tensor/examples/grids/grids.go index f56658e1d6..9415813604 100644 --- a/tensor/examples/grids/grids.go +++ b/tensor/examples/grids/grids.go @@ -9,6 +9,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/core" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" "cogentcore.org/core/tensor/tensorcore" ) @@ -17,17 +18,13 @@ import ( var tsv embed.FS func main() { - pats := table.NewTable("pats") - pats.SetMetaData("name", "TrainPats") - pats.SetMetaData("desc", "Training patterns") + pats := table.NewTable("TrainPats") + pats.Meta.SetDoc("Training patterns") // todo: meta data for grid size - errors.Log(pats.OpenFS(tsv, "random_5x5_25.tsv", table.Tab)) + errors.Log(pats.OpenFS(tsv, "random_5x5_25.tsv", tensor.Tab)) b := core.NewBody("grids") - tv := core.NewTabs(b) - - // nt, _ := tv.NewTab("First") nt, _ := tv.NewTab("Patterns") etv := tensorcore.NewTable(nt).SetTable(pats) b.AddTopBar(func(bar *core.Frame) { diff --git a/tensor/stats/convolve/table.go b/tensor/stats/convolve/table.go index 1381e21ef7..ebe2ea88a2 100644 --- a/tensor/stats/convolve/table.go +++ b/tensor/stats/convolve/table.go @@ -4,13 +4,7 @@ package convolve -import ( - "reflect" - - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/table" -) - +/* // SmoothTable returns a cloned table with each of the floating-point // columns in the source table smoothed over rows. // khalf is the half-width of the Gaussian smoothing kernel, @@ -34,3 +28,4 @@ func SmoothTable(src *table.Table, khalf int) *table.Table { } return dest } +*/ diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index 832611dc96..a3bcd155b6 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -88,7 +88,7 @@ type GLM struct { Table *table.Table // tensor columns from table with the respective variables - IndepVars, DepVars, PredVars, ErrVars tensor.Tensor + IndepVars, DepVars, PredVars, ErrVars *tensor.Indexed // Number of independent and dependent variables NIndepVars, NDepVars int @@ -110,7 +110,8 @@ func (glm *GLM) Defaults() { func (glm *GLM) init(nIv, nDv int) { glm.NIndepVars = nIv glm.NDepVars = nDv - glm.Coeff.SetShape([]int{nDv, nIv + 1}, "DepVars", "IndepVars") + glm.Coeff.SetShape(nDv, nIv+1) + glm.Coeff.SetNames("DepVars", "IndepVars") glm.R2 = make([]float64, nDv) glm.ObsVariance = make([]float64, nDv) glm.ErrVariance = make([]float64, nDv) @@ -122,39 +123,26 @@ func (glm *GLM) init(nIv, nDv int) { // each of the Vars args specifies a column in the table, which can have either a // single scalar value for each row, or a tensor cell with multiple values. // predVars and errVars (predicted values and error values) are optional. -func (glm *GLM) SetTable(ix *table.Table, indepVars, depVars, predVars, errVars string) error { - dt := ix.Table - iv, err := dt.Column(indepVars) - if err != nil { - return err - } - dv, err := dt.Column(depVars) - if err != nil { - return err - } - var pv, ev tensor.Tensor +func (glm *GLM) SetTable(dt *table.Table, indepVars, depVars, predVars, errVars string) error { + iv := dt.Column(indepVars) + dv := dt.Column(depVars) + var pv, ev *tensor.Indexed if predVars != "" { - pv, err = dt.Column(predVars) - if err != nil { - return err - } + pv = dt.Column(predVars) } if errVars != "" { - ev, err = dt.Column(errVars) - if err != nil { - return err - } + ev = dt.Column(errVars) } - if pv != nil && !pv.Shape().IsEqual(dv.Shape()) { + if pv != nil && !pv.Tensor.Shape().IsEqual(dv.Tensor.Shape()) { return fmt.Errorf("predVars must have same shape as depVars") } - if ev != nil && !ev.Shape().IsEqual(dv.Shape()) { + if ev != nil && !ev.Tensor.Shape().IsEqual(dv.Tensor.Shape()) { return fmt.Errorf("errVars must have same shape as depVars") } _, nIv := iv.RowCellSize() _, nDv := dv.RowCellSize() glm.init(nIv, nDv) - glm.Table = ix + glm.Table = dt glm.IndepVars = iv glm.DepVars = dv glm.PredVars = pv @@ -168,7 +156,7 @@ func (glm *GLM) SetTable(ix *table.Table, indepVars, depVars, predVars, errVars // Initial values of the coefficients, and other parameters for the regression, // should be set prior to running. func (glm *GLM) Run() { - ix := glm.Table + dt := glm.Table iv := glm.IndepVars dv := glm.DepVars pv := glm.PredVars @@ -190,7 +178,7 @@ func (glm *GLM) Run() { lastItr := false sse := 0.0 prevmse := 0.0 - n := ix.Len() + n := dt.NumRows() norm := 1.0 / float64(n) lrate := norm * glm.LRate for itr := 0; itr < glm.MaxIters; itr++ { @@ -202,14 +190,14 @@ func (glm *GLM) Run() { lrate *= 0.5 } for i := 0; i < n; i++ { - row := ix.Indexes[i] + row := dt.Index(i) for di := 0; di < nDv; di++ { pred := 0.0 for ii := 0; ii < nIv; ii++ { - pred += glm.Coeff.Value([]int{di, ii}) * iv.FloatRowCell(row, ii) + pred += glm.Coeff.Float(di, ii) * iv.FloatRowCell(row, ii) } if !glm.ZeroOffset { - pred += glm.Coeff.Value([]int{di, nIv}) + pred += glm.Coeff.Float(di, nIv) } targ := dv.FloatRowCell(row, di) err := targ - pred @@ -221,9 +209,9 @@ func (glm *GLM) Run() { dc.Values[di*nCi+nIv] += err } if lastItr { - pv.SetFloatRowCell(row, di, pred) + pv.SetFloatRowCell(pred, row, di) if ev != nil { - ev.SetFloatRowCell(row, di, err) + ev.SetFloatRowCell(err, row, di) } } } @@ -262,7 +250,7 @@ func (glm *GLM) Run() { obsMeans := make([]float64, nDv) errMeans := make([]float64, nDv) for i := 0; i < n; i++ { - row := ix.Indexes[i] + row := dt.Indexes[i] for di := 0; di < nDv; di++ { obsMeans[di] += dv.FloatRowCell(row, di) errMeans[di] += ev.FloatRowCell(row, di) @@ -275,7 +263,7 @@ func (glm *GLM) Run() { glm.ErrVariance[di] = 0 } for i := 0; i < n; i++ { - row := ix.Indexes[i] + row := dt.Indexes[i] for di := 0; di < nDv; di++ { o := dv.FloatRowCell(row, di) - obsMeans[di] glm.ObsVariance[di] += o * o @@ -317,7 +305,7 @@ func (glm *GLM) Coeffs() string { } str += " = " for ii := 0; ii <= glm.NIndepVars; ii++ { - str += fmt.Sprintf("\t%8.6g", glm.Coeff.Value([]int{di, ii})) + str += fmt.Sprintf("\t%8.6g", glm.Coeff.Float(di, ii)) if ii < glm.NIndepVars { str += " * " if len(glm.IndepNames) > ii && glm.IndepNames[di] != "" { diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 1dd164b61e..50c9299ca0 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -14,15 +14,6 @@ import ( "cogentcore.org/core/tensor/table" ) -// All returns a single "split" with all of the rows in given view -// useful for leveraging the aggregation management functions in splits -// func All(ix *table.Table) *table.Splits { -// spl := &table.Splits{} -// spl.Levels = []string{"All"} -// spl.New(ix.Table, []string{"All"}, ix.Indexes...) -// return spl -// } - // Groups generates indexes for each unique value in each of the given tensors. // One can then use the resulting indexes for the [tensor.Indexed] indexes to // perform computations restricted to grouped subsets of data, as in the @@ -104,6 +95,27 @@ func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { Groups(dir, dv.ColumnList(columns...)...) } +// GroupAll copies all indexes from the first given tensor, +// into an "All/All" tensor in the given [datafs], which can then +// be used with [GroupStats] to generate summary statistics across +// all the data. See [Groups] for more general documentation. +func GroupAll(dir *datafs.Data, tsrs ...*tensor.Indexed) { + gd, err := dir.RecycleDir("Groups") + if errors.Log(err) != nil { + return + } + tsr := tsrs[0] + nr := tsr.NumRows() + if nr == 0 { + return + } + td, _ := gd.Mkdir("All") + it := datafs.NewValue[int](td, "All", nr) + for j := range nr { + it.SetIntRow(tsr.Index(j), j) // key to indirect through any existing indexes + } +} + // todo: GroupCombined // GroupStats computes the given stats function on the unique grouped indexes diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 3c1b2087d9..b7ce7a57dc 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -634,7 +634,7 @@ func (tb *Table) SelectedColumnStrings(colName string) []string { // Copy / Cut / Paste func (tb *Table) MakeToolbar(p *tree.Plan) { - if tb.Table == nil || tb.Table == nil { + if tb.Table == nil { return } tree.Add(p, func(w *core.FuncButton) { diff --git a/tensor/tensorcore/typegen.go b/tensor/tensorcore/typegen.go index 8a42d7c352..f1ce441040 100644 --- a/tensor/tensorcore/typegen.go +++ b/tensor/tensorcore/typegen.go @@ -10,27 +10,27 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/tensorcore.Table", IDName: "table", Doc: "Table provides a GUI widget for representing [table.Table] values.", Embeds: []types.Field{{Name: "ListBase"}}, Fields: []types.Field{{Name: "Table", Doc: "the idx view of the table that we're a view of"}, {Name: "TensorDisplay", Doc: "overall display options for tensor display"}, {Name: "ColumnTensorDisplay", Doc: "per column tensor display params"}, {Name: "ColumnTensorBlank", Doc: "per column blank tensor values"}, {Name: "NCols", Doc: "number of columns in table (as of last update)"}, {Name: "SortIndex", Doc: "current sort index"}, {Name: "SortDescending", Doc: "whether current sort order is descending"}, {Name: "headerWidths", Doc: "headerWidths has number of characters in each header, per visfields"}, {Name: "colMaxWidths", Doc: "colMaxWidths records maximum width in chars of string type fields"}, {Name: "BlankString", Doc: "\tblank values for out-of-range rows"}, {Name: "BlankFloat"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/tensorcore.Table", IDName: "table", Doc: "Table provides a GUI widget for representing [table.Table] values.", Embeds: []types.Field{{Name: "ListBase"}}, Fields: []types.Field{{Name: "Table", Doc: "Table is the table that we're a view of."}, {Name: "TensorDisplay", Doc: "overall display options for tensor display."}, {Name: "ColumnTensorDisplay", Doc: "per column tensor display params."}, {Name: "ColumnTensorBlank", Doc: "per column blank tensor values."}, {Name: "NCols", Doc: "number of columns in table (as of last update)."}, {Name: "SortIndex", Doc: "current sort index."}, {Name: "SortDescending", Doc: "whether current sort order is descending."}, {Name: "headerWidths", Doc: "headerWidths has number of characters in each header, per visfields."}, {Name: "colMaxWidths", Doc: "colMaxWidths records maximum width in chars of string type fields."}, {Name: "BlankString", Doc: "\tblank values for out-of-range rows."}, {Name: "BlankFloat"}}}) // NewTable returns a new [Table] with the given optional parent: // Table provides a GUI widget for representing [table.Table] values. func NewTable(parent ...tree.Node) *Table { return tree.New[Table](parent...) } // SetNCols sets the [Table.NCols]: -// number of columns in table (as of last update) +// number of columns in table (as of last update). func (t *Table) SetNCols(v int) *Table { t.NCols = v; return t } // SetSortIndex sets the [Table.SortIndex]: -// current sort index +// current sort index. func (t *Table) SetSortIndex(v int) *Table { t.SortIndex = v; return t } // SetSortDescending sets the [Table.SortDescending]: -// whether current sort order is descending +// whether current sort order is descending. func (t *Table) SetSortDescending(v bool) *Table { t.SortDescending = v; return t } // SetBlankString sets the [Table.BlankString]: // -// blank values for out-of-range rows +// blank values for out-of-range rows. func (t *Table) SetBlankString(v string) *Table { t.BlankString = v; return t } // SetBlankFloat sets the [Table.BlankFloat] diff --git a/tensor/tensormpi/table.go b/tensor/tensormpi/table.go index da874b1a8a..c21908293d 100644 --- a/tensor/tensormpi/table.go +++ b/tensor/tensormpi/table.go @@ -13,15 +13,15 @@ import ( // dest will have np * src.Rows Rows, filled with each processor's data, in order. // dest must be a clone of src: if not same number of cols, will be configured from src. func GatherTableRows(dest, src *table.Table, comm *mpi.Comm) { - sr := src.Rows + sr := src.NumRows() np := mpi.WorldSize() dr := np * sr - if len(dest.Columns) != len(src.Columns) { + if dest.NumColumns() != src.NumColumns() { *dest = *src.Clone() } dest.SetNumRows(dr) - for ci, st := range src.Columns { - dt := dest.Columns[ci] + for ci, st := range src.Columns.Values { + dt := dest.Columns.Values[ci] GatherTensorRows(dt, st, comm) } } @@ -33,13 +33,13 @@ func GatherTableRows(dest, src *table.Table, comm *mpi.Comm) { // dest will be a clone of src if not the same (cos & rows), // does nothing for strings. func ReduceTable(dest, src *table.Table, comm *mpi.Comm, op mpi.Op) { - sr := src.Rows - if len(dest.Columns) != len(src.Columns) { + sr := src.NumRows() + if dest.NumColumns() != src.NumColumns() { *dest = *src.Clone() } dest.SetNumRows(sr) - for ci, st := range src.Columns { - dt := dest.Columns[ci] + for ci, st := range src.Columns.Values { + dt := dest.Columns.Values[ci] ReduceTensor(dt, st, comm, op) } } From 1d82fe005cc821764f0fc665e72df48aecee0e8a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 23:29:16 -0700 Subject: [PATCH 048/311] fix randx tests --- base/randx/dists_test.go | 119 ++++++++++++++------------------------- 1 file changed, 43 insertions(+), 76 deletions(-) diff --git a/base/randx/dists_test.go b/base/randx/dists_test.go index 63fedb24df..936e26a78b 100644 --- a/base/randx/dists_test.go +++ b/base/randx/dists_test.go @@ -8,7 +8,7 @@ import ( "math" "testing" - "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/table" ) @@ -25,18 +25,15 @@ func TestGaussianGen(t *testing.T) { for i := 0; i < nsamp; i++ { vl := GaussianGen(mean, sig) - dt.SetFloat("Val", i, vl) + dt.Column("Val").SetFloatRow(vl, i) } - ix := table.NewIndexed(dt) - desc := stats.DescAll(ix) + dir, _ := datafs.NewDir("Desc") + stats.DescribeTableAll(dir, dt) + desc := dir.GetDirTable(nil) + // fmt.Println(desc.Columns.Keys) - meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] - stdRow := errors.Log1(desc.RowsByString("Stat", "Std", table.Equals, table.UseCase))[0] - // minRow := errors.Log1(desc.RowsByString("Stat", "Min", table.Equals, table.UseCase))[0] - // maxRow := errors.Log1(desc.RowsByString("Stat", "Max", table.Equals, table.UseCase))[0] - - actMean := desc.Float("Val", meanRow) - actStd := desc.Float("Val", stdRow) + actMean := desc.Column("Val/Mean").FloatRow(0) + actStd := desc.Column("Val/Std").FloatRow(0) if math.Abs(actMean-mean) > tol { t.Errorf("Gaussian: mean %g\t out of tolerance vs target: %g\n", actMean, mean) @@ -44,9 +41,6 @@ func TestGaussianGen(t *testing.T) { if math.Abs(actStd-sig) > tol { t.Errorf("Gaussian: stdev %g\t out of tolerance vs target: %g\n", actStd, sig) } - // b := bytes.NewBuffer(nil) - // desc.WriteCSV(b, table.Tab, table.Headers) - // fmt.Printf("%s\n", string(b.Bytes())) } func TestBinomialGen(t *testing.T) { @@ -61,21 +55,15 @@ func TestBinomialGen(t *testing.T) { for i := 0; i < nsamp; i++ { vl := BinomialGen(n, p) - dt.SetFloat("Val", i, vl) - } - ix := table.NewIndexed(dt) - desc := stats.DescAll(ix) - - meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] - stdRow := errors.Log1(desc.RowsByString("Stat", "Std", table.Equals, table.UseCase))[0] - minRow := errors.Log1(desc.RowsByString("Stat", "Min", table.Equals, table.UseCase))[0] - maxRow := errors.Log1(desc.RowsByString("Stat", "Max", table.Equals, table.UseCase))[0] - - actMean := desc.Float("Val", meanRow) - actStd := desc.Float("Val", stdRow) - actMin := desc.Float("Val", minRow) - actMax := desc.Float("Val", maxRow) - + dt.Column("Val").SetFloat(vl, i) + } + dir, _ := datafs.NewDir("Desc") + stats.DescribeTableAll(dir, dt) + desc := dir.GetDirTable(nil) + actMean := desc.Column("Val/Mean").FloatRow(0) + actStd := desc.Column("Val/Std").FloatRow(0) + actMin := desc.Column("Val/Min").FloatRow(0) + actMax := desc.Column("Val/Max").FloatRow(0) mean := n * p if math.Abs(actMean-mean) > tol { t.Errorf("Binomial: mean %g\t out of tolerance vs target: %g\n", actMean, mean) @@ -90,9 +78,6 @@ func TestBinomialGen(t *testing.T) { if actMax < 0 { t.Errorf("Binomial: max %g\t should not be > 1\n", actMax) } - // b := bytes.NewBuffer(nil) - // desc.WriteCSV(b, table.Tab, table.Headers) - // fmt.Printf("%s\n", string(b.Bytes())) } func TestPoissonGen(t *testing.T) { @@ -106,20 +91,15 @@ func TestPoissonGen(t *testing.T) { for i := 0; i < nsamp; i++ { vl := PoissonGen(lambda) - dt.SetFloat("Val", i, vl) + dt.Column("Val").SetFloatRow(vl, i) } - ix := table.NewIndexed(dt) - desc := stats.DescAll(ix) - - meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] - stdRow := errors.Log1(desc.RowsByString("Stat", "Std", table.Equals, table.UseCase))[0] - minRow := errors.Log1(desc.RowsByString("Stat", "Min", table.Equals, table.UseCase))[0] - // maxRow := errors.Log1(desc.RowsByString("Stat", "Max", table.Equals, table.UseCase))[0] - - actMean := desc.Float("Val", meanRow) - actStd := desc.Float("Val", stdRow) - actMin := desc.Float("Val", minRow) - // actMax := desc.Float("Val", maxRow) + dir, _ := datafs.NewDir("Desc") + stats.DescribeTableAll(dir, dt) + desc := dir.GetDirTable(nil) + actMean := desc.Column("Val/Mean").FloatRow(0) + actStd := desc.Column("Val/Std").FloatRow(0) + actMin := desc.Column("Val/Min").FloatRow(0) + // actMax := desc.Column("Val/Max").FloatRow(0) mean := lambda if math.Abs(actMean-mean) > tol { @@ -135,9 +115,6 @@ func TestPoissonGen(t *testing.T) { // if actMax < 0 { // t.Errorf("Poisson: max %g\t should not be > 1\n", actMax) // } - // b := bytes.NewBuffer(nil) - // desc.WriteCSV(b, table.Tab, table.Headers) - // fmt.Printf("%s\n", string(b.Bytes())) } func TestGammaGen(t *testing.T) { @@ -152,17 +129,15 @@ func TestGammaGen(t *testing.T) { for i := 0; i < nsamp; i++ { vl := GammaGen(alpha, beta) - dt.SetFloat("Val", i, vl) - } - ix := table.NewIndexed(dt) - desc := stats.DescAll(ix) - - meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] - stdRow := errors.Log1(desc.RowsByString("Stat", "Std", table.Equals, table.UseCase))[0] - - actMean := desc.Float("Val", meanRow) - actStd := desc.Float("Val", stdRow) - + dt.Column("Val").SetFloatRow(vl, i) + } + dir, _ := datafs.NewDir("Desc") + stats.DescribeTableAll(dir, dt) + desc := dir.GetDirTable(nil) + actMean := desc.Column("Val/Mean").FloatRow(0) + actStd := desc.Column("Val/Std").FloatRow(0) + // actMin := desc.Column("Val/Min").FloatRow(0) + // actMax := desc.Column("Val/Max").FloatRow(0) mean := alpha / beta if math.Abs(actMean-mean) > tol { t.Errorf("Gamma: mean %g\t out of tolerance vs target: %g\n", actMean, mean) @@ -171,9 +146,6 @@ func TestGammaGen(t *testing.T) { if math.Abs(actStd-sig) > tol { t.Errorf("Gamma: stdev %g\t out of tolerance vs target: %g\n", actStd, sig) } - // b := bytes.NewBuffer(nil) - // desc.WriteCSV(b, table.Tab, table.Headers) - // fmt.Printf("%s\n", string(b.Bytes())) } func TestBetaGen(t *testing.T) { @@ -188,17 +160,15 @@ func TestBetaGen(t *testing.T) { for i := 0; i < nsamp; i++ { vl := BetaGen(alpha, beta) - dt.SetFloat("Val", i, vl) - } - ix := table.NewIndexed(dt) - desc := stats.DescAll(ix) - - meanRow := errors.Log1(desc.RowsByString("Stat", "Mean", table.Equals, table.UseCase))[0] - stdRow := errors.Log1(desc.RowsByString("Stat", "Std", table.Equals, table.UseCase))[0] - - actMean := desc.Float("Val", meanRow) - actStd := desc.Float("Val", stdRow) - + dt.Column("Val").SetFloatRow(vl, i) + } + dir, _ := datafs.NewDir("Desc") + stats.DescribeTableAll(dir, dt) + desc := dir.GetDirTable(nil) + actMean := desc.Column("Val/Mean").FloatRow(0) + actStd := desc.Column("Val/Std").FloatRow(0) + // actMin := desc.Column("Val/Min").FloatRow(0) + // actMax := desc.Column("Val/Max").FloatRow(0) mean := alpha / (alpha + beta) if math.Abs(actMean-mean) > tol { t.Errorf("Beta: mean %g\t out of tolerance vs target: %g\n", actMean, mean) @@ -208,7 +178,4 @@ func TestBetaGen(t *testing.T) { if math.Abs(actStd-sig) > tol { t.Errorf("Beta: stdev %g\t out of tolerance vs target: %g\n", actStd, sig) } - // b := bytes.NewBuffer(nil) - // desc.WriteCSV(b, table.Tab, table.Headers) - // fmt.Printf("%s\n", string(b.Bytes())) } From 0e2c21cf4260892a7c1b38a40f803d4b33df1dda Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 23:32:52 -0700 Subject: [PATCH 049/311] fix plotcore tests --- enums/enumgen/testdata/enumgen.go | 2 +- plot/plotcore/ploteditor.go | 2 +- plot/plotcore/ploteditor_test.go | 3 ++- plot/plotcore/tablexy.go | 4 ++-- types/typegen/testdata/typegen.go | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/enums/enumgen/testdata/enumgen.go b/enums/enumgen/testdata/enumgen.go index c2481dfa1d..48f2da15e6 100644 --- a/enums/enumgen/testdata/enumgen.go +++ b/enums/enumgen/testdata/enumgen.go @@ -1,4 +1,4 @@ -// Code generated by "enumgen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build2217811126/b647/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. +// Code generated by "enumgen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build16270102/b655/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. package testdata diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 0d68d7fb2d..95febfe8f6 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -284,7 +284,7 @@ func (pl *PlotEditor) GoUpdatePlot() { if core.TheApp.Platform() == system.Web { time.Sleep(time.Millisecond) // critical to prevent hanging! } - if !pl.IsVisible() || pl.table == nil || pl.table == nil || pl.inPlot { + if !pl.IsVisible() || pl.table == nil || pl.inPlot { return } pl.Scene.AsyncLock() diff --git a/plot/plotcore/ploteditor_test.go b/plot/plotcore/ploteditor_test.go index 2ecaa13bd8..411f15943b 100644 --- a/plot/plotcore/ploteditor_test.go +++ b/plot/plotcore/ploteditor_test.go @@ -8,6 +8,7 @@ import ( "testing" "cogentcore.org/core/core" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) @@ -21,7 +22,7 @@ func TestTablePlotEditor(t *testing.T) { b := core.NewBody() epc := table.NewTable("epc") - epc.OpenCSV("testdata/ra25epoch.tsv", table.Tab) + epc.OpenCSV("testdata/ra25epoch.tsv", tensor.Tab) pl := NewPlotEditor(b) pl.Options.Title = "RA25 Epoch Train" diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index da94a4e0e2..e8e4e647f1 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -167,7 +167,7 @@ func (txy *tableXY) Value(row int) float32 { // tRowXValue returns an x value at given actual row in table func (txy *tableXY) tRowXValue(row int) float32 { - if txy.table == nil || txy.table == nil { + if txy.table == nil { return 0 } xc := txy.table.ColumnByIndex(txy.xColumn) @@ -210,7 +210,7 @@ func (txy *tableXY) xValue(row int) float32 { // XY returns an x, y pair at given row in table func (txy *tableXY) XY(row int) (x, y float32) { - if txy.table == nil || txy.table == nil { + if txy.table == nil { return 0, 0 } x = txy.xValue(row) diff --git a/types/typegen/testdata/typegen.go b/types/typegen/testdata/typegen.go index d42cc65c16..117616b8ec 100644 --- a/types/typegen/testdata/typegen.go +++ b/types/typegen/testdata/typegen.go @@ -1,4 +1,4 @@ -// Code generated by "typegen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build2217811126/b974/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. +// Code generated by "typegen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build16270102/b976/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. package testdata From 8966509df1185b99e0f50e7bf2fdfd75e295701f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 16 Sep 2024 23:42:58 -0700 Subject: [PATCH 050/311] pca / svg tolerance --- tensor/stats/metric/metric_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index a6634cfd12..ea50cefe28 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -140,7 +140,10 @@ func TestPCAIris(t *testing.T) { 2.4615836931965167, 2.6716628159090594, } - assert.Equal(t, trgprjns, tensor.AsFloat64(prjns.Tensor).Values[:10]) + pj64 := tensor.AsFloat64(prjns.Tensor) + for i, v := range pj64.Values[:10] { + assert.InDelta(t, trgprjns[i], v, errtol) + } //////////////////////////////////////////////////////////// // SVD @@ -167,5 +170,8 @@ func TestPCAIris(t *testing.T) { -2.4615836931965185, -2.671662815909061, } - assert.Equal(t, trgprjns, tensor.AsFloat64(prjns.Tensor).Values[:10]) + pj64 = tensor.AsFloat64(prjns.Tensor) + for i, v := range pj64.Values[:10] { + assert.InDelta(t, trgprjns[i], v, errtol) + } } From 9dccd5a862113d3c7fa654abb1e1fee6a129d981 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 09:05:40 -0700 Subject: [PATCH 051/311] tensor robust to rows = 0; stop enforcing the rows = 1 constraint --- tensor/README.md | 2 +- tensor/base.go | 4 +++- tensor/bits.go | 4 +++- tensor/examples/datafs-sim/sim.go | 14 +++++-------- tensor/shape.go | 18 +++++++++++++---- tensor/table/columns.go | 12 +++++------ tensor/table/table.go | 6 +++--- tensor/tensor.go | 3 +++ tensor/tensor_test.go | 33 ++++++++++++++++++++++++++++++- 9 files changed, 69 insertions(+), 27 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index c75b4aa780..a526c0a0f9 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -52,7 +52,7 @@ In general, **1D** refers to a flat, 1-dimensional list. There are various stan `ix` is the `Indexed` tensor for these examples: -## Table Access +## Tensor Access ### 1D diff --git a/tensor/base.go b/tensor/base.go index 8890a01976..c5140a7d74 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -96,8 +96,10 @@ func (tsr *Base[T]) SetNames(names ...string) { } // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. +// It is safe to set this to 0. For incrementally growing tensors (e.g., a log) +// it is best to first set the anticipated full size, which allocates the +// full amount of memory, and then set to 0 and grow incrementally. func (tsr *Base[T]) SetNumRows(rows int) { - rows = max(1, rows) // must be > 0 _, cells := tsr.shape.RowCellSize() nln := rows * cells tsr.shape.Sizes[0] = rows diff --git a/tensor/bits.go b/tensor/bits.go index 5b7e778815..a1f6829540 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -137,8 +137,10 @@ func (tsr *Bits) SetNames(names ...string) { } // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. +// It is safe to set this to 0. For incrementally growing tensors (e.g., a log) +// it is best to first set the anticipated full size, which allocates the +// full amount of memory, and then set to 0 and grow incrementally. func (tsr *Bits) SetNumRows(rows int) { - rows = max(1, rows) // must be > 0 _, cells := tsr.shape.RowCellSize() nln := rows * cells tsr.shape.Sizes[0] = rows diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 8a5cb41124..a0064118e5 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -81,17 +81,13 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { alllogd, _ := dir.Mkdir("AllTrials") for _, st := range sitems { nm := st.Tensor.Metadata().GetName() - lt := alllogd.NewOfType(nm, st.Tensor.DataType()) + // allocate full size + lt := alllogd.NewOfType(nm, st.Tensor.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) + lt.Tensor.SetShape(0) // then truncate to 0 lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source tensor.SetCalcFunc(lt.Tensor, func() error { - // todo: helper for below - row := 0 - if lt.Tensor.NumDims() == 0 { - lt.Tensor.SetShape(1) - } else { - row = lt.Tensor.DimSize(0) - lt.Tensor.SetShape(row + 1) - } + row := lt.Tensor.DimSize(0) + lt.Tensor.SetShape(row + 1) if st.Tensor.IsString() { lt.SetStringRow(st.StringRow(0), row) } else { diff --git a/tensor/shape.go b/tensor/shape.go index 638030561b..8e8c49efa2 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -48,6 +48,7 @@ func (sh *Shape) SetNames(names ...string) { sh.Names = nil } else { sh.Names = slicesx.SetLength(names, len(sh.Sizes)) + copy(sh.Names, names) } } @@ -69,11 +70,11 @@ func (sh *Shape) Len() int { if len(sh.Sizes) == 0 { return 0 } - o := int(1) + ln := 1 for _, v := range sh.Sizes { - o *= v + ln *= v } - return int(o) + return ln } // NumDims returns the total number of dimensions. @@ -143,8 +144,14 @@ func (sh *Shape) RowCellSize() (rows, cells int) { rows = sh.Sizes[0] if len(sh.Sizes) == 1 { cells = 1 - } else { + } else if rows > 0 { cells = sh.Len() / rows + } else { + ln := 1 + for _, v := range sh.Sizes[1:] { + ln *= v + } + cells = ln } return } @@ -166,6 +173,9 @@ func (sh *Shape) Index(offset int) []int { rem := offset for i := nd - 1; i >= 0; i-- { s := sh.Sizes[i] + if s == 0 { + return index + } iv := rem % s rem /= s index[i] = iv diff --git a/tensor/table/columns.go b/tensor/table/columns.go index 9dfc8d2add..4881831051 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -24,11 +24,11 @@ func NewColumns() *Columns { } // SetNumRows sets the number of rows in the table, across all columns. -// If rows = 0 then the effective number of rows in tensors is 1, -// as this dim cannot be 0. +// It is safe to set this to 0. For incrementally growing tables (e.g., a log) +// it is best to first set the anticipated full size, which allocates the +// full amount of memory, and then set to 0 and grow incrementally. func (cl *Columns) SetNumRows(rows int) *Columns { //types:add cl.Rows = rows // can be 0 - rows = max(1, rows) for _, tsr := range cl.Values { tsr.SetNumRows(rows) } @@ -44,8 +44,7 @@ func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { if err != nil { return err } - rows := max(1, cl.Rows) - tsr.SetNumRows(rows) + tsr.SetNumRows(cl.Rows) tsr.Metadata().SetName(name) return nil } @@ -58,8 +57,7 @@ func (cl *Columns) InsertColumn(idx int, name string, tsr tensor.Tensor) error { if err != nil { return err } - rows := max(1, cl.Rows) - tsr.SetNumRows(rows) + tsr.SetNumRows(cl.Rows) return nil } diff --git a/tensor/table/table.go b/tensor/table/table.go index ae27afb406..e5adfa611d 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -148,7 +148,7 @@ func (dt *Table) ColumnIndexList(names ...string) []int { // (which must be unique). If no cellSizes are specified, it holds scalar values, // otherwise the cells are n-dimensional tensors of given size. func AddColumn[T tensor.DataTypes](dt *Table, name string, cellSizes ...int) tensor.Tensor { - rows := max(1, dt.Columns.Rows) + rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) tsr := tensor.New[T](sz...) tsr.SetNames("Row") @@ -161,7 +161,7 @@ func AddColumn[T tensor.DataTypes](dt *Table, name string, cellSizes ...int) ten // If no cellSizes are specified, it holds scalar values, // otherwise the cells are n-dimensional tensors of given size. func InsertColumn[T tensor.DataTypes](dt *Table, name string, idx int, cellSizes ...int) tensor.Tensor { - rows := max(1, dt.Columns.Rows) + rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) tsr := tensor.New[T](sz...) tsr.SetNames("Row") @@ -189,7 +189,7 @@ func (dt *Table) InsertColumn(idx int, name string, tsr tensor.Tensor) error { // otherwise the cells are n-dimensional tensors of given size. // Supported types include string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. func (dt *Table) AddColumnOfType(name string, typ reflect.Kind, cellSizes ...int) tensor.Tensor { - rows := max(1, dt.Columns.Rows) + rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) tsr := tensor.NewOfType(typ, sz...) tsr.SetNames("Row") diff --git a/tensor/tensor.go b/tensor/tensor.go index 98114fb1cc..8a42693c44 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -224,6 +224,9 @@ type Tensor interface { CopyCellsFrom(from Tensor, to, start, n int) // SetNumRows sets the number of rows (outermost dimension). + // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) + // it is best to first set the anticipated full size, which allocates the + // full amount of memory, and then set to 0 and grow incrementally. SetNumRows(rows int) // Metadata returns the metadata for this tensor, which can be used diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 4450c88093..9fb0987aa2 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -15,6 +15,7 @@ import ( func TestTensorString(t *testing.T) { tsr := New[string](4, 2) tsr.SetNames("Row", "Vals") + assert.Equal(t, []string{"Row", "Vals"}, tsr.Shape().Names) assert.Equal(t, 8, tsr.Len()) assert.Equal(t, true, tsr.IsString()) assert.Equal(t, reflect.String, tsr.DataType()) @@ -73,7 +74,8 @@ func TestTensorString(t *testing.T) { func TestTensorFloat64(t *testing.T) { tsr := New[float64](4, 2) - tsr.SetNames("Row", "Vals") + tsr.SetNames("Row") + assert.Equal(t, []string{"Row", ""}, tsr.Shape().Names) assert.Equal(t, 8, tsr.Len()) assert.Equal(t, false, tsr.IsString()) assert.Equal(t, reflect.Float64, tsr.DataType()) @@ -181,3 +183,32 @@ func TestSortFilter(t *testing.T) { tsr.FilterString("1", Exclude, Equals, UseCase) assert.Equal(t, []int{0, 2, 3, 4}, tsr.Indexes) } + +func TestGrowRow(t *testing.T) { + tsr := NewFloat64(1000) + assert.Equal(t, 1000, cap(tsr.Values)) + assert.Equal(t, 1000, tsr.Len()) + tsr.SetNumRows(0) + assert.Equal(t, 1000, cap(tsr.Values)) + assert.Equal(t, 0, tsr.Len()) + tsr.SetNumRows(1) + assert.Equal(t, 1000, cap(tsr.Values)) + assert.Equal(t, 1, tsr.Len()) + + tsr2 := NewFloat64(1000, 10, 10) + assert.Equal(t, 100000, cap(tsr2.Values)) + assert.Equal(t, 100000, tsr2.Len()) + tsr2.SetNumRows(0) + assert.Equal(t, 100000, cap(tsr2.Values)) + assert.Equal(t, 0, tsr2.Len()) + tsr2.SetNumRows(1) + assert.Equal(t, 100000, cap(tsr2.Values)) + assert.Equal(t, 100, tsr2.Len()) + + bits := NewBits(1000) + assert.Equal(t, 1000, bits.Len()) + bits.SetNumRows(0) + assert.Equal(t, 0, bits.Len()) + bits.SetNumRows(1) + assert.Equal(t, 1, bits.Len()) +} From cefcca9e29208d5ed0dd0cc135ab1c05594a8b14 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 09:37:05 -0700 Subject: [PATCH 052/311] better tensor printing -- always print up to max len, and add separate function where you can control how much is printed. getting different results on cluster and pca tests -- unclear what is going on. --- tensor/base.go | 29 ++++++++++++++++++++----- tensor/bits.go | 2 +- tensor/indexed.go | 2 +- tensor/number.go | 2 +- tensor/stats/cluster/clust_test.go | 3 +++ tensor/stats/cluster/testdata/faces.dat | 26 +++++++++++----------- tensor/stats/metric/matrix.go | 2 ++ tensor/string.go | 2 +- tensor/tensor.go | 4 ++++ 9 files changed, 50 insertions(+), 22 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index c5140a7d74..3ca8eb6b38 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -195,14 +195,27 @@ func (tsr *Base[T]) SymmetricDim() int { return tsr.shape.DimSize(nd - 1) } +// Sprint returns a string representation of the given tensor, +// with a maximum length of as given: output is terminated +// when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. +func Sprint(tsr Tensor, maxLen int) string { + return stringIndexed(tsr, maxLen, nil) +} + +// SprintIndexed returns a string representation of the given Indexed tensor, +// with a maximum length of as given: output is terminated +// when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. +func SprintIndexed(tsr *Indexed, maxLen int) string { + return stringIndexed(tsr.Tensor, maxLen, tsr.Indexes) +} + // stringIndexed is the underlying impl of String that works for indexed // data as well. -func stringIndexed(tsr Tensor, idxs []int) string { - str := tsr.Label() - sz := tsr.Len() - if sz > 1000 { - return str +func stringIndexed(tsr Tensor, maxLen int, idxs []int) string { + if maxLen == 0 { + maxLen = MaxSprintLength } + str := tsr.Label() var b strings.Builder b.WriteString(str) b.WriteString("\n") @@ -211,6 +224,7 @@ func stringIndexed(tsr Tensor, idxs []int) string { if idxs != nil { rows = min(rows, len(idxs)) } + ctr := 0 for r := range rows { rc, _ := Projection2DCoords(tsr.Shape(), oddRow, r, 0) b.WriteString(fmt.Sprintf("%v: ", rc)) @@ -226,6 +240,11 @@ func stringIndexed(tsr Tensor, idxs []int) string { } } b.WriteString("\n") + ctr += cols + if ctr > maxLen { + b.WriteString("...\n") + break + } } return b.String() } diff --git a/tensor/bits.go b/tensor/bits.go index a1f6829540..325815aee8 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -66,7 +66,7 @@ func BoolToInt(bv bool) int { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *Bits) String() string { - return stringIndexed(tsr, nil) + return stringIndexed(tsr, 0, nil) } func (tsr *Bits) IsString() bool { diff --git a/tensor/indexed.go b/tensor/indexed.go index bcb97e716d..e4f8a87393 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -94,7 +94,7 @@ func (ix *Indexed) Index(idx int) int { // String satisfies the fmt.Stringer interface for string of tensor data. func (ix *Indexed) String() string { - return stringIndexed(ix.Tensor, ix.Indexes) + return stringIndexed(ix.Tensor, 0, ix.Indexes) } // Label satisfies the core.Labeler interface for a summary description of the tensor. diff --git a/tensor/number.go b/tensor/number.go index 501b1e9f1f..6fa725009a 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -123,7 +123,7 @@ func NewNumberFromSlice[T num.Number](vals []T) Tensor { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *Number[T]) String() string { - return stringIndexed(tsr, nil) + return stringIndexed(tsr, 0, nil) } func (tsr *Number[T]) IsString() bool { diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 1bc7b63655..6a23e31c4b 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -5,6 +5,7 @@ package cluster import ( + "fmt" "testing" "cogentcore.org/core/base/tolassert" @@ -37,6 +38,8 @@ func TestClust(t *testing.T) { out := tensor.NewFloat64Indexed() metric.Matrix(metric.Euclidean.FuncName(), in, out) + fmt.Println(out.String()) + cl := Cluster(Avg.String(), out, dt.Column("Name")) var dists []float64 diff --git a/tensor/stats/cluster/testdata/faces.dat b/tensor/stats/cluster/testdata/faces.dat index 53912dc13a..ad88d27b6f 100644 --- a/tensor/stats/cluster/testdata/faces.dat +++ b/tensor/stats/cluster/testdata/faces.dat @@ -1,13 +1,13 @@ -_H: $Name %Input[2:0,0]<2:16,16> %Input[2:0,1] %Input[2:0,2] %Input[2:0,3] %Input[2:0,4] %Input[2:0,5] %Input[2:0,6] %Input[2:0,7] %Input[2:0,8] %Input[2:0,9] %Input[2:0,10] %Input[2:0,11] %Input[2:0,12] %Input[2:0,13] %Input[2:0,14] %Input[2:0,15] %Input[2:1,0] %Input[2:1,1] %Input[2:1,2] %Input[2:1,3] %Input[2:1,4] %Input[2:1,5] %Input[2:1,6] %Input[2:1,7] %Input[2:1,8] %Input[2:1,9] %Input[2:1,10] %Input[2:1,11] %Input[2:1,12] %Input[2:1,13] %Input[2:1,14] %Input[2:1,15] %Input[2:2,0] %Input[2:2,1] %Input[2:2,2] %Input[2:2,3] %Input[2:2,4] %Input[2:2,5] %Input[2:2,6] %Input[2:2,7] %Input[2:2,8] %Input[2:2,9] %Input[2:2,10] %Input[2:2,11] %Input[2:2,12] %Input[2:2,13] %Input[2:2,14] %Input[2:2,15] %Input[2:3,0] %Input[2:3,1] %Input[2:3,2] %Input[2:3,3] %Input[2:3,4] %Input[2:3,5] %Input[2:3,6] %Input[2:3,7] %Input[2:3,8] %Input[2:3,9] %Input[2:3,10] %Input[2:3,11] %Input[2:3,12] %Input[2:3,13] %Input[2:3,14] %Input[2:3,15] %Input[2:4,0] %Input[2:4,1] %Input[2:4,2] %Input[2:4,3] %Input[2:4,4] %Input[2:4,5] %Input[2:4,6] %Input[2:4,7] %Input[2:4,8] %Input[2:4,9] %Input[2:4,10] %Input[2:4,11] %Input[2:4,12] %Input[2:4,13] %Input[2:4,14] %Input[2:4,15] %Input[2:5,0] %Input[2:5,1] %Input[2:5,2] %Input[2:5,3] %Input[2:5,4] %Input[2:5,5] %Input[2:5,6] %Input[2:5,7] %Input[2:5,8] %Input[2:5,9] %Input[2:5,10] %Input[2:5,11] %Input[2:5,12] %Input[2:5,13] %Input[2:5,14] %Input[2:5,15] %Input[2:6,0] %Input[2:6,1] %Input[2:6,2] %Input[2:6,3] %Input[2:6,4] %Input[2:6,5] %Input[2:6,6] %Input[2:6,7] %Input[2:6,8] %Input[2:6,9] %Input[2:6,10] %Input[2:6,11] %Input[2:6,12] %Input[2:6,13] %Input[2:6,14] %Input[2:6,15] %Input[2:7,0] %Input[2:7,1] %Input[2:7,2] %Input[2:7,3] %Input[2:7,4] %Input[2:7,5] %Input[2:7,6] %Input[2:7,7] %Input[2:7,8] %Input[2:7,9] %Input[2:7,10] %Input[2:7,11] %Input[2:7,12] %Input[2:7,13] %Input[2:7,14] %Input[2:7,15] %Input[2:8,0] %Input[2:8,1] %Input[2:8,2] %Input[2:8,3] %Input[2:8,4] %Input[2:8,5] %Input[2:8,6] %Input[2:8,7] %Input[2:8,8] %Input[2:8,9] %Input[2:8,10] %Input[2:8,11] %Input[2:8,12] %Input[2:8,13] %Input[2:8,14] %Input[2:8,15] %Input[2:9,0] %Input[2:9,1] %Input[2:9,2] %Input[2:9,3] %Input[2:9,4] %Input[2:9,5] %Input[2:9,6] %Input[2:9,7] %Input[2:9,8] %Input[2:9,9] %Input[2:9,10] %Input[2:9,11] %Input[2:9,12] %Input[2:9,13] %Input[2:9,14] %Input[2:9,15] %Input[2:10,0] %Input[2:10,1] %Input[2:10,2] %Input[2:10,3] %Input[2:10,4] %Input[2:10,5] %Input[2:10,6] %Input[2:10,7] %Input[2:10,8] %Input[2:10,9] %Input[2:10,10] %Input[2:10,11] %Input[2:10,12] %Input[2:10,13] %Input[2:10,14] %Input[2:10,15] %Input[2:11,0] %Input[2:11,1] %Input[2:11,2] %Input[2:11,3] %Input[2:11,4] %Input[2:11,5] %Input[2:11,6] %Input[2:11,7] %Input[2:11,8] %Input[2:11,9] %Input[2:11,10] %Input[2:11,11] %Input[2:11,12] %Input[2:11,13] %Input[2:11,14] %Input[2:11,15] %Input[2:12,0] %Input[2:12,1] %Input[2:12,2] %Input[2:12,3] %Input[2:12,4] %Input[2:12,5] %Input[2:12,6] %Input[2:12,7] %Input[2:12,8] %Input[2:12,9] %Input[2:12,10] %Input[2:12,11] %Input[2:12,12] %Input[2:12,13] %Input[2:12,14] %Input[2:12,15] %Input[2:13,0] %Input[2:13,1] %Input[2:13,2] %Input[2:13,3] %Input[2:13,4] %Input[2:13,5] %Input[2:13,6] %Input[2:13,7] %Input[2:13,8] %Input[2:13,9] %Input[2:13,10] %Input[2:13,11] %Input[2:13,12] %Input[2:13,13] %Input[2:13,14] %Input[2:13,15] %Input[2:14,0] %Input[2:14,1] %Input[2:14,2] %Input[2:14,3] %Input[2:14,4] %Input[2:14,5] %Input[2:14,6] %Input[2:14,7] %Input[2:14,8] %Input[2:14,9] %Input[2:14,10] %Input[2:14,11] %Input[2:14,12] %Input[2:14,13] %Input[2:14,14] %Input[2:14,15] %Input[2:15,0] %Input[2:15,1] %Input[2:15,2] %Input[2:15,3] %Input[2:15,4] %Input[2:15,5] %Input[2:15,6] %Input[2:15,7] %Input[2:15,8] %Input[2:15,9] %Input[2:15,10] %Input[2:15,11] %Input[2:15,12] %Input[2:15,13] %Input[2:15,14] %Input[2:15,15] %Emotion[2:0,0]<2:1,2> %Emotion[2:0,1] %Gender[2:0,0]<2:1,2> %Gender[2:0,1] %Identity[2:0,0]<2:1,10> %Identity[2:0,1] %Identity[2:0,2] %Identity[2:0,3] %Identity[2:0,4] %Identity[2:0,5] %Identity[2:0,6] %Identity[2:0,7] %Identity[2:0,8] %Identity[2:0,9] -_D: Alberto_happy 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 1 0 0 0 0 0 0 0 0 0 -_D: Alberto_sad 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 1 0 0 0 0 0 0 0 0 0 -_D: Betty_happy 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 1 0 1 0 0 1 0 0 1 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 1 0 1 0 1 0 1 0 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 1 0 0 0 0 0 0 0 0 -_D: Betty_sad 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 0 1 1 0 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 -_D: Lisa_happy 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 1 0 1 0 0 1 0 0 1 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 1 1 0 1 0 1 0 1 0 1 1 0 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 0 -_D: Lisa_sad 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 0 1 1 0 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0 0 -_D: Mark_happy 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 1 0 1 0 0 1 0 1 0 1 1 0 0 0 1 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 0 0 -_D: Mark_sad 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 1 0 0 1 0 0 0 1 1 0 0 0 1 0 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 -_D: Wendy_happy 0 1 1 1 0 0 1 1 1 1 0 0 1 1 1 0 0 0 0 1 0 1 0 0 0 0 1 0 1 0 0 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 0 0 1 0 0 1 0 0 1 0 1 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 1 1 0 1 0 1 0 1 0 1 1 0 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 1 0 0 0 0 0 -_D: Wendy_sad 0 1 1 1 0 0 1 1 1 1 0 0 1 1 1 0 0 0 0 1 0 1 0 0 0 0 1 1 1 0 0 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 0 0 1 0 0 0 1 1 0 0 1 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 0 1 0 0 0 1 1 0 0 0 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 -_D: Zane_happy 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 -_D: Zane_sad 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 0 1 0 0 0 0 0 1 0 0 0 0 +$Name %Input[2:0,0]<2:16,16> %Input[2:0,1] %Input[2:0,2] %Input[2:0,3] %Input[2:0,4] %Input[2:0,5] %Input[2:0,6] %Input[2:0,7] %Input[2:0,8] %Input[2:0,9] %Input[2:0,10] %Input[2:0,11] %Input[2:0,12] %Input[2:0,13] %Input[2:0,14] %Input[2:0,15] %Input[2:1,0] %Input[2:1,1] %Input[2:1,2] %Input[2:1,3] %Input[2:1,4] %Input[2:1,5] %Input[2:1,6] %Input[2:1,7] %Input[2:1,8] %Input[2:1,9] %Input[2:1,10] %Input[2:1,11] %Input[2:1,12] %Input[2:1,13] %Input[2:1,14] %Input[2:1,15] %Input[2:2,0] %Input[2:2,1] %Input[2:2,2] %Input[2:2,3] %Input[2:2,4] %Input[2:2,5] %Input[2:2,6] %Input[2:2,7] %Input[2:2,8] %Input[2:2,9] %Input[2:2,10] %Input[2:2,11] %Input[2:2,12] %Input[2:2,13] %Input[2:2,14] %Input[2:2,15] %Input[2:3,0] %Input[2:3,1] %Input[2:3,2] %Input[2:3,3] %Input[2:3,4] %Input[2:3,5] %Input[2:3,6] %Input[2:3,7] %Input[2:3,8] %Input[2:3,9] %Input[2:3,10] %Input[2:3,11] %Input[2:3,12] %Input[2:3,13] %Input[2:3,14] %Input[2:3,15] %Input[2:4,0] %Input[2:4,1] %Input[2:4,2] %Input[2:4,3] %Input[2:4,4] %Input[2:4,5] %Input[2:4,6] %Input[2:4,7] %Input[2:4,8] %Input[2:4,9] %Input[2:4,10] %Input[2:4,11] %Input[2:4,12] %Input[2:4,13] %Input[2:4,14] %Input[2:4,15] %Input[2:5,0] %Input[2:5,1] %Input[2:5,2] %Input[2:5,3] %Input[2:5,4] %Input[2:5,5] %Input[2:5,6] %Input[2:5,7] %Input[2:5,8] %Input[2:5,9] %Input[2:5,10] %Input[2:5,11] %Input[2:5,12] %Input[2:5,13] %Input[2:5,14] %Input[2:5,15] %Input[2:6,0] %Input[2:6,1] %Input[2:6,2] %Input[2:6,3] %Input[2:6,4] %Input[2:6,5] %Input[2:6,6] %Input[2:6,7] %Input[2:6,8] %Input[2:6,9] %Input[2:6,10] %Input[2:6,11] %Input[2:6,12] %Input[2:6,13] %Input[2:6,14] %Input[2:6,15] %Input[2:7,0] %Input[2:7,1] %Input[2:7,2] %Input[2:7,3] %Input[2:7,4] %Input[2:7,5] %Input[2:7,6] %Input[2:7,7] %Input[2:7,8] %Input[2:7,9] %Input[2:7,10] %Input[2:7,11] %Input[2:7,12] %Input[2:7,13] %Input[2:7,14] %Input[2:7,15] %Input[2:8,0] %Input[2:8,1] %Input[2:8,2] %Input[2:8,3] %Input[2:8,4] %Input[2:8,5] %Input[2:8,6] %Input[2:8,7] %Input[2:8,8] %Input[2:8,9] %Input[2:8,10] %Input[2:8,11] %Input[2:8,12] %Input[2:8,13] %Input[2:8,14] %Input[2:8,15] %Input[2:9,0] %Input[2:9,1] %Input[2:9,2] %Input[2:9,3] %Input[2:9,4] %Input[2:9,5] %Input[2:9,6] %Input[2:9,7] %Input[2:9,8] %Input[2:9,9] %Input[2:9,10] %Input[2:9,11] %Input[2:9,12] %Input[2:9,13] %Input[2:9,14] %Input[2:9,15] %Input[2:10,0] %Input[2:10,1] %Input[2:10,2] %Input[2:10,3] %Input[2:10,4] %Input[2:10,5] %Input[2:10,6] %Input[2:10,7] %Input[2:10,8] %Input[2:10,9] %Input[2:10,10] %Input[2:10,11] %Input[2:10,12] %Input[2:10,13] %Input[2:10,14] %Input[2:10,15] %Input[2:11,0] %Input[2:11,1] %Input[2:11,2] %Input[2:11,3] %Input[2:11,4] %Input[2:11,5] %Input[2:11,6] %Input[2:11,7] %Input[2:11,8] %Input[2:11,9] %Input[2:11,10] %Input[2:11,11] %Input[2:11,12] %Input[2:11,13] %Input[2:11,14] %Input[2:11,15] %Input[2:12,0] %Input[2:12,1] %Input[2:12,2] %Input[2:12,3] %Input[2:12,4] %Input[2:12,5] %Input[2:12,6] %Input[2:12,7] %Input[2:12,8] %Input[2:12,9] %Input[2:12,10] %Input[2:12,11] %Input[2:12,12] %Input[2:12,13] %Input[2:12,14] %Input[2:12,15] %Input[2:13,0] %Input[2:13,1] %Input[2:13,2] %Input[2:13,3] %Input[2:13,4] %Input[2:13,5] %Input[2:13,6] %Input[2:13,7] %Input[2:13,8] %Input[2:13,9] %Input[2:13,10] %Input[2:13,11] %Input[2:13,12] %Input[2:13,13] %Input[2:13,14] %Input[2:13,15] %Input[2:14,0] %Input[2:14,1] %Input[2:14,2] %Input[2:14,3] %Input[2:14,4] %Input[2:14,5] %Input[2:14,6] %Input[2:14,7] %Input[2:14,8] %Input[2:14,9] %Input[2:14,10] %Input[2:14,11] %Input[2:14,12] %Input[2:14,13] %Input[2:14,14] %Input[2:14,15] %Input[2:15,0] %Input[2:15,1] %Input[2:15,2] %Input[2:15,3] %Input[2:15,4] %Input[2:15,5] %Input[2:15,6] %Input[2:15,7] %Input[2:15,8] %Input[2:15,9] %Input[2:15,10] %Input[2:15,11] %Input[2:15,12] %Input[2:15,13] %Input[2:15,14] %Input[2:15,15] %Emotion[2:0,0]<2:1,2> %Emotion[2:0,1] %Gender[2:0,0]<2:1,2> %Gender[2:0,1] %Identity[2:0,0]<2:1,10> %Identity[2:0,1] %Identity[2:0,2] %Identity[2:0,3] %Identity[2:0,4] %Identity[2:0,5] %Identity[2:0,6] %Identity[2:0,7] %Identity[2:0,8] %Identity[2:0,9] +Alberto_happy 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 1 0 0 0 0 0 0 0 0 0 +Alberto_sad 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 1 0 0 0 0 0 0 0 0 0 +Betty_happy 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 1 0 1 0 0 1 0 0 1 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 1 0 1 0 1 0 1 0 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 1 0 0 0 0 0 0 0 0 +Betty_sad 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 0 1 1 0 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0 0 +Lisa_happy 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 1 0 1 0 0 1 0 0 1 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 1 1 0 1 0 1 0 1 0 1 1 0 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 0 +Lisa_sad 1 0 1 0 0 0 1 1 1 1 0 0 0 1 0 1 0 1 0 1 0 1 0 0 0 0 1 0 1 0 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 1 0 1 0 0 0 1 1 0 0 1 0 0 1 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0 0 +Mark_happy 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 1 0 1 0 0 1 0 1 0 1 1 0 0 0 1 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 0 0 +Mark_sad 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 1 0 0 1 0 0 0 1 1 0 0 0 1 0 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 1 0 1 0 0 0 1 0 0 0 0 0 0 +Wendy_happy 0 1 1 1 0 0 1 1 1 1 0 0 1 1 1 0 0 0 0 1 0 1 0 0 0 0 1 0 1 0 0 0 0 0 1 0 1 0 0 1 1 0 0 1 0 1 0 0 0 0 0 1 0 0 1 0 0 1 0 1 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 1 1 0 1 0 1 0 1 0 1 1 0 0 0 0 0 1 0 1 0 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 1 0 0 0 0 0 +Wendy_sad 0 1 1 1 0 0 1 1 1 1 0 0 1 1 1 0 0 0 0 1 0 1 0 0 0 0 1 1 1 0 0 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 0 0 1 0 0 0 1 1 0 0 1 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 0 1 0 0 0 1 1 0 0 0 0 0 1 1 1 0 0 0 1 1 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 1 0 0 0 0 0 +Zane_happy 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 1 0 1 0 0 1 0 1 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 1 0 0 1 0 0 0 0 0 1 0 0 0 0 +Zane_sad 0 0 0 0 0 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 1 0 0 1 0 0 1 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 1 1 0 0 0 0 1 1 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 1 1 0 0 0 0 1 0 0 0 0 0 1 1 0 0 0 1 0 0 0 1 0 0 1 0 0 0 1 0 0 0 0 1 0 1 1 0 0 0 0 1 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 0 1 0 0 0 0 0 1 0 0 0 0 diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index d88f946d19..0ba63e37ee 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -5,6 +5,7 @@ package metric import ( + "fmt" "log/slog" "cogentcore.org/core/math32/vecint" @@ -30,6 +31,7 @@ func init() { // to the upper triangular region, for maximum efficiency. func Matrix(funcName string, in, out *tensor.Indexed) { rows, cells := in.RowCellSize() + fmt.Println(rows, cells) if rows == 0 || cells == 0 { return } diff --git a/tensor/string.go b/tensor/string.go index 24b8c2b62e..74ac0c1193 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -64,7 +64,7 @@ func Float64ToString(val float64) string { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *String) String() string { - return stringIndexed(tsr, nil) + return stringIndexed(tsr, 0, nil) } func (tsr *String) IsString() bool { diff --git a/tensor/tensor.go b/tensor/tensor.go index 8a42693c44..a5ab3bb999 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -22,6 +22,10 @@ type DataTypes interface { string | bool | float32 | float64 | int | int32 | byte } +// MaxSprintLength is the default maximum length of a String() representation +// of a tensor, as generated by the Sprint function. Defaults to 1000. +var MaxSprintLength = 1000 + // todo: add a conversion function to copy data from Column-Major to a tensor: // It is also possible to use Column-Major order, which is used in R, Julia, and MATLAB // where the inner-most index is first and outermost last. From e6a9b2df19dcdba17d3bb36494cc57a8d61bf15e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 09:59:25 -0700 Subject: [PATCH 053/311] remove print --- tensor/stats/cluster/clust_test.go | 3 --- tensor/stats/metric/matrix.go | 2 -- 2 files changed, 5 deletions(-) diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 6a23e31c4b..1bc7b63655 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -5,7 +5,6 @@ package cluster import ( - "fmt" "testing" "cogentcore.org/core/base/tolassert" @@ -38,8 +37,6 @@ func TestClust(t *testing.T) { out := tensor.NewFloat64Indexed() metric.Matrix(metric.Euclidean.FuncName(), in, out) - fmt.Println(out.String()) - cl := Cluster(Avg.String(), out, dt.Column("Name")) var dists []float64 diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 0ba63e37ee..d88f946d19 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -5,7 +5,6 @@ package metric import ( - "fmt" "log/slog" "cogentcore.org/core/math32/vecint" @@ -31,7 +30,6 @@ func init() { // to the upper triangular region, for maximum efficiency. func Matrix(funcName string, in, out *tensor.Indexed) { rows, cells := in.RowCellSize() - fmt.Println(rows, cells) if rows == 0 || cells == 0 { return } From 30ea9f84ec5e6d62ce30e012beb7b012dffbbe5d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 10:36:04 -0700 Subject: [PATCH 054/311] cells test --- tensor/tensor_test.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 4450c88093..eb5ce22426 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -133,11 +133,37 @@ func TestSlice(t *testing.T) { } } } + + res := `Tensor: [20] +[0]: 200 +[1]: 201 +[2]: 202 +[3]: 203 +[4]: 204 +[5]: 210 +[6]: 211 +[7]: 212 +[8]: 213 +[9]: 214 +[10]: 220 +[11]: 221 +[12]: 222 +[13]: 223 +[14]: 224 +[15]: 230 +[16]: 231 +[17]: 232 +[18]: 233 +[19]: 234 +` + // fmt.Println(ft.Cells1D(1)) + assert.Equal(t, res, ft.Cells1D(1).String()) + // fmt.Println(ft.String()) sf := NewFloat64Indexed() Slice(ft, sf, Range{}, Range{Start: 1, End: 2}) // fmt.Println(sf.String()) - res := `Tensor: [3, 1, 5] + res = `Tensor: [3, 1, 5] [0 0]: 110 111 112 113 114 [1 0]: 210 211 212 213 214 [2 0]: 310 311 312 313 314 From a7e70b3805e20035ae653fbd9bbb638a120b009d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 11:26:16 -0700 Subject: [PATCH 055/311] fixed bug with rows = 0 -- strides needs to be robust to this case. --- tensor/shape.go | 5 ++++- tensor/table/table_test.go | 24 ++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/tensor/shape.go b/tensor/shape.go index 8e8c49efa2..e1f094107c 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -205,6 +205,10 @@ func (sh *Shape) String() string { // RowMajorStrides returns strides for sizes where the first dimension is outermost // and subsequent dimensions are progressively inner. func RowMajorStrides(sizes ...int) []int { + if len(sizes) == 0 { + return nil + } + sizes[0] = max(1, sizes[0]) // critical for strides to not be nil due to rows = 0 rem := int(1) for _, v := range sizes { rem *= v @@ -212,7 +216,6 @@ func RowMajorStrides(sizes ...int) []int { if rem == 0 { strides := make([]int, len(sizes)) - rem := int(1) for i := range strides { strides[i] = rem } diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index ff7e8d3d2b..befd574acf 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -21,8 +21,8 @@ func TestAdd3DCol(t *testing.T) { t.Errorf("Add4DCol: # of dims != 4\n") } - if col.Shape().DimSize(0) != 1 { - t.Errorf("Add4DCol: dim 0 len != 1, was: %v\n", col.Shape().DimSize(0)) + if col.Shape().DimSize(0) != 0 { + t.Errorf("Add4DCol: dim 0 len != 0, was: %v\n", col.Shape().DimSize(0)) } if col.Shape().DimSize(1) != 11 { @@ -112,3 +112,23 @@ func TestInsertDeleteRows(t *testing.T) { dt.DeleteRows(1, 2) assert.Equal(t, []int{0, 1, 2}, dt.Indexes) } + +func TestCells(t *testing.T) { + dt := NewTable() + err := dt.OpenCSV("../stats/cluster/testdata/faces.dat", tensor.Tab) + assert.NoError(t, err) + in := dt.Column("Input") + for i := range 10 { + vals := make([]float32, 16) + for j := range 16 { + vals[j] = float32(in.FloatRowCell(i, j)) + } + // fmt.Println(s) + ss := in.Tensor.SubSpace(i).(*tensor.Float32) + // fmt.Println(ss.Values[:16]) + cl := in.Cells1D(i).Tensor.(*tensor.Float32) + // fmt.Println(cl.Values[:16]) + assert.Equal(t, vals, ss.Values[:16]) + assert.Equal(t, vals, cl.Values[:16]) + } +} From 4ce9a234df0933e694ebf87e334dccbba3c2a4f3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 13:02:04 -0700 Subject: [PATCH 056/311] fix filetree time ordering and add docs on dynamic row sizing --- filetree/node.go | 2 +- tensor/README.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/filetree/node.go b/filetree/node.go index 433ae767bf..0d8d2a4a53 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -337,7 +337,7 @@ func (fn *Node) dirFileList() []fs.FileInfo { func sortByModTime(files []fs.FileInfo) { slices.SortFunc(files, func(a, b fs.FileInfo) int { - return a.ModTime().Compare(b.ModTime()) + return -a.ModTime().Compare(b.ModTime()) }) } diff --git a/tensor/README.md b/tensor/README.md index a526c0a0f9..5748682800 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -48,6 +48,10 @@ In general, **1D** refers to a flat, 1-dimensional list. There are various stan * **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations. +## Dynamic row sizing (e.g., for logs) + +The `SetNumRows` method can be used to progressively increase the number of rows to fit more data, as is typically the case when logging data (often using a [table](table)). You can set the row dimension to 0 to start -- that is (now) safe. However, for greatest efficiency, it is best to set the number of rows to the largest expected size first, and _then_ set it back to 0. The underlying slice of data retains its capacity when sized back down. During incremental increasing of the slice size, if it runs out of capacity, all the elements need to be copied, so it is more efficient to establish the capacity up front instead of having multiple incremental re-allocations. + # Cheat Sheet `ix` is the `Indexed` tensor for these examples: From 2d11d961c478315c8793cf953085ff3374e390bc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 16:01:46 -0700 Subject: [PATCH 057/311] start on goal: working on docs for expressions so we know what we need to do --- goal/README.md | 647 ++++++++++++++++++ {shell => goal}/builtins.go | 0 {shell/cmd/cosh => goal/cmd/goal}/cfg.cosh | 0 .../cmd/cosh/cosh.go => goal/cmd/goal/goal.go | 0 .../cosh/test.cosh => goal/cmd/goal/test.goal | 0 .../cmd/cosh => goal/cmd/goal}/testdata/make | 4 +- {shell/cmd/cosh => goal/cmd/goal}/typegen.go | 0 {shell => goal}/complete.go | 0 {shell => goal}/exec.go | 0 {shell => goal}/exec_test.go | 0 {shell => goal}/execwords.go | 0 .../cosh/coshlib.go => goal/goalib/goalib.go | 6 +- .../cogentcore_org-core-base-datasize.go | 0 .../cogentcore_org-core-base-elide.go | 0 .../cogentcore_org-core-base-errors.go | 0 .../cogentcore_org-core-base-fsx.go | 0 .../cogentcore_org-core-base-strcase.go | 0 .../cogentcore_org-core-base-stringsx.go | 0 .../cogentcore_org-core-shell-cosh.go | 0 {shell => goal}/interpreter/imports.go | 0 {shell => goal}/interpreter/imports_test.go | 0 {shell => goal}/interpreter/interpreter.go | 0 {shell => goal}/interpreter/make | 0 {shell => goal}/paths.go | 0 {shell => goal}/run.go | 0 {shell => goal}/shell.go | 0 {shell => goal}/token.go | 0 {shell => goal}/transpile.go | 0 {shell => goal}/transpile_test.go | 0 shell/README.md | 190 ----- tensor/README.md | 6 +- tensor/funcs.go | 2 +- 32 files changed, 656 insertions(+), 199 deletions(-) create mode 100644 goal/README.md rename {shell => goal}/builtins.go (100%) rename {shell/cmd/cosh => goal/cmd/goal}/cfg.cosh (100%) rename shell/cmd/cosh/cosh.go => goal/cmd/goal/goal.go (100%) rename shell/cmd/cosh/test.cosh => goal/cmd/goal/test.goal (100%) rename {shell/cmd/cosh => goal/cmd/goal}/testdata/make (78%) rename {shell/cmd/cosh => goal/cmd/goal}/typegen.go (100%) rename {shell => goal}/complete.go (100%) rename {shell => goal}/exec.go (100%) rename {shell => goal}/exec_test.go (100%) rename {shell => goal}/execwords.go (100%) rename shell/cosh/coshlib.go => goal/goalib/goalib.go (94%) rename {shell => goal}/interpreter/cogentcore_org-core-base-datasize.go (100%) rename {shell => goal}/interpreter/cogentcore_org-core-base-elide.go (100%) rename {shell => goal}/interpreter/cogentcore_org-core-base-errors.go (100%) rename {shell => goal}/interpreter/cogentcore_org-core-base-fsx.go (100%) rename {shell => goal}/interpreter/cogentcore_org-core-base-strcase.go (100%) rename {shell => goal}/interpreter/cogentcore_org-core-base-stringsx.go (100%) rename {shell => goal}/interpreter/cogentcore_org-core-shell-cosh.go (100%) rename {shell => goal}/interpreter/imports.go (100%) rename {shell => goal}/interpreter/imports_test.go (100%) rename {shell => goal}/interpreter/interpreter.go (100%) rename {shell => goal}/interpreter/make (100%) rename {shell => goal}/paths.go (100%) rename {shell => goal}/run.go (100%) rename {shell => goal}/shell.go (100%) rename {shell => goal}/token.go (100%) rename {shell => goal}/transpile.go (100%) rename {shell => goal}/transpile_test.go (100%) delete mode 100644 shell/README.md diff --git a/goal/README.md b/goal/README.md new file mode 100644 index 0000000000..016cc71075 --- /dev/null +++ b/goal/README.md @@ -0,0 +1,647 @@ +# Goal: Go augmented language + +_Goal_ is an augmented version of the _Go_ language, that transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: + +* Shell scripting, where you want to be able to directly call other executable programs with arguments, without having to navigate all the complexity of the standard [os.exec](https://pkg.go.dev/os/exec) package. + +* Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and iterators etc. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. + +The main goal of _Goal_ is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where it is particularly important. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, _Goal_ retains in its _Go_ foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. + +For the shell scripting aspect of _Goal_, the simple idea is that each line of code is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below). If a line starts off with something like `ls -la...` then it is clear that it is not valid Go code, and it is therefore processed as a shell command. + +You can intermix Go within a shell line by wrapping an expression with `{ }` braces, and a Go expression can contain shell code by using backticks (\`) (raw quoted strings use double-backticks \`\`). Here's an example: +```go +for i, f := range goal.SplitLines(`ls -la`) { // `ls` executes, returns string + echo {i} {strings.ToLower(f)} // {} surrounds Go within shell +} +``` +where `goal.SplitLines` is a function that runs `strings.Split(arg, "\n")`, defined in the `goal` standard library of such frequently-used helper functions. + +For the mathematical expressions, we use `$` symbols (as in markdown) to demarcate such expressions: +```go +for _, x := range $[1,2,3]$ { + fmt.Println($x^2$) +} +``` + +In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. + +We henceforth refer to shell code as `exec` code (in reference to the Go & Cogent `exec` package that we use to execute programs), given the potential ambituity of the entire `cosh` language being the shell. There are different syntactic formatting rules for these two domains of Go and Exec, within cosh: + +* Go code is processed and formatted as usual (e.g., white space is irrelevant, etc). +* Exec code is space separated, like normal command-line invocations. + +You can easily perform handy duration and data size formatting: + +```go +22010706 * time.Nanosecond // 22.010706ms +datasize.Size(44610930) // 42.5 MB +``` + +# Special syntax + +## Multiple statements per line + +* Multiple statements can be combined on one line, separated by `;` as in regular Go and shell languages. Critically, the language determination for the first statement determines the language for the remaining statements; you cannot intermix the two on one line, when using `;` +# Exec mode + +## Environment variables + +* `set ` (space delimited as in all exec mode, no equals) + +## Output redirction + +* Standard output redirect: `>` and `>&` (and `|`, `|&` if needed) + +## Control flow + +* Any error stops the script execution, except for statements wrapped in `[ ]`, indicating an "optional" statement, e.g.: + +```sh +cd some; [mkdir sub]; cd sub +``` + +* `&` at the end of a statement runs in the background (as in bash) -- otherwise it waits until it completes before it continues. + +* `jobs`, `fg`, `bg`, and `kill` builtin commands function as in usual bash. + +## Exec functions (aliases) + +Use the `command` keyword to define new functions for Exec mode execution, which can then be used like any other command, for example: + +```sh +command list { + ls -la args... +} +``` + +```sh +cd data +list *.tsv +``` + +The `command` is transpiled into a Go function that takes `args ...string`. In the command function body, you can use the `args...` expression to pass all of the args, or `args[1]` etc to refer to specific positional indexes, as usual. + +The command function name is registered so that the standard shell execution code can run the function, passing the args. You can also call it directly from Go code using the standard parentheses expression. + +# Script Files and Makefile-like functionality + +As with most scripting languages, a file of cosh code can be made directly executable by appending a "shebang" expression at the start of the file: + +```sh +#!/usr/bin/env cosh +``` + +When executed this way, any additional args are available via an `args []any` variable, which can be passed to a command as follows: +```go +install {args...} +``` +or by referring to specific arg indexes etc. + +To make a script behave like a standard Makefile, you can define different `command`s for each of the make commands, and then add the following at the end of the file to use the args to run commands: + +```go +shell.RunCommands(args) +``` + +See [make](cmd/cosh/testdata/make) for an example, in `cmd/cosh/testdata/make`, which can be run for example using: + +```sh +./make build +``` + +Note that there is nothing special about the name `make` here, so this can be done with any file. + +The `make` package defines a number of useful utility functions that accomplish the standard dependency and file timestamp checking functionality from the standard `make` command, as in the [magefile](https://magefile.org/dependencies/) system. Note that the cosh direct exec command syntax makes the resulting make files much closer to a standard bash-like Makefile, while still having all the benefits of Go control and expressions, compared to magefile. + +TODO: implement and document above. + +# SSH connections to remote hosts + +Any number of active SSH connections can be maintained and used dynamically within a script, including simple ways of copying data among the different hosts (including the local host). The Go mode execution is always on the local host in one running process, and only the shell commands are executed remotely, enabling a unique ability to easily coordinate and distribute processing and data across various hosts. + +Each host maintains its own working directory and environment variables, which can be configured and re-used by default whenever using a given host. + +* `cossh hostname.org [name]` establishes a connection, using given optional name to refer to this connection. If the name is not provided, a sequential number will be used, starting with 1, with 0 referring always to the local host. + +* `@name` then refers to the given host in all subsequent commands, with `@0` referring to the local host where the cosh script is running. + +### Explicit per-command specification of host + +```sh +@name cd subdir; ls +``` + +### Default host + +```sh +@name // or: +cossh @name +``` + +uses the given host for all subsequent commands (unless explicitly specified), until the default is changed. Use `cossh @0` to return to localhost. + +### Redirect input / output among hosts + +The output of a remote host command can be sent to a file on the local host: +```sh +@name cat hostfile.tsv > @0:localfile.tsv +``` +Note the use of the `:` colon delimiter after the host name here. TODO: You cannot send output to a remote host file (e.g., `> @host:remotefile.tsv`) -- maybe with sftp? + +The output of any command can also be piped to a remote host as its standard input: +```sh +ls *.tsv | @host cat > files.txt +``` + +### scp to copy files easily + +The builtin `scp` function allows easy copying of files across hosts, using the persistent connections established with `cossh` instead of creating new connections as in the standard scp command. + +`scp` is _always_ run from the local host, with the remote host filename specified as `@name:remotefile` + +```sh +scp @name:hostfile.tsv localfile.tsv +``` + +TODO: Importantly, file wildcard globbing works as expected: +```sh +scp @name:*.tsv @0:data/ +``` + +and entire directories can be copied, as in `cp -a` or `cp -r` (this behavior is automatic and does not require a flag). + +### Close connections + +```sh +cossh close +``` + +Will close all active connections and return the default host to @0. All active connections are also automatically closed when the shell terminates. + +# Other Utilties + +** need a replacement for findnm -- very powerful but garbage.. + +# Rules for Go vs. Shell determination + +The critical extension from standard Go syntax is for lines that are processed by the `Exec` functions, used for running arbitrary programs on the user's executable path. Here are the rules (word = IDENT token): + +* Backticks "``" anywhere: Exec. Returns a `string`. +* Within Exec, `{}`: Go +* Line starts with `Go` Keyword: Go +* Line is one word: Exec +* Line starts with `path`: Exec +* Line starts with `"string"`: Exec +* Line starts with `word word`: Exec +* Line starts with `word {`: Exec +* Otherwise: Go + +# TODO: + +* likewise, need to run everything effectively as a bg job with our own explicit Wait, which we can then communicate with to move from fg to bg. + +# Math Expression Details + +In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. The `np.` prefix on numpy global functions is optional, and corresponding field-like properties of tensors turn into the appropriate methods during the transpiling process. + +All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. + +Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) + +| Goal | Python | MATLAB | Notes | +----------------------------------- +| same: | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of array `a` | +| `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of array `a` | +| same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` array | +| same: | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of array a | +| `tensor([[1., 2., 3.], [4., 5., 6.]])` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D array | +| same: | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks a, b, c, and d | +| same: | `a[-1]` | `a(end)` | access last element | +| same: | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D array a | +| same: | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D array a; unspecified dimensions are equivalent to `:` | +| same: | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | +| same: | `a[-5:]` | `a(end-4:end,:)` | first 5 rows of 2D array a | +| same: | `a[0:3, 4:9]` | `a(1:3,5:9)` | last 5 rows of 2D array a | +| same: | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | The first through third rows and fifth through ninth columns of a 2D array, a. | + + +rows 2,4 and 5 and columns 1 and 3. This allows the matrix to be modified, and doesn’t require a regular slice. + +a(3:2:21,:) + +a[2:21:2,:] + +every other row of a, starting with the third and going to the twenty-first + +a(1:2:end,:) + +a[::2, :] + +every other row of a, starting with the first + +a(end:-1:1,:) or flipud(a) + +a[::-1,:] + +a with rows in reverse order + +a([1:end 1],:) + +a[np.r_[:len(a),0]] + +a with copy of the first row appended to the end + +a.' + +a.transpose() or a.T + +transpose of a + +a' + +a.conj().transpose() or a.conj().T + +conjugate transpose of a + +a * b + +a @ b + +matrix multiply + +a .* b + +a * b + +element-wise multiply + +a./b + +a/b + +element-wise divide + +a.^3 + +a**3 + +element-wise exponentiation + +(a > 0.5) + +(a > 0.5) + +matrix whose i,jth element is (a_ij > 0.5). The MATLAB result is an array of logical values 0 and 1. The NumPy result is an array of the boolean values False and True. + +find(a > 0.5) + +np.nonzero(a > 0.5) + +find the indices where (a > 0.5) + +a(:,find(v > 0.5)) + +a[:,np.nonzero(v > 0.5)[0]] + +extract the columns of a where vector v > 0.5 + +a(:,find(v>0.5)) + +a[:, v.T > 0.5] + +extract the columns of a where column vector v > 0.5 + +a(a<0.5)=0 + +a[a < 0.5]=0 + +a with elements less than 0.5 zeroed out + +a .* (a>0.5) + +a * (a > 0.5) + +a with elements less than 0.5 zeroed out + +a(:) = 3 + +a[:] = 3 + +set all values to the same scalar value + +y=x + +y = x.copy() + +NumPy assigns by reference + +y=x(2,:) + +y = x[1, :].copy() + +NumPy slices are by reference + +y=x(:) + +y = x.flatten() + +turn array into vector (note that this forces a copy). To obtain the same data ordering as in MATLAB, use x.flatten('F'). + +1:10 + +np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j] + +create an increasing vector (see note RANGES) + +0:9 + +np.arange(10.) or np.r_[:10.] or np.r_[:9:10j] + +create an increasing vector (see note RANGES) + +[1:10]' + +np.arange(1.,11.)[:, np.newaxis] + +create a column vector + +zeros(3,4) + +np.zeros((3, 4)) + +3x4 two-dimensional array full of 64-bit floating point zeros + +zeros(3,4,5) + +np.zeros((3, 4, 5)) + +3x4x5 three-dimensional array full of 64-bit floating point zeros + +ones(3,4) + +np.ones((3, 4)) + +3x4 two-dimensional array full of 64-bit floating point ones + +eye(3) + +np.eye(3) + +3x3 identity matrix + +diag(a) + +np.diag(a) + +returns a vector of the diagonal elements of 2D array, a + +diag(v,0) + +np.diag(v, 0) + +returns a square diagonal matrix whose nonzero values are the elements of vector, v + +rng(42,'twister') +rand(3,4) +from numpy.random import default_rng +rng = default_rng(42) +rng.random(3, 4) +or older version: random.rand((3, 4)) + +generate a random 3x4 array with default random number generator and seed = 42 + +linspace(1,3,4) + +np.linspace(1,3,4) + +4 equally spaced samples between 1 and 3, inclusive + +[x,y]=meshgrid(0:8,0:5) + +np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.]) + +two 2D arrays: one of x values, the other of y values + +ogrid[0:9.,0:6.] or np.ix_(np.r_[0:9.],np.r_[0:6.] + +the best way to eval functions on a grid + +[x,y]=meshgrid([1,2,4],[2,4,5]) + +np.meshgrid([1,2,4],[2,4,5]) + +np.ix_([1,2,4],[2,4,5]) + +the best way to eval functions on a grid + +repmat(a, m, n) + +np.tile(a, (m, n)) + +create m by n copies of a + +[a b] + +np.concatenate((a,b),1) or np.hstack((a,b)) or np.column_stack((a,b)) or np.c_[a,b] + +concatenate columns of a and b + +[a; b] + +np.concatenate((a,b)) or np.vstack((a,b)) or np.r_[a,b] + +concatenate rows of a and b + +max(max(a)) + +a.max() or np.nanmax(a) + +maximum element of a (with ndims(a)<=2 for MATLAB, if there are NaN’s, nanmax will ignore these and return largest value) + +max(a) + +a.max(0) + +maximum element of each column of array a + +max(a,[],2) + +a.max(1) + +maximum element of each row of array a + +max(a,b) + +np.maximum(a, b) + +compares a and b element-wise, and returns the maximum value from each pair + +norm(v) + +np.sqrt(v @ v) or np.linalg.norm(v) + +L2 norm of vector v + +a & b + +logical_and(a,b) + +element-by-element AND operator (NumPy ufunc) See note LOGICOPS + +a | b + +np.logical_or(a,b) + +element-by-element OR operator (NumPy ufunc) See note LOGICOPS + +bitand(a,b) + +a & b + +bitwise AND operator (Python native and NumPy ufunc) + +bitor(a,b) + +a | b + +bitwise OR operator (Python native and NumPy ufunc) + +inv(a) + +linalg.inv(a) + +inverse of square 2D array a + +pinv(a) + +linalg.pinv(a) + +pseudo-inverse of 2D array a + +rank(a) + +np.linalg.matrix_rank(a) + +matrix rank of a 2D array a + +a\b + +linalg.solve(a, b) if a is square; linalg.lstsq(a, b) otherwise + +solution of a x = b for x + +b/a + +Solve a.T x.T = b.T instead + +solution of x a = b for x + +[U,S,V]=svd(a) + +U, S, Vh = linalg.svd(a); V = Vh.T + +singular value decomposition of a + +chol(a) + +linalg.cholesky(a) + +Cholesky factorization of a 2D array + +[V,D]=eig(a) + +D,V = linalg.eig(a) + +eigenvalues + and eigenvectors + of a, where + +[V,D]=eig(a,b) + +D,V = linalg.eig(a, b) + +eigenvalues + and eigenvectors + of a, b where + +[V,D]=eigs(a,3) + +D,V = eigs(a, k=3) + +find the k=3 largest eigenvalues and eigenvectors of 2D array, a + +[Q,R]=qr(a,0) + +Q,R = linalg.qr(a) + +QR decomposition + +[L,U,P]=lu(a) where a==P'*L*U + +P,L,U = linalg.lu(a) where a == P@L@U + +LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) + +conjgrad + +cg + +conjugate gradients solver + +fft(a) + +np.fft.fft(a) + +Fourier transform of a + +ifft(a) + +np.fft.ifft(a) + +inverse Fourier transform of a + +sort(a) + +np.sort(a) or a.sort(axis=0) + +sort each column of a 2D array, a + +sort(a, 2) + +np.sort(a, axis=1) or a.sort(axis=1) + +sort the each row of 2D array, a + +[b,I]=sortrows(a,1) + +I = np.argsort(a[:, 0]); b = a[I,:] + +save the array a as array b with rows sorted by the first column + +x = Z\y + +x = linalg.lstsq(Z, y) + +perform a linear regression of the form + +decimate(x, q) + +signal.resample(x, np.ceil(len(x)/q)) + +downsample with low-pass filtering + +unique(a) + +np.unique(a) + +a vector of unique values in array a + +squeeze(a) + +a.squeeze() + +remove singleton dimensions of array a. Note that MATLAB will always return arrays of 2D or higher while NumPy will return arrays of 0D or higher + + diff --git a/shell/builtins.go b/goal/builtins.go similarity index 100% rename from shell/builtins.go rename to goal/builtins.go diff --git a/shell/cmd/cosh/cfg.cosh b/goal/cmd/goal/cfg.cosh similarity index 100% rename from shell/cmd/cosh/cfg.cosh rename to goal/cmd/goal/cfg.cosh diff --git a/shell/cmd/cosh/cosh.go b/goal/cmd/goal/goal.go similarity index 100% rename from shell/cmd/cosh/cosh.go rename to goal/cmd/goal/goal.go diff --git a/shell/cmd/cosh/test.cosh b/goal/cmd/goal/test.goal similarity index 100% rename from shell/cmd/cosh/test.cosh rename to goal/cmd/goal/test.goal diff --git a/shell/cmd/cosh/testdata/make b/goal/cmd/goal/testdata/make similarity index 78% rename from shell/cmd/cosh/testdata/make rename to goal/cmd/goal/testdata/make index a47bd26d3f..c19daef44c 100755 --- a/shell/cmd/cosh/testdata/make +++ b/goal/cmd/goal/testdata/make @@ -1,5 +1,5 @@ -#!/usr/bin/env cosh -// test makefile for cosh. +#!/usr/bin/env goal +// test makefile for goal. // example usage: // ./make build diff --git a/shell/cmd/cosh/typegen.go b/goal/cmd/goal/typegen.go similarity index 100% rename from shell/cmd/cosh/typegen.go rename to goal/cmd/goal/typegen.go diff --git a/shell/complete.go b/goal/complete.go similarity index 100% rename from shell/complete.go rename to goal/complete.go diff --git a/shell/exec.go b/goal/exec.go similarity index 100% rename from shell/exec.go rename to goal/exec.go diff --git a/shell/exec_test.go b/goal/exec_test.go similarity index 100% rename from shell/exec_test.go rename to goal/exec_test.go diff --git a/shell/execwords.go b/goal/execwords.go similarity index 100% rename from shell/execwords.go rename to goal/execwords.go diff --git a/shell/cosh/coshlib.go b/goal/goalib/goalib.go similarity index 94% rename from shell/cosh/coshlib.go rename to goal/goalib/goalib.go index ddd1499063..556abbc947 100644 --- a/shell/cosh/coshlib.go +++ b/goal/goalib/goalib.go @@ -2,9 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package cosh defines convenient utility functions for -// use in the cosh shell, available with the cosh prefix. -package cosh +// Package goalib defines convenient utility functions for +// use in the goal shell, available with the goalib prefix. +package goalib import ( "io/fs" diff --git a/shell/interpreter/cogentcore_org-core-base-datasize.go b/goal/interpreter/cogentcore_org-core-base-datasize.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-base-datasize.go rename to goal/interpreter/cogentcore_org-core-base-datasize.go diff --git a/shell/interpreter/cogentcore_org-core-base-elide.go b/goal/interpreter/cogentcore_org-core-base-elide.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-base-elide.go rename to goal/interpreter/cogentcore_org-core-base-elide.go diff --git a/shell/interpreter/cogentcore_org-core-base-errors.go b/goal/interpreter/cogentcore_org-core-base-errors.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-base-errors.go rename to goal/interpreter/cogentcore_org-core-base-errors.go diff --git a/shell/interpreter/cogentcore_org-core-base-fsx.go b/goal/interpreter/cogentcore_org-core-base-fsx.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-base-fsx.go rename to goal/interpreter/cogentcore_org-core-base-fsx.go diff --git a/shell/interpreter/cogentcore_org-core-base-strcase.go b/goal/interpreter/cogentcore_org-core-base-strcase.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-base-strcase.go rename to goal/interpreter/cogentcore_org-core-base-strcase.go diff --git a/shell/interpreter/cogentcore_org-core-base-stringsx.go b/goal/interpreter/cogentcore_org-core-base-stringsx.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-base-stringsx.go rename to goal/interpreter/cogentcore_org-core-base-stringsx.go diff --git a/shell/interpreter/cogentcore_org-core-shell-cosh.go b/goal/interpreter/cogentcore_org-core-shell-cosh.go similarity index 100% rename from shell/interpreter/cogentcore_org-core-shell-cosh.go rename to goal/interpreter/cogentcore_org-core-shell-cosh.go diff --git a/shell/interpreter/imports.go b/goal/interpreter/imports.go similarity index 100% rename from shell/interpreter/imports.go rename to goal/interpreter/imports.go diff --git a/shell/interpreter/imports_test.go b/goal/interpreter/imports_test.go similarity index 100% rename from shell/interpreter/imports_test.go rename to goal/interpreter/imports_test.go diff --git a/shell/interpreter/interpreter.go b/goal/interpreter/interpreter.go similarity index 100% rename from shell/interpreter/interpreter.go rename to goal/interpreter/interpreter.go diff --git a/shell/interpreter/make b/goal/interpreter/make similarity index 100% rename from shell/interpreter/make rename to goal/interpreter/make diff --git a/shell/paths.go b/goal/paths.go similarity index 100% rename from shell/paths.go rename to goal/paths.go diff --git a/shell/run.go b/goal/run.go similarity index 100% rename from shell/run.go rename to goal/run.go diff --git a/shell/shell.go b/goal/shell.go similarity index 100% rename from shell/shell.go rename to goal/shell.go diff --git a/shell/token.go b/goal/token.go similarity index 100% rename from shell/token.go rename to goal/token.go diff --git a/shell/transpile.go b/goal/transpile.go similarity index 100% rename from shell/transpile.go rename to goal/transpile.go diff --git a/shell/transpile_test.go b/goal/transpile_test.go similarity index 100% rename from shell/transpile_test.go rename to goal/transpile_test.go diff --git a/shell/README.md b/shell/README.md deleted file mode 100644 index c66364d59c..0000000000 --- a/shell/README.md +++ /dev/null @@ -1,190 +0,0 @@ -# Cogent Shell (cosh) - -Cogent Shell (cosh) is a shell that combines the best parts of Go and command-based shell languages like `bash` to provide an integrated shell experience that allows you to easily run terminal commands while using Go for complicated logic. This allows you to write concise, elegant, and readable shell code that runs quickly on all platforms, and transpiles to Go (i.e, can be compiled by `go build`). - -The simple idea is that each line is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below), and they can be intermixed by wrapping Go within `{ }` and shell code from within backticks (\`). We henceforth refer to shell code as `exec` code (in reference to the Go & Cogent `exec` package that we use to execute programs), given the potential ambituity of the entire `cosh` language being the shell. There are different syntactic formatting rules for these two domains of Go and Exec, within cosh: - -* Go code is processed and formatted as usual (e.g., white space is irrelevant, etc). -* Exec code is space separated, like normal command-line invocations. - -Examples: - -```go -for i, f := range cosh.SplitLines(`ls -la`) { // `ls` executes returns string - echo {i} {strings.ToLower(f)} // {} surrounds go within shell -} -``` - -`shell.SplitLines` is a function that runs `strings.Split(arg, "\n")`, defined in the cosh standard library of such frequently-used helper functions. - -You can easily perform handy duration and data size formatting: - -```go -22010706 * time.Nanosecond // 22.010706ms -datasize.Size(44610930) // 42.5 MB -``` - -# Special syntax - -## Multiple statements per line - -* Multiple statements can be combined on one line, separated by `;` as in regular Go and shell languages. Critically, the language determination for the first statement determines the language for the remaining statements; you cannot intermix the two on one line, when using `;` -# Exec mode - -## Environment variables - -* `set ` (space delimited as in all exec mode, no equals) - -## Output redirction - -* Standard output redirect: `>` and `>&` (and `|`, `|&` if needed) - -## Control flow - -* Any error stops the script execution, except for statements wrapped in `[ ]`, indicating an "optional" statement, e.g.: - -```sh -cd some; [mkdir sub]; cd sub -``` - -* `&` at the end of a statement runs in the background (as in bash) -- otherwise it waits until it completes before it continues. - -* `jobs`, `fg`, `bg`, and `kill` builtin commands function as in usual bash. - -## Exec functions (aliases) - -Use the `command` keyword to define new functions for Exec mode execution, which can then be used like any other command, for example: - -```sh -command list { - ls -la args... -} -``` - -```sh -cd data -list *.tsv -``` - -The `command` is transpiled into a Go function that takes `args ...string`. In the command function body, you can use the `args...` expression to pass all of the args, or `args[1]` etc to refer to specific positional indexes, as usual. - -The command function name is registered so that the standard shell execution code can run the function, passing the args. You can also call it directly from Go code using the standard parentheses expression. - -# Script Files and Makefile-like functionality - -As with most scripting languages, a file of cosh code can be made directly executable by appending a "shebang" expression at the start of the file: - -```sh -#!/usr/bin/env cosh -``` - -When executed this way, any additional args are available via an `args []any` variable, which can be passed to a command as follows: -```go -install {args...} -``` -or by referring to specific arg indexes etc. - -To make a script behave like a standard Makefile, you can define different `command`s for each of the make commands, and then add the following at the end of the file to use the args to run commands: - -```go -shell.RunCommands(args) -``` - -See [make](cmd/cosh/testdata/make) for an example, in `cmd/cosh/testdata/make`, which can be run for example using: - -```sh -./make build -``` - -Note that there is nothing special about the name `make` here, so this can be done with any file. - -The `make` package defines a number of useful utility functions that accomplish the standard dependency and file timestamp checking functionality from the standard `make` command, as in the [magefile](https://magefile.org/dependencies/) system. Note that the cosh direct exec command syntax makes the resulting make files much closer to a standard bash-like Makefile, while still having all the benefits of Go control and expressions, compared to magefile. - -TODO: implement and document above. - -# SSH connections to remote hosts - -Any number of active SSH connections can be maintained and used dynamically within a script, including simple ways of copying data among the different hosts (including the local host). The Go mode execution is always on the local host in one running process, and only the shell commands are executed remotely, enabling a unique ability to easily coordinate and distribute processing and data across various hosts. - -Each host maintains its own working directory and environment variables, which can be configured and re-used by default whenever using a given host. - -* `cossh hostname.org [name]` establishes a connection, using given optional name to refer to this connection. If the name is not provided, a sequential number will be used, starting with 1, with 0 referring always to the local host. - -* `@name` then refers to the given host in all subsequent commands, with `@0` referring to the local host where the cosh script is running. - -### Explicit per-command specification of host - -```sh -@name cd subdir; ls -``` - -### Default host - -```sh -@name // or: -cossh @name -``` - -uses the given host for all subsequent commands (unless explicitly specified), until the default is changed. Use `cossh @0` to return to localhost. - -### Redirect input / output among hosts - -The output of a remote host command can be sent to a file on the local host: -```sh -@name cat hostfile.tsv > @0:localfile.tsv -``` -Note the use of the `:` colon delimiter after the host name here. TODO: You cannot send output to a remote host file (e.g., `> @host:remotefile.tsv`) -- maybe with sftp? - -The output of any command can also be piped to a remote host as its standard input: -```sh -ls *.tsv | @host cat > files.txt -``` - -### scp to copy files easily - -The builtin `scp` function allows easy copying of files across hosts, using the persistent connections established with `cossh` instead of creating new connections as in the standard scp command. - -`scp` is _always_ run from the local host, with the remote host filename specified as `@name:remotefile` - -```sh -scp @name:hostfile.tsv localfile.tsv -``` - -TODO: Importantly, file wildcard globbing works as expected: -```sh -scp @name:*.tsv @0:data/ -``` - -and entire directories can be copied, as in `cp -a` or `cp -r` (this behavior is automatic and does not require a flag). - -### Close connections - -```sh -cossh close -``` - -Will close all active connections and return the default host to @0. All active connections are also automatically closed when the shell terminates. - -# Other Utilties - -** need a replacement for findnm -- very powerful but garbage.. - -# Rules for Go vs. Shell determination - -The critical extension from standard Go syntax is for lines that are processed by the `Exec` functions, used for running arbitrary programs on the user's executable path. Here are the rules (word = IDENT token): - -* Backticks "``" anywhere: Exec. Returns a `string`. -* Within Exec, `{}`: Go -* Line starts with `Go` Keyword: Go -* Line is one word: Exec -* Line starts with `path`: Exec -* Line starts with `"string"`: Exec -* Line starts with `word word`: Exec -* Line starts with `word {`: Exec -* Otherwise: Go - -# TODO: - -* likewise, need to run everything effectively as a bg job with our own explicit Wait, which we can then communicate with to move from fg to bg. - - diff --git a/tensor/README.md b/tensor/README.md index 5748682800..dddb2f0a92 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -6,7 +6,7 @@ The `tensor.Indexed` type provides the universal representation of a homogenous The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. -The [cosl](../cosl) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. +The [goal](../goal) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. Use the `[Set]FloatRow[Cell]` methods wherever possible, for the most efficient and natural indirection through the `tensor.Indexed` indexes. The 1D methods on underlying tensor data do not indirect through the indexes. @@ -18,11 +18,11 @@ All tensor package functions are registered using a global name-to-function map Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including [plot/plotcore](../plot/plotcore). -* [datafs](datafs) provides a virtual filesystem (FS) for organizing arbitrary collections of data, supporting interactive, ad-hoc (notebook style) as well as systematic data processing. Interactive [cosl](../cosl) shell commands (`cd`, `ls`, `mkdir` etc) can be used to navigate the data space, with numerical expressions immediately available to operate on the data and save results back to the filesystem. Furthermore, the data can be directly copied to / from the OS filesystem to persist it, and `cosl` can transparently access data on remote systems through ssh. Furthermore, the [databrowser](databrowser) provides a fully interactive GUI for inspecting and plotting data. +* [datafs](datafs) provides a virtual filesystem (FS) for organizing arbitrary collections of data, supporting interactive, ad-hoc (notebook style) as well as systematic data processing. Interactive [goal](../goal) shell commands (`cd`, `ls`, `mkdir` etc) can be used to navigate the data space, with numerical expressions immediately available to operate on the data and save results back to the filesystem. Furthermore, the data can be directly copied to / from the OS filesystem to persist it, and `goal` can transparently access data on remote systems through ssh. Furthermore, the [databrowser](databrowser) provides a fully interactive GUI for inspecting and plotting data. * [tensorcore](tensorcore) provides core widgets for graphically displaying the `Tensor` and `Table` data, which are used in `datafs`. -* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data, including the standard `+, -, *, /` operators. `cosl` then calls these functions. +* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data, including the standard `+, -, *, /` operators. `goal` then calls these functions. * [plot/plotcore](../plot/plotcore) supports interactive plotting of `Table` data. diff --git a/tensor/funcs.go b/tensor/funcs.go index 4be480c480..91f17dd151 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -66,7 +66,7 @@ func NewFunc(name string, fun any, out int, stringFirst ...bool) (*Func, error) var Funcs map[string]*Func // AddFunc adds given named function to the global tensor named function -// registry, which is used in cosl to call functions by name, and +// registry, which is used in goal to call functions by name, and // in specific packages to call functions by enum String() names. // Use the standard Go CamelCase name -- will be auto-lowercased. // The number of output arguments must be provided here, From fe6870cdffe45faf3dd918d7aa4d25d7736a345e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 16:04:55 -0700 Subject: [PATCH 058/311] fix table syntax --- goal/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/goal/README.md b/goal/README.md index 016cc71075..7ac960f8e9 100644 --- a/goal/README.md +++ b/goal/README.md @@ -210,9 +210,9 @@ All elements of a _Goal_ math expression are [tensors](../tensor), specifically Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) -| Goal | Python | MATLAB | Notes | ------------------------------------ -| same: | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of array `a` | +| Goal | Python | MATLAB | Notes | +| ----- | ------ | ------ | ------ | +| same: | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of array `a` | | `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of array `a` | | same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` array | | same: | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of array a | From 2a9223fe25b2103e1a1d9f617cf91bf7e3f2c2ac Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 17:16:38 -0700 Subject: [PATCH 059/311] renamed everything to Goal, started on math mode: need an ast. --- goal/README.md | 6 +- goal/builtins.go | 100 ++++++++-------- goal/complete.go | 18 +-- goal/exec.go | 110 ++++++++--------- goal/exec_test.go | 4 +- goal/execwords.go | 2 +- goal/{shell.go => goal.go} | 234 ++++++++++++++++++------------------- goal/math.go | 13 +++ goal/paths.go | 2 +- goal/run.go | 22 ++-- goal/token.go | 8 +- goal/transpile.go | 99 ++++++++-------- goal/transpile_test.go | 29 +++-- 13 files changed, 340 insertions(+), 307 deletions(-) rename goal/{shell.go => goal.go} (68%) create mode 100644 goal/math.go diff --git a/goal/README.md b/goal/README.md index 7ac960f8e9..30c6fa752e 100644 --- a/goal/README.md +++ b/goal/README.md @@ -1,12 +1,14 @@ # Goal: Go augmented language -_Goal_ is an augmented version of the _Go_ language, that transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: +_Goal_ is an augmented version of the _Go_ language, which combines the best parts of _Go_, `bash`, and Python, to provide and integrated shell and numerical expression processing experience. + +_Goal_ transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: * Shell scripting, where you want to be able to directly call other executable programs with arguments, without having to navigate all the complexity of the standard [os.exec](https://pkg.go.dev/os/exec) package. * Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and iterators etc. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. -The main goal of _Goal_ is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where it is particularly important. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, _Goal_ retains in its _Go_ foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. +The main goal of _Goal_ is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where the Go language has been a barrier. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, _Goal_ retains in its _Go_ foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. For the shell scripting aspect of _Goal_, the simple idea is that each line of code is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below). If a line starts off with something like `ls -la...` then it is clear that it is not valid Go code, and it is therefore processed as a shell command. diff --git a/goal/builtins.go b/goal/builtins.go index 941c24cd53..9251361663 100644 --- a/goal/builtins.go +++ b/goal/builtins.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "context" @@ -20,24 +20,24 @@ import ( ) // InstallBuiltins adds the builtin shell commands to [Shell.Builtins]. -func (sh *Shell) InstallBuiltins() { - sh.Builtins = make(map[string]func(cmdIO *exec.CmdIO, args ...string) error) - sh.Builtins["cd"] = sh.Cd - sh.Builtins["exit"] = sh.Exit - sh.Builtins["jobs"] = sh.JobsCmd - sh.Builtins["kill"] = sh.Kill - sh.Builtins["set"] = sh.Set - sh.Builtins["add-path"] = sh.AddPath - sh.Builtins["which"] = sh.Which - sh.Builtins["source"] = sh.Source - sh.Builtins["cossh"] = sh.CoSSH - sh.Builtins["scp"] = sh.Scp - sh.Builtins["debug"] = sh.Debug - sh.Builtins["history"] = sh.History +func (gl *Goal) InstallBuiltins() { + gl.Builtins = make(map[string]func(cmdIO *exec.CmdIO, args ...string) error) + gl.Builtins["cd"] = gl.Cd + gl.Builtins["exit"] = gl.Exit + gl.Builtins["jobs"] = gl.JobsCmd + gl.Builtins["kill"] = gl.Kill + gl.Builtins["set"] = gl.Set + gl.Builtins["add-path"] = gl.AddPath + gl.Builtins["which"] = gl.Which + gl.Builtins["source"] = gl.Source + gl.Builtins["cossh"] = gl.CoSSH + gl.Builtins["scp"] = gl.Scp + gl.Builtins["debug"] = gl.Debug + gl.Builtins["history"] = gl.History } // Cd changes the current directory. -func (sh *Shell) Cd(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Cd(cmdIO *exec.CmdIO, args ...string) error { if len(args) > 1 { return fmt.Errorf("no more than one argument can be passed to cd") } @@ -63,18 +63,18 @@ func (sh *Shell) Cd(cmdIO *exec.CmdIO, args ...string) error { if err != nil { return err } - sh.Config.Dir = dir + gl.Config.Dir = dir return nil } // Exit exits the shell. -func (sh *Shell) Exit(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Exit(cmdIO *exec.CmdIO, args ...string) error { os.Exit(0) return nil } // Set sets the given environment variable to the given value. -func (sh *Shell) Set(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Set(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 2 { return fmt.Errorf("expected two arguments, got %d", len(args)) } @@ -82,8 +82,8 @@ func (sh *Shell) Set(cmdIO *exec.CmdIO, args ...string) error { } // JobsCmd is the builtin jobs command -func (sh *Shell) JobsCmd(cmdIO *exec.CmdIO, args ...string) error { - for i, jb := range sh.Jobs { +func (gl *Goal) JobsCmd(cmdIO *exec.CmdIO, args ...string) error { + for i, jb := range gl.Jobs { cmdIO.Printf("[%d] %s\n", i+1, jb.String()) } return nil @@ -91,27 +91,27 @@ func (sh *Shell) JobsCmd(cmdIO *exec.CmdIO, args ...string) error { // Kill kills a job by job number or PID. // Just expands the job id expressions %n into PIDs and calls system kill. -func (sh *Shell) Kill(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Kill(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { return fmt.Errorf("cosh kill: expected at least one argument") } - sh.JobIDExpand(args) - sh.Config.RunIO(cmdIO, "kill", args...) + gl.JobIDExpand(args) + gl.Config.RunIO(cmdIO, "kill", args...) return nil } // Fg foregrounds a job by job number -func (sh *Shell) Fg(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Fg(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 1 { return fmt.Errorf("cosh fg: requires exactly one job id argument") } jid := args[0] - exp := sh.JobIDExpand(args) + exp := gl.JobIDExpand(args) if exp != 1 { return fmt.Errorf("cosh fg: argument was not a job id in the form %%n") } jno, _ := strconv.Atoi(jid[1:]) // guaranteed good - job := sh.Jobs[jno] + job := gl.Jobs[jno] cmdIO.Printf("foregrounding job [%d]\n", jno) _ = job // todo: the problem here is we need to change the stdio for running job @@ -123,7 +123,7 @@ func (sh *Shell) Fg(cmdIO *exec.CmdIO, args ...string) error { } // AddPath adds the given path(s) to $PATH. -func (sh *Shell) AddPath(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) AddPath(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { return fmt.Errorf("cosh add-path expected at least one argument") } @@ -141,30 +141,30 @@ func (sh *Shell) AddPath(cmdIO *exec.CmdIO, args ...string) error { // Which reports the executable associated with the given command. // Processes builtins and commands, and if not found, then passes on // to exec which. -func (sh *Shell) Which(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Which(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 1 { return fmt.Errorf("cosh which: requires one argument") } cmd := args[0] - if _, hasCmd := sh.Commands[cmd]; hasCmd { + if _, hasCmd := gl.Commands[cmd]; hasCmd { cmdIO.Println(cmd, "is a user-defined command") return nil } - if _, hasBlt := sh.Builtins[cmd]; hasBlt { + if _, hasBlt := gl.Builtins[cmd]; hasBlt { cmdIO.Println(cmd, "is a cosh builtin command") return nil } - sh.Config.RunIO(cmdIO, "which", args...) + gl.Config.RunIO(cmdIO, "which", args...) return nil } // Source loads and evaluates the given file(s) -func (sh *Shell) Source(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Source(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { return fmt.Errorf("cosh source: requires at least one argument") } for _, fn := range args { - sh.TranspileCodeFromFile(fn) + gl.TranspileCodeFromFile(fn) } // note that we do not execute the file -- just loads it in return nil @@ -176,18 +176,18 @@ func (sh *Shell) Source(cmdIO *exec.CmdIO, args ...string) error { // - host [name] -- connects to a server specified in first arg and switches // to using it, with optional name instead of default sequential number. // - close -- closes all open connections, or the specified one -func (sh *Shell) CoSSH(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) CoSSH(cmdIO *exec.CmdIO, args ...string) error { if len(args) < 1 { return fmt.Errorf("cossh: requires at least one argument") } cmd := args[0] var err error host := "" - name := fmt.Sprintf("%d", 1+len(sh.SSHClients)) + name := fmt.Sprintf("%d", 1+len(gl.SSHClients)) con := false switch { case cmd == "close": - sh.CloseSSH() + gl.CloseSSH() return nil case cmd == "@" && len(args) == 2: name = args[1] @@ -200,19 +200,19 @@ func (sh *Shell) CoSSH(cmdIO *exec.CmdIO, args ...string) error { host = args[0] } if con { - cl := sshclient.NewClient(sh.SSH) + cl := sshclient.NewClient(gl.SSH) err = cl.Connect(host) if err != nil { return err } - sh.SSHClients[name] = cl - sh.SSHActive = name + gl.SSHClients[name] = cl + gl.SSHActive = name } else { if name == "0" { - sh.SSHActive = "" + gl.SSHActive = "" } else { - sh.SSHActive = name - cl := sh.ActiveSSH() + gl.SSHActive = name + cl := gl.ActiveSSH() if cl == nil { err = fmt.Errorf("cosh: ssh connection named: %q not found", name) } @@ -226,7 +226,7 @@ func (sh *Shell) CoSSH(cmdIO *exec.CmdIO, args ...string) error { // The order is from -> to, as in standard cp. // The remote filename is automatically relative to the current working // directory on the remote host. -func (sh *Shell) Scp(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Scp(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 2 { return fmt.Errorf("scp: requires exactly two arguments") } @@ -250,12 +250,12 @@ func (sh *Shell) Scp(cmdIO *exec.CmdIO, args ...string) error { host := hfn[1:ci] hfn = hfn[ci+1:] - cl, err := sh.SSHByHost(host) + cl, err := gl.SSHByHost(host) if err != nil { return err } - ctx := sh.Ctx + ctx := gl.Ctx if ctx == nil { ctx = context.Background() } @@ -269,7 +269,7 @@ func (sh *Shell) Scp(cmdIO *exec.CmdIO, args ...string) error { } // Debug changes log level -func (sh *Shell) Debug(cmdIO *exec.CmdIO, args ...string) error { +func (gl *Goal) Debug(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { if logx.UserLevel == slog.LevelDebug { logx.UserLevel = slog.LevelInfo @@ -289,8 +289,8 @@ func (sh *Shell) Debug(cmdIO *exec.CmdIO, args ...string) error { } // History shows history -func (sh *Shell) History(cmdIO *exec.CmdIO, args ...string) error { - n := len(sh.Hist) +func (gl *Goal) History(cmdIO *exec.CmdIO, args ...string) error { + n := len(gl.Hist) nh := n if len(args) == 1 { an, err := strconv.Atoi(args[0]) @@ -302,7 +302,7 @@ func (sh *Shell) History(cmdIO *exec.CmdIO, args ...string) error { return fmt.Errorf("history: uses at most one argument") } for i := n - nh; i < n; i++ { - cmdIO.Printf("%d:\t%s\n", i, sh.Hist[i]) + cmdIO.Printf("%d:\t%s\n", i, gl.Hist[i]) } return nil } diff --git a/goal/complete.go b/goal/complete.go index bf79e0bf92..25caa5f51d 100644 --- a/goal/complete.go +++ b/goal/complete.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "os" @@ -16,14 +16,14 @@ import ( ) // CompleteMatch is the [complete.MatchFunc] for the shell. -func (sh *Shell) CompleteMatch(data any, text string, posLine, posChar int) (md complete.Matches) { +func (gl *Goal) CompleteMatch(data any, text string, posLine, posChar int) (md complete.Matches) { comps := complete.Completions{} text = text[:posChar] md.Seed = complete.SeedPath(text) fullPath := complete.SeedSpace(text) fullPath = errors.Log1(homedir.Expand(fullPath)) parent := strings.TrimSuffix(fullPath, md.Seed) - dir := filepath.Join(sh.Config.Dir, parent) + dir := filepath.Join(gl.Config.Dir, parent) if filepath.IsAbs(parent) { dir = parent } @@ -37,18 +37,18 @@ func (sh *Shell) CompleteMatch(data any, text string, posLine, posChar int) (md comps = append(comps, complete.Completion{ Text: name, Icon: icon, - Desc: filepath.Join(sh.Config.Dir, name), + Desc: filepath.Join(gl.Config.Dir, name), }) } if parent == "" { - for cmd := range sh.Builtins { + for cmd := range gl.Builtins { comps = append(comps, complete.Completion{ Text: cmd, Icon: icons.Terminal, Desc: "Builtin command: " + cmd, }) } - for cmd := range sh.Commands { + for cmd := range gl.Commands { comps = append(comps, complete.Completion{ Text: cmd, Icon: icons.Terminal, @@ -63,18 +63,18 @@ func (sh *Shell) CompleteMatch(data any, text string, posLine, posChar int) (md } // CompleteEdit is the [complete.EditFunc] for the shell. -func (sh *Shell) CompleteEdit(data any, text string, cursorPos int, completion complete.Completion, seed string) (ed complete.Edit) { +func (gl *Goal) CompleteEdit(data any, text string, cursorPos int, completion complete.Completion, seed string) (ed complete.Edit) { return complete.EditWord(text, cursorPos, completion.Text, seed) } // ReadlineCompleter implements [github.com/ergochat/readline.AutoCompleter]. type ReadlineCompleter struct { - Shell *Shell + Goal *Goal } func (rc *ReadlineCompleter) Do(line []rune, pos int) (newLine [][]rune, length int) { text := string(line) - md := rc.Shell.CompleteMatch(nil, text, 0, pos) + md := rc.Goal.CompleteMatch(nil, text, 0, pos) res := [][]rune{} for _, match := range md.Matches { after := strings.TrimPrefix(match.Text, md.Seed) diff --git a/goal/exec.go b/goal/exec.go index e79cb151cc..a8006d909d 100644 --- a/goal/exec.go +++ b/goal/exec.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "bytes" @@ -26,17 +26,17 @@ import ( // - start = calls Start on the command, which then runs asynchronously, with // a goroutine forked to Wait for it and close its IO // - output = return the output of the command as a string (otherwise return is "") -func (sh *Shell) Exec(errOk, start, output bool, cmd any, args ...any) string { +func (gl *Goal) Exec(errOk, start, output bool, cmd any, args ...any) string { out := "" - if !errOk && len(sh.Errors) > 0 { + if !errOk && len(gl.Errors) > 0 { return out } - cmdIO := exec.NewCmdIO(&sh.Config) + cmdIO := exec.NewCmdIO(&gl.Config) cmdIO.StackStart() if start { cmdIO.PushIn(nil) // no stdin for bg } - cl, scmd, sargs := sh.ExecArgs(cmdIO, errOk, cmd, args...) + cl, scmd, sargs := gl.ExecArgs(cmdIO, errOk, cmd, args...) if scmd == "" { return out } @@ -52,35 +52,35 @@ func (sh *Shell) Exec(errOk, start, output bool, cmd any, args ...any) string { err = cl.Run(&cmdIO.StdIOState, scmd, sargs...) } if !errOk { - sh.AddError(err) + gl.AddError(err) } } else { ran := false - ran, out = sh.RunBuiltinOrCommand(cmdIO, errOk, output, scmd, sargs...) + ran, out = gl.RunBuiltinOrCommand(cmdIO, errOk, output, scmd, sargs...) if !ran { - sh.isCommand.Push(false) + gl.isCommand.Push(false) switch { case start: - err = sh.Config.StartIO(cmdIO, scmd, sargs...) - sh.Jobs.Push(cmdIO) + err = gl.Config.StartIO(cmdIO, scmd, sargs...) + gl.Jobs.Push(cmdIO) go func() { if !cmdIO.OutIsPipe() { - fmt.Printf("[%d] %s\n", len(sh.Jobs), cmdIO.String()) + fmt.Printf("[%d] %s\n", len(gl.Jobs), cmdIO.String()) } cmdIO.Cmd.Wait() cmdIO.PopToStart() - sh.DeleteJob(cmdIO) + gl.DeleteJob(cmdIO) }() case output: cmdIO.PushOut(nil) - out, err = sh.Config.OutputIO(cmdIO, scmd, sargs...) + out, err = gl.Config.OutputIO(cmdIO, scmd, sargs...) default: - err = sh.Config.RunIO(cmdIO, scmd, sargs...) + err = gl.Config.RunIO(cmdIO, scmd, sargs...) } if !errOk { - sh.AddError(err) + gl.AddError(err) } - sh.isCommand.Pop() + gl.isCommand.Pop() } } if !start { @@ -91,64 +91,64 @@ func (sh *Shell) Exec(errOk, start, output bool, cmd any, args ...any) string { // RunBuiltinOrCommand runs a builtin or a command, returning true if it ran, // and the output string if running in output mode. -func (sh *Shell) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, output bool, cmd string, args ...string) (bool, string) { +func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, output bool, cmd string, args ...string) (bool, string) { out := "" - cmdFun, hasCmd := sh.Commands[cmd] - bltFun, hasBlt := sh.Builtins[cmd] + cmdFun, hasCmd := gl.Commands[cmd] + bltFun, hasBlt := gl.Builtins[cmd] if !hasCmd && !hasBlt { return false, out } if hasCmd { - sh.commandArgs.Push(args) - sh.isCommand.Push(true) + gl.commandArgs.Push(args) + gl.isCommand.Push(true) } // note: we need to set both os. and wrapper versions, so it works the same // in compiled vs. interpreted mode - oldsh := sh.Config.StdIO.Set(&cmdIO.StdIO) - oldwrap := sh.StdIOWrappers.SetWrappers(&cmdIO.StdIO) + oldsh := gl.Config.StdIO.Set(&cmdIO.StdIO) + oldwrap := gl.StdIOWrappers.SetWrappers(&cmdIO.StdIO) oldstd := cmdIO.SetToOS() if output { obuf := &bytes.Buffer{} // os.Stdout = obuf // needs a file - sh.Config.StdIO.Out = obuf - sh.StdIOWrappers.SetWrappedOut(obuf) + gl.Config.StdIO.Out = obuf + gl.StdIOWrappers.SetWrappedOut(obuf) cmdIO.PushOut(obuf) if hasCmd { cmdFun(args...) } else { - sh.AddError(bltFun(cmdIO, args...)) + gl.AddError(bltFun(cmdIO, args...)) } out = strings.TrimSuffix(obuf.String(), "\n") } else { if hasCmd { cmdFun(args...) } else { - sh.AddError(bltFun(cmdIO, args...)) + gl.AddError(bltFun(cmdIO, args...)) } } if hasCmd { - sh.isCommand.Pop() - sh.commandArgs.Pop() + gl.isCommand.Pop() + gl.commandArgs.Pop() } oldstd.SetToOS() - sh.StdIOWrappers.SetWrappers(oldwrap) - sh.Config.StdIO = *oldsh + gl.StdIOWrappers.SetWrappers(oldwrap) + gl.Config.StdIO = *oldsh return true, out } -func (sh *Shell) HandleArgErr(errok bool, err error) error { +func (gl *Goal) HandleArgErr(errok bool, err error) error { if err == nil { return err } if errok { - sh.Config.StdIO.ErrPrintln(err.Error()) + gl.Config.StdIO.ErrPrintln(err.Error()) } else { - sh.AddError(err) + gl.AddError(err) } return err } @@ -156,16 +156,16 @@ func (sh *Shell) HandleArgErr(errok bool, err error) error { // ExecArgs processes the args to given exec command, // handling all of the input / output redirection and // file globbing, homedir expansion, etc. -func (sh *Shell) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) (*sshclient.Client, string, []string) { - if len(sh.Jobs) > 0 { - jb := sh.Jobs.Peek() +func (gl *Goal) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) (*sshclient.Client, string, []string) { + if len(gl.Jobs) > 0 { + jb := gl.Jobs.Peek() if jb.OutIsPipe() { cmdIO.PushIn(jb.PipeIn.Peek()) } } scmd := reflectx.ToString(cmd) - cl := sh.ActiveSSH() - isCmd := sh.isCommand.Peek() + cl := gl.ActiveSSH() + isCmd := gl.isCommand.Peek() sargs := make([]string, 0, len(args)) var err error for _, a := range args { @@ -175,7 +175,7 @@ func (sh *Shell) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) ( } if cl == nil { s, err = homedir.Expand(s) - sh.HandleArgErr(errOk, err) + gl.HandleArgErr(errOk, err) // note: handling globbing in a later pass, to not clutter.. } else { if s[0] == '~' { @@ -190,18 +190,18 @@ func (sh *Shell) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) ( cl = nil } else { hnm := scmd[1:] - if scl, ok := sh.SSHClients[hnm]; ok { + if scl, ok := gl.SSHClients[hnm]; ok { newHost = hnm cl = scl } else { - sh.HandleArgErr(errOk, fmt.Errorf("cosh: ssh connection named: %q not found", hnm)) + gl.HandleArgErr(errOk, fmt.Errorf("cosh: ssh connection named: %q not found", hnm)) } } if len(sargs) > 0 { scmd = sargs[0] sargs = sargs[1:] } else { // just a ssh switch - sh.SSHActive = newHost + gl.SSHActive = newHost return nil, "", nil } } @@ -209,11 +209,11 @@ func (sh *Shell) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) ( s := sargs[i] switch { case s[0] == '>': - sargs = sh.OutToFile(cl, cmdIO, errOk, sargs, i) + sargs = gl.OutToFile(cl, cmdIO, errOk, sargs, i) case s[0] == '|': - sargs = sh.OutToPipe(cl, cmdIO, errOk, sargs, i) + sargs = gl.OutToPipe(cl, cmdIO, errOk, sargs, i) case cl == nil && isCmd && strings.HasPrefix(s, "args"): - sargs = sh.CmdArgs(errOk, sargs, i) + sargs = gl.CmdArgs(errOk, sargs, i) i-- // back up because we consume this one } } @@ -235,7 +235,7 @@ func (sh *Shell) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) ( } // OutToFile processes the > arg that sends output to a file -func (sh *Shell) OutToFile(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, sargs []string, i int) []string { +func (gl *Goal) OutToFile(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, sargs []string, i int) []string { n := len(sargs) s := sargs[i] sn := len(s) @@ -260,7 +260,7 @@ func (sh *Shell) OutToFile(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, narg = 1 } if fn == "" { - sh.HandleArgErr(errOk, fmt.Errorf("cosh: no output file specified")) + gl.HandleArgErr(errOk, fmt.Errorf("cosh: no output file specified")) return sargs } if cl != nil { @@ -285,13 +285,13 @@ func (sh *Shell) OutToFile(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, cmdIO.PushErr(f) } } else { - sh.HandleArgErr(errOk, err) + gl.HandleArgErr(errOk, err) } return sargs } // OutToPipe processes the | arg that sends output to a pipe -func (sh *Shell) OutToPipe(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, sargs []string, i int) []string { +func (gl *Goal) OutToPipe(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, sargs []string, i int) []string { s := sargs[i] sn := len(s) errf := false @@ -309,11 +309,11 @@ func (sh *Shell) OutToPipe(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, } // CmdArgs processes expressions involving "args" for commands -func (sh *Shell) CmdArgs(errOk bool, sargs []string, i int) []string { +func (gl *Goal) CmdArgs(errOk bool, sargs []string, i int) []string { // n := len(sargs) // s := sargs[i] // sn := len(s) - args := sh.commandArgs.Peek() + args := gl.commandArgs.Peek() // fmt.Println("command args:", args) @@ -327,8 +327,8 @@ func (sh *Shell) CmdArgs(errOk bool, sargs []string, i int) []string { } // CancelExecution calls the Cancel() function if set. -func (sh *Shell) CancelExecution() { - if sh.Cancel != nil { - sh.Cancel() +func (gl *Goal) CancelExecution() { + if gl.Cancel != nil { + gl.Cancel() } } diff --git a/goal/exec_test.go b/goal/exec_test.go index 657bf0275d..9efb0a82c7 100644 --- a/goal/exec_test.go +++ b/goal/exec_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "testing" @@ -11,5 +11,5 @@ import ( ) func TestExec(t *testing.T) { - assert.Equal(t, "hi", NewShell().Output("echo", "hi")) + assert.Equal(t, "hi", NewGoal().Output("echo", "hi")) } diff --git a/goal/execwords.go b/goal/execwords.go index e5515195fe..8d94db2e99 100644 --- a/goal/execwords.go +++ b/goal/execwords.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "fmt" diff --git a/goal/shell.go b/goal/goal.go similarity index 68% rename from goal/shell.go rename to goal/goal.go index 655262a17e..d2d5b538ea 100644 --- a/goal/shell.go +++ b/goal/goal.go @@ -2,10 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package shell provides the Cogent Shell (cosh), which combines the best parts -// of Go and bash to provide an integrated shell experience that allows you to -// easily run terminal commands while using Go for complicated logic. -package shell +// Package goal provides the Goal Go augmented language transpiler, +// which combines the best parts of Go, bash, and Python to provide +// an integrated shell and numerical expression processing experience. +package goal import ( "context" @@ -30,8 +30,8 @@ import ( "golang.org/x/tools/imports" ) -// Shell represents one running shell context. -type Shell struct { +// Goal represents one running Goal language context. +type Goal struct { // Config is the [exec.Config] used to run commands. Config exec.Config @@ -115,98 +115,98 @@ type Shell struct { lastCommand string } -// NewShell returns a new [Shell] with default options. -func NewShell() *Shell { - sh := &Shell{ +// NewGoal returns a new [Goal] with default options. +func NewGoal() *Goal { + gl := &Goal{ Config: exec.Config{ Dir: errors.Log1(os.Getwd()), Env: map[string]string{}, Buffer: false, }, } - sh.FuncToVar = true - sh.Config.StdIO.SetFromOS() - sh.SSH = sshclient.NewConfig(&sh.Config) - sh.SSHClients = make(map[string]*sshclient.Client) - sh.Commands = make(map[string]func(args ...string)) - sh.InstallBuiltins() - return sh + gl.FuncToVar = true + gl.Config.StdIO.SetFromOS() + gl.SSH = sshclient.NewConfig(&gl.Config) + gl.SSHClients = make(map[string]*sshclient.Client) + gl.Commands = make(map[string]func(args ...string)) + gl.InstallBuiltins() + return gl } // StartContext starts a processing context, // setting the Ctx and Cancel Fields. // Call EndContext when current operation finishes. -func (sh *Shell) StartContext() context.Context { - sh.Ctx, sh.Cancel = context.WithCancel(context.Background()) - return sh.Ctx +func (gl *Goal) StartContext() context.Context { + gl.Ctx, gl.Cancel = context.WithCancel(context.Background()) + return gl.Ctx } // EndContext ends a processing context, clearing the // Ctx and Cancel fields. -func (sh *Shell) EndContext() { - sh.Ctx = nil - sh.Cancel = nil +func (gl *Goal) EndContext() { + gl.Ctx = nil + gl.Cancel = nil } // SaveOrigStdIO saves the current Config.StdIO as the original to revert to // after an error, and sets the StdIOWrappers to use them. -func (sh *Shell) SaveOrigStdIO() { - sh.OrigStdIO = sh.Config.StdIO - sh.StdIOWrappers.NewWrappers(&sh.OrigStdIO) +func (gl *Goal) SaveOrigStdIO() { + gl.OrigStdIO = gl.Config.StdIO + gl.StdIOWrappers.NewWrappers(&gl.OrigStdIO) } // RestoreOrigStdIO reverts to using the saved OrigStdIO -func (sh *Shell) RestoreOrigStdIO() { - sh.Config.StdIO = sh.OrigStdIO - sh.OrigStdIO.SetToOS() - sh.StdIOWrappers.SetWrappers(&sh.OrigStdIO) +func (gl *Goal) RestoreOrigStdIO() { + gl.Config.StdIO = gl.OrigStdIO + gl.OrigStdIO.SetToOS() + gl.StdIOWrappers.SetWrappers(&gl.OrigStdIO) } // Close closes any resources associated with the shell, // including terminating any commands that are not running "nohup" // in the background. -func (sh *Shell) Close() { - sh.CloseSSH() +func (gl *Goal) Close() { + gl.CloseSSH() // todo: kill jobs etc } // CloseSSH closes all open ssh client connections -func (sh *Shell) CloseSSH() { - sh.SSHActive = "" - for _, cl := range sh.SSHClients { +func (gl *Goal) CloseSSH() { + gl.SSHActive = "" + for _, cl := range gl.SSHClients { cl.Close() } - sh.SSHClients = make(map[string]*sshclient.Client) + gl.SSHClients = make(map[string]*sshclient.Client) } // ActiveSSH returns the active ssh client -func (sh *Shell) ActiveSSH() *sshclient.Client { - if sh.SSHActive == "" { +func (gl *Goal) ActiveSSH() *sshclient.Client { + if gl.SSHActive == "" { return nil } - return sh.SSHClients[sh.SSHActive] + return gl.SSHClients[gl.SSHActive] } // Host returns the name we're running commands on, // which is empty if localhost (default). -func (sh *Shell) Host() string { - cl := sh.ActiveSSH() +func (gl *Goal) Host() string { + cl := gl.ActiveSSH() if cl == nil { return "" } - return "@" + sh.SSHActive + ":" + cl.Host + return "@" + gl.SSHActive + ":" + cl.Host } // HostAndDir returns the name we're running commands on, // which is empty if localhost (default), // and the current directory on that host. -func (sh *Shell) HostAndDir() string { +func (gl *Goal) HostAndDir() string { host := "" - dir := sh.Config.Dir + dir := gl.Config.Dir home := errors.Log1(homedir.Dir()) - cl := sh.ActiveSSH() + cl := gl.ActiveSSH() if cl != nil { - host = "@" + sh.SSHActive + ":" + cl.Host + ":" + host = "@" + gl.SSHActive + ":" + cl.Host + ":" dir = cl.Dir home = cl.HomeDir } @@ -219,44 +219,44 @@ func (sh *Shell) HostAndDir() string { } // SSHByHost returns the SSH client for given host name, with err if not found -func (sh *Shell) SSHByHost(host string) (*sshclient.Client, error) { - if scl, ok := sh.SSHClients[host]; ok { +func (gl *Goal) SSHByHost(host string) (*sshclient.Client, error) { + if scl, ok := gl.SSHClients[host]; ok { return scl, nil } return nil, fmt.Errorf("ssh connection named: %q not found", host) } // TotalDepth returns the sum of any unresolved paren, brace, or bracket depths. -func (sh *Shell) TotalDepth() int { - return num.Abs(sh.ParenDepth) + num.Abs(sh.BraceDepth) + num.Abs(sh.BrackDepth) +func (gl *Goal) TotalDepth() int { + return num.Abs(gl.ParenDepth) + num.Abs(gl.BraceDepth) + num.Abs(gl.BrackDepth) } // ResetCode resets the stack of transpiled code -func (sh *Shell) ResetCode() { - sh.Chunks = nil - sh.Lines = nil +func (gl *Goal) ResetCode() { + gl.Chunks = nil + gl.Lines = nil } // ResetDepth resets the current depths to 0 -func (sh *Shell) ResetDepth() { - sh.ParenDepth, sh.BraceDepth, sh.BrackDepth, sh.TypeDepth, sh.DeclDepth = 0, 0, 0, 0, 0 +func (gl *Goal) ResetDepth() { + gl.ParenDepth, gl.BraceDepth, gl.BrackDepth, gl.TypeDepth, gl.DeclDepth = 0, 0, 0, 0, 0 } // DepthError reports an error if any of the parsing depths are not zero, // to be called at the end of transpiling a complete block of code. -func (sh *Shell) DepthError() error { - if sh.TotalDepth() == 0 { +func (gl *Goal) DepthError() error { + if gl.TotalDepth() == 0 { return nil } str := "" - if sh.ParenDepth != 0 { - str += fmt.Sprintf("Incomplete parentheses (), remaining depth: %d\n", sh.ParenDepth) + if gl.ParenDepth != 0 { + str += fmt.Sprintf("Incomplete parentheses (), remaining depth: %d\n", gl.ParenDepth) } - if sh.BraceDepth != 0 { - str += fmt.Sprintf("Incomplete braces [], remaining depth: %d\n", sh.BraceDepth) + if gl.BraceDepth != 0 { + str += fmt.Sprintf("Incomplete braces [], remaining depth: %d\n", gl.BraceDepth) } - if sh.BrackDepth != 0 { - str += fmt.Sprintf("Incomplete brackets {}, remaining depth: %d\n", sh.BrackDepth) + if gl.BrackDepth != 0 { + str += fmt.Sprintf("Incomplete brackets {}, remaining depth: %d\n", gl.BrackDepth) } if str != "" { slog.Error(str) @@ -266,61 +266,61 @@ func (sh *Shell) DepthError() error { } // AddLine adds line on the stack -func (sh *Shell) AddLine(ln string) { - sh.Lines = append(sh.Lines, ln) +func (gl *Goal) AddLine(ln string) { + gl.Lines = append(gl.Lines, ln) } // Code returns the current transpiled lines, // split into chunks that should be compiled separately. -func (sh *Shell) Code() string { - sh.AddChunk() - if len(sh.Chunks) == 0 { +func (gl *Goal) Code() string { + gl.AddChunk() + if len(gl.Chunks) == 0 { return "" } - return strings.Join(sh.Chunks, "\n") + return strings.Join(gl.Chunks, "\n") } // AddChunk adds current lines into a chunk of code // that should be compiled separately. -func (sh *Shell) AddChunk() { - if len(sh.Lines) == 0 { +func (gl *Goal) AddChunk() { + if len(gl.Lines) == 0 { return } - sh.Chunks = append(sh.Chunks, strings.Join(sh.Lines, "\n")) - sh.Lines = nil + gl.Chunks = append(gl.Chunks, strings.Join(gl.Lines, "\n")) + gl.Lines = nil } // TranspileCode processes each line of given code, // adding the results to the LineStack -func (sh *Shell) TranspileCode(code string) { +func (gl *Goal) TranspileCode(code string) { lns := strings.Split(code, "\n") n := len(lns) if n == 0 { return } for _, ln := range lns { - hasDecl := sh.DeclDepth > 0 - tl := sh.TranspileLine(ln) - sh.AddLine(tl) - if sh.BraceDepth == 0 && sh.BrackDepth == 0 && sh.ParenDepth == 1 && sh.lastCommand != "" { - sh.lastCommand = "" - nl := len(sh.Lines) - sh.Lines[nl-1] = sh.Lines[nl-1] + ")" - sh.ParenDepth-- + hasDecl := gl.DeclDepth > 0 + tl := gl.TranspileLine(ln) + gl.AddLine(tl) + if gl.BraceDepth == 0 && gl.BrackDepth == 0 && gl.ParenDepth == 1 && gl.lastCommand != "" { + gl.lastCommand = "" + nl := len(gl.Lines) + gl.Lines[nl-1] = gl.Lines[nl-1] + ")" + gl.ParenDepth-- } - if hasDecl && sh.DeclDepth == 0 { // break at decl - sh.AddChunk() + if hasDecl && gl.DeclDepth == 0 { // break at decl + gl.AddChunk() } } } // TranspileCodeFromFile transpiles the code in given file -func (sh *Shell) TranspileCodeFromFile(file string) error { +func (gl *Goal) TranspileCodeFromFile(file string) error { b, err := os.ReadFile(file) if err != nil { return err } - sh.TranspileCode(string(b)) + gl.TranspileCode(string(b)) return nil } @@ -328,7 +328,7 @@ func (sh *Shell) TranspileCodeFromFile(file string) error { // given output Go file. If no existing package declaration // is found, then package main and func main declarations are // added. This also affects how functions are interpreted. -func (sh *Shell) TranspileFile(in string, out string) error { +func (gl *Goal) TranspileFile(in string, out string) error { b, err := os.ReadFile(in) if err != nil { return err @@ -343,27 +343,27 @@ func (sh *Shell) TranspileFile(in string, out string) error { } } if hasPackage { - sh.FuncToVar = false // use raw functions + gl.FuncToVar = false // use raw functions } - sh.TranspileCode(code) - sh.FuncToVar = true + gl.TranspileCode(code) + gl.FuncToVar = true if err != nil { return err } gen := "// Code generated by \"cosh build\"; DO NOT EDIT.\n\n" if hasPackage { - sh.Lines = slices.Insert(sh.Lines, 0, gen) + gl.Lines = slices.Insert(gl.Lines, 0, gen) } else { - sh.Lines = slices.Insert(sh.Lines, 0, gen, "package main", "", "func main() {", "shell := shell.NewShell()") - sh.Lines = append(sh.Lines, "}") + gl.Lines = slices.Insert(gl.Lines, 0, gen, "package main", "", "func main() {", "shell := shell.NewShell()") + gl.Lines = append(gl.Lines, "}") } - src := []byte(sh.Code()) + src := []byte(gl.Code()) res, err := imports.Process(out, src, nil) if err != nil { res = src slog.Error(err.Error()) } else { - err = sh.DepthError() + err = gl.DepthError() } werr := os.WriteFile(out, res, 0666) return errors.Join(err, werr) @@ -373,19 +373,19 @@ func (sh *Shell) TranspileFile(in string, out string) error { // and calls the Cancel function if set, to stop execution. // This is the main way that shell errors are handled. // It also prints the error. -func (sh *Shell) AddError(err error) error { +func (gl *Goal) AddError(err error) error { if err == nil { return nil } - sh.Errors = append(sh.Errors, err) + gl.Errors = append(gl.Errors, err) logx.PrintlnError(err) - sh.CancelExecution() + gl.CancelExecution() return err } // TranspileConfig transpiles the .cosh startup config file in the user's // home directory if it exists. -func (sh *Shell) TranspileConfig() error { +func (gl *Goal) TranspileConfig() error { path, err := homedir.Expand("~/.cosh") if err != nil { return err @@ -397,29 +397,29 @@ func (sh *Shell) TranspileConfig() error { } return err } - sh.TranspileCode(string(b)) + gl.TranspileCode(string(b)) return nil } // AddHistory adds given line to the Hist record of commands -func (sh *Shell) AddHistory(line string) { - sh.Hist = append(sh.Hist, line) +func (gl *Goal) AddHistory(line string) { + gl.Hist = append(gl.Hist, line) } // SaveHistory saves up to the given number of lines of current history // to given file, e.g., ~/.coshhist for the default cosh program. // If n is <= 0 all lines are saved. n is typically 500 by default. -func (sh *Shell) SaveHistory(n int, file string) error { +func (gl *Goal) SaveHistory(n int, file string) error { path, err := homedir.Expand(file) if err != nil { return err } - hn := len(sh.Hist) + hn := len(gl.Hist) sn := hn if n > 0 { sn = min(n, hn) } - lh := strings.Join(sh.Hist[hn-sn:hn], "\n") + lh := strings.Join(gl.Hist[hn-sn:hn], "\n") err = os.WriteFile(path, []byte(lh), 0666) if err != nil { return err @@ -429,7 +429,7 @@ func (sh *Shell) SaveHistory(n int, file string) error { // OpenHistory opens Hist history lines from given file, // e.g., ~/.coshhist -func (sh *Shell) OpenHistory(file string) error { +func (gl *Goal) OpenHistory(file string) error { path, err := homedir.Expand(file) if err != nil { return err @@ -438,20 +438,20 @@ func (sh *Shell) OpenHistory(file string) error { if err != nil { return err } - sh.Hist = strings.Split(string(b), "\n") + gl.Hist = strings.Split(string(b), "\n") return nil } // AddCommand adds given command to list of available commands -func (sh *Shell) AddCommand(name string, cmd func(args ...string)) { - sh.Commands[name] = cmd +func (gl *Goal) AddCommand(name string, cmd func(args ...string)) { + gl.Commands[name] = cmd } // RunCommands runs the given command(s). This is typically called // from a Makefile-style cosh script. -func (sh *Shell) RunCommands(cmds []any) error { +func (gl *Goal) RunCommands(cmds []any) error { for _, cmd := range cmds { - if cmdFun, hasCmd := sh.Commands[reflectx.ToString(cmd)]; hasCmd { + if cmdFun, hasCmd := gl.Commands[reflectx.ToString(cmd)]; hasCmd { cmdFun() } else { return errors.Log(fmt.Errorf("command %q not found", cmd)) @@ -461,10 +461,10 @@ func (sh *Shell) RunCommands(cmds []any) error { } // DeleteJob deletes the given job and returns true if successful, -func (sh *Shell) DeleteJob(cmdIO *exec.CmdIO) bool { - idx := slices.Index(sh.Jobs, cmdIO) +func (gl *Goal) DeleteJob(cmdIO *exec.CmdIO) bool { + idx := slices.Index(gl.Jobs, cmdIO) if idx >= 0 { - sh.Jobs = slices.Delete(sh.Jobs, idx, idx+1) + gl.Jobs = slices.Delete(gl.Jobs, idx, idx+1) return true } return false @@ -472,20 +472,20 @@ func (sh *Shell) DeleteJob(cmdIO *exec.CmdIO) bool { // JobIDExpand expands %n job id values in args with the full PID // returns number of PIDs expanded -func (sh *Shell) JobIDExpand(args []string) int { +func (gl *Goal) JobIDExpand(args []string) int { exp := 0 for i, id := range args { if id[0] == '%' { idx, err := strconv.Atoi(id[1:]) if err == nil { - if idx > 0 && idx <= len(sh.Jobs) { - jb := sh.Jobs[idx-1] + if idx > 0 && idx <= len(gl.Jobs) { + jb := gl.Jobs[idx-1] if jb.Cmd != nil && jb.Cmd.Process != nil { args[i] = fmt.Sprintf("%d", jb.Cmd.Process.Pid) exp++ } } else { - sh.AddError(fmt.Errorf("cosh: job number out of range: %d", idx)) + gl.AddError(fmt.Errorf("cosh: job number out of range: %d", idx)) } } } diff --git a/goal/math.go b/goal/math.go new file mode 100644 index 0000000000..0c762c4123 --- /dev/null +++ b/goal/math.go @@ -0,0 +1,13 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package goal + +import "fmt" + +func (gl *Goal) TranspileMath(toks Tokens) Tokens { + fmt.Printf("in math") + + return toks +} diff --git a/goal/paths.go b/goal/paths.go index 3dc2029484..e9a6ac73ff 100644 --- a/goal/paths.go +++ b/goal/paths.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "go/token" diff --git a/goal/run.go b/goal/run.go index d6347a962f..1725f30eca 100644 --- a/goal/run.go +++ b/goal/run.go @@ -2,14 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal // Run executes the given command string, waiting for the command to finish, // handling the given arguments appropriately. // If there is any error, it adds it to the shell, and triggers CancelExecution. // It forwards output to [exec.Config.Stdout] and [exec.Config.Stderr] appropriately. -func (sh *Shell) Run(cmd any, args ...any) { - sh.Exec(false, false, false, cmd, args...) +func (gl *Goal) Run(cmd any, args ...any) { + gl.Exec(false, false, false, cmd, args...) } // RunErrOK executes the given command string, waiting for the command to finish, @@ -17,28 +17,28 @@ func (sh *Shell) Run(cmd any, args ...any) { // It does not stop execution if there is an error. // If there is any error, it adds it to the shell. It forwards output to // [exec.Config.Stdout] and [exec.Config.Stderr] appropriately. -func (sh *Shell) RunErrOK(cmd any, args ...any) { - sh.Exec(true, false, false, cmd, args...) +func (gl *Goal) RunErrOK(cmd any, args ...any) { + gl.Exec(true, false, false, cmd, args...) } // Start starts the given command string for running in the background, // handling the given arguments appropriately. // If there is any error, it adds it to the shell. It forwards output to // [exec.Config.Stdout] and [exec.Config.Stderr] appropriately. -func (sh *Shell) Start(cmd any, args ...any) { - sh.Exec(false, true, false, cmd, args...) +func (gl *Goal) Start(cmd any, args ...any) { + gl.Exec(false, true, false, cmd, args...) } // Output executes the given command string, handling the given arguments // appropriately. If there is any error, it adds it to the shell. It returns // the stdout as a string and forwards stderr to [exec.Config.Stderr] appropriately. -func (sh *Shell) Output(cmd any, args ...any) string { - return sh.Exec(false, false, true, cmd, args...) +func (gl *Goal) Output(cmd any, args ...any) string { + return gl.Exec(false, false, true, cmd, args...) } // OutputErrOK executes the given command string, handling the given arguments // appropriately. If there is any error, it adds it to the shell. It returns // the stdout as a string and forwards stderr to [exec.Config.Stderr] appropriately. -func (sh *Shell) OutputErrOK(cmd any, args ...any) string { - return sh.Exec(true, false, true, cmd, args...) +func (gl *Goal) OutputErrOK(cmd any, args ...any) string { + return gl.Exec(true, false, true, cmd, args...) } diff --git a/goal/token.go b/goal/token.go index a84c01a790..e3adb6f39e 100644 --- a/goal/token.go +++ b/goal/token.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "go/scanner" @@ -309,11 +309,11 @@ func (tk Tokens) BracketDepths() (paren, brace, brack int) { } // Tokens converts the string into tokens -func (sh *Shell) Tokens(ln string) Tokens { +func (gl *Goal) Tokens(ln string) Tokens { fset := token.NewFileSet() f := fset.AddFile("", fset.Base(), len(ln)) var sc scanner.Scanner - sc.Init(f, []byte(ln), sh.errHandler, scanner.ScanComments|2) // 2 is non-exported dontInsertSemis + sc.Init(f, []byte(ln), gl.errHandler, scanner.ScanComments|2) // 2 is non-exported dontInsertSemis // note to Go team: just export this stuff. seriously. var toks Tokens @@ -328,6 +328,6 @@ func (sh *Shell) Tokens(ln string) Tokens { return toks } -func (sh *Shell) errHandler(pos token.Position, msg string) { +func (gl *Goal) errHandler(pos token.Position, msg string) { logx.PrintlnDebug("Scan Error:", pos, msg) } diff --git a/goal/transpile.go b/goal/transpile.go index 305e832255..642de5b93c 100644 --- a/goal/transpile.go +++ b/goal/transpile.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "fmt" @@ -15,55 +15,55 @@ import ( // TranspileLine is the main function for parsing a single line of shell input, // returning a new transpiled line of code that converts Exec code into corresponding // Go function calls. -func (sh *Shell) TranspileLine(ln string) string { +func (gl *Goal) TranspileLine(ln string) string { if len(ln) == 0 { return ln } if strings.HasPrefix(ln, "#!") { return "" } - toks := sh.TranspileLineTokens(ln) + toks := gl.TranspileLineTokens(ln) paren, brace, brack := toks.BracketDepths() - sh.ParenDepth += paren - sh.BraceDepth += brace - sh.BrackDepth += brack - if sh.TypeDepth > 0 && sh.BraceDepth == 0 { - sh.TypeDepth = 0 + gl.ParenDepth += paren + gl.BraceDepth += brace + gl.BrackDepth += brack + if gl.TypeDepth > 0 && gl.BraceDepth == 0 { + gl.TypeDepth = 0 } - if sh.DeclDepth > 0 && sh.ParenDepth == 0 { - sh.DeclDepth = 0 + if gl.DeclDepth > 0 && gl.ParenDepth == 0 { + gl.DeclDepth = 0 } // logx.PrintlnDebug("depths: ", sh.ParenDepth, sh.BraceDepth, sh.BrackDepth) return toks.Code() } // TranspileLineTokens returns the tokens for the full line -func (sh *Shell) TranspileLineTokens(ln string) Tokens { +func (gl *Goal) TranspileLineTokens(ln string) Tokens { if ln == "" { return nil } - toks := sh.Tokens(ln) + toks := gl.Tokens(ln) n := len(toks) if n == 0 { return toks } ewords, err := ExecWords(ln) if err != nil { - sh.AddError(err) + gl.AddError(err) return nil } logx.PrintlnDebug("\n########## line:\n", ln, "\nTokens:\n", toks.String(), "\nWords:\n", ewords) if toks[0].Tok == token.TYPE { - sh.TypeDepth++ + gl.TypeDepth++ } if toks[0].Tok == token.IMPORT || toks[0].Tok == token.VAR || toks[0].Tok == token.CONST { - sh.DeclDepth++ + gl.DeclDepth++ } - if sh.TypeDepth > 0 || sh.DeclDepth > 0 { + if gl.TypeDepth > 0 || gl.DeclDepth > 0 { logx.PrintlnDebug("go: type / decl defn") - return sh.TranspileGo(toks) + return gl.TranspileGo(toks) } t0 := toks[0] @@ -73,28 +73,33 @@ func (sh *Shell) TranspileLineTokens(ln string) Tokens { f0exec := (t0.Tok == token.IDENT && ExecWordIsCommand(ewords[0])) switch { + case t0.Tok == token.ILLEGAL: + fmt.Println(t0.Str) + if t0.Str != "" && t0.Str[:1] == "$" { + return gl.TranspileMath(toks[1:]) + } case t0.Tok == token.LBRACE: logx.PrintlnDebug("go: { } line") - return sh.TranspileGo(toks[1 : n-1]) + return gl.TranspileGo(toks[1 : n-1]) case t0.Tok == token.LBRACK: logx.PrintlnDebug("exec: [ ] line") - return sh.TranspileExec(ewords, false) // it processes the [ ] + return gl.TranspileExec(ewords, false) // it processes the [ ] case t0.Tok == token.ILLEGAL: logx.PrintlnDebug("exec: illegal") - return sh.TranspileExec(ewords, false) + return gl.TranspileExec(ewords, false) case t0.IsBacktickString(): logx.PrintlnDebug("exec: backquoted string") - exe := sh.TranspileExecString(t0.Str, false) + exe := gl.TranspileExecString(t0.Str, false) if n > 1 { // todo: is this an error? - exe.AddTokens(sh.TranspileGo(toks[1:])) + exe.AddTokens(gl.TranspileGo(toks[1:])) } return exe case t0.Tok == token.IDENT && t0.Str == "command": - sh.lastCommand = toks[1].Str // 1 is the name -- triggers AddCommand + gl.lastCommand = toks[1].Str // 1 is the name -- triggers AddCommand toks = toks[2:] // get rid of first toks.Insert(0, token.IDENT, "shell.AddCommand") toks.Insert(1, token.LPAREN) - toks.Insert(2, token.STRING, `"`+sh.lastCommand+`"`) + toks.Insert(2, token.STRING, `"`+gl.lastCommand+`"`) toks.Insert(3, token.COMMA) toks.Insert(4, token.FUNC) toks.Insert(5, token.LPAREN) @@ -102,51 +107,51 @@ func (sh *Shell) TranspileLineTokens(ln string) Tokens { toks.Insert(7, token.ELLIPSIS) toks.Insert(8, token.IDENT, "string") toks.Insert(9, token.RPAREN) - toks.AddTokens(sh.TranspileGo(toks[11:])) + toks.AddTokens(gl.TranspileGo(toks[11:])) case t0.IsGo(): if t0.Tok == token.GO { if !toks.Contains(token.LPAREN) { logx.PrintlnDebug("exec: go command") - return sh.TranspileExec(ewords, false) + return gl.TranspileExec(ewords, false) } } logx.PrintlnDebug("go keyword") - return sh.TranspileGo(toks) + return gl.TranspileGo(toks) case toks[n-1].Tok == token.INC: - return sh.TranspileGo(toks) + return gl.TranspileGo(toks) case t0pn > 0: // path expr logx.PrintlnDebug("exec: path...") - return sh.TranspileExec(ewords, false) + return gl.TranspileExec(ewords, false) case t0.Tok == token.STRING: logx.PrintlnDebug("exec: string...") - return sh.TranspileExec(ewords, false) + return gl.TranspileExec(ewords, false) case f0exec && en == 1: logx.PrintlnDebug("exec: 1 word") - return sh.TranspileExec(ewords, false) + return gl.TranspileExec(ewords, false) case !f0exec: // exec must be IDENT logx.PrintlnDebug("go: not ident") - return sh.TranspileGo(toks) + return gl.TranspileGo(toks) case f0exec && en > 1 && (ewords[1][0] == '=' || ewords[1][0] == ':' || ewords[1][0] == '+' || toks[1].Tok == token.COMMA): logx.PrintlnDebug("go: assignment or defn") - return sh.TranspileGo(toks) + return gl.TranspileGo(toks) case f0exec: // now any ident logx.PrintlnDebug("exec: ident..") - return sh.TranspileExec(ewords, false) + return gl.TranspileExec(ewords, false) default: logx.PrintlnDebug("go: default") - return sh.TranspileGo(toks) + return gl.TranspileGo(toks) } return toks } // TranspileGo returns transpiled tokens assuming Go code. // Unpacks any backtick encapsulated shell commands. -func (sh *Shell) TranspileGo(toks Tokens) Tokens { +func (gl *Goal) TranspileGo(toks Tokens) Tokens { n := len(toks) if n == 0 { return toks } - if sh.FuncToVar && toks[0].Tok == token.FUNC { // reorder as an assignment + if gl.FuncToVar && toks[0].Tok == token.FUNC { // reorder as an assignment if len(toks) > 1 && toks[1].Tok == token.IDENT { toks[0] = toks[1] toks.Insert(1, token.DEFINE) @@ -155,8 +160,8 @@ func (sh *Shell) TranspileGo(toks Tokens) Tokens { } gtoks := make(Tokens, 0, len(toks)) // return tokens for _, tok := range toks { - if sh.TypeDepth == 0 && tok.IsBacktickString() { - gtoks = append(gtoks, sh.TranspileExecString(tok.Str, true)...) + if gl.TypeDepth == 0 && tok.IsBacktickString() { + gtoks = append(gtoks, gl.TranspileExecString(tok.Str, true)...) } else { gtoks = append(gtoks, tok) } @@ -167,20 +172,20 @@ func (sh *Shell) TranspileGo(toks Tokens) Tokens { // TranspileExecString returns transpiled tokens assuming Exec code, // from a backtick-encoded string, with the given bool indicating // whether [Output] is needed. -func (sh *Shell) TranspileExecString(str string, output bool) Tokens { +func (gl *Goal) TranspileExecString(str string, output bool) Tokens { if len(str) <= 1 { return nil } ewords, err := ExecWords(str[1 : len(str)-1]) // enclosed string if err != nil { - sh.AddError(err) + gl.AddError(err) } - return sh.TranspileExec(ewords, output) + return gl.TranspileExec(ewords, output) } // TranspileExec returns transpiled tokens assuming Exec code, // with the given bools indicating the type of run to execute. -func (sh *Shell) TranspileExec(ewords []string, output bool) Tokens { +func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { n := len(ewords) if n == 0 { return nil @@ -225,10 +230,10 @@ func (sh *Shell) TranspileExec(ewords []string, output bool) Tokens { switch { case f == "{": // embedded go if n < i+3 { - sh.AddError(fmt.Errorf("cosh: no matching right brace } found in exec command line")) + gl.AddError(fmt.Errorf("cosh: no matching right brace } found in exec command line")) } else { gstr := ewords[i+1] - etoks.AddTokens(sh.TranspileGo(sh.Tokens(gstr))) + etoks.AddTokens(gl.TranspileGo(gl.Tokens(gstr))) etoks.Add(token.COMMA) i += 2 } @@ -245,12 +250,12 @@ func (sh *Shell) TranspileExec(ewords []string, output bool) Tokens { etoks.Add(token.COMMA) endExec() etoks.Add(token.SEMICOLON) - etoks.AddTokens(sh.TranspileExec(ewords[i+1:], output)) + etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)) return etoks case f == ";": endExec() etoks.Add(token.SEMICOLON) - etoks.AddTokens(sh.TranspileExec(ewords[i+1:], output)) + etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)) return etoks default: if f[0] == '"' || f[0] == '`' { diff --git a/goal/transpile_test.go b/goal/transpile_test.go index 94d07d470f..45bae8364f 100644 --- a/goal/transpile_test.go +++ b/goal/transpile_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package shell +package goal import ( "testing" @@ -74,9 +74,9 @@ func TestPaths(t *testing.T) { {"../an-other/dir/", `../an-other/dir/`}, {"https://google.com/search?q=hello%20world#body", `https://google.com/search?q=hello%20world#body`}, } - sh := NewShell() + gl := NewGoal() for _, test := range tests { - toks := sh.Tokens(test.i) + toks := gl.Tokens(test.i) p, _ := toks.Path(false) assert.Equal(t, test.e, p) } @@ -170,9 +170,9 @@ func TestTranspile(t *testing.T) { {"var data map[string]any", "var data map[string]any"}, } - sh := NewShell() + gl := NewGoal() for _, test := range tests { - o := sh.TranspileLine(test.i) + o := gl.TranspileLine(test.i) assert.Equal(t, test.e, o) } } @@ -190,10 +190,23 @@ shell.Run("ls", "-la", "args...") })`}, } - sh := NewShell() + gl := NewGoal() for _, test := range tests { - sh.TranspileCode(test.i) - o := sh.Code() + gl.TranspileCode(test.i) + o := gl.Code() + assert.Equal(t, test.e, o) + } +} + +func TestMath(t *testing.T) { + // logx.UserLevel = slog.LevelDebug + tests := []exIn{ + {"$x := 1", `x := tensor.NewFloat64Scalar(1)`}, + } + + gl := NewGoal() + for _, test := range tests { + o := gl.TranspileLine(test.i) assert.Equal(t, test.e, o) } } From ff52be5fd797e99fd78ed39591ec578c1d09a065 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 23:04:28 -0700 Subject: [PATCH 060/311] lots of updating from cosh / shell -> goal -- mostly working as a shell now with some issues.. and successfully transpiling a simple scalar initializer expression -- first $ math expr handled! --- base/fileinfo/mimetype.go | 2 +- goal/README.md | 14 +- goal/builtins.go | 18 +-- goal/cmd/goal/goal.go | 28 ++-- goal/cmd/goal/test.goal | 7 +- goal/cmd/goal/typegen.go | 8 +- goal/exec.go | 6 +- goal/execwords.go | 2 +- goal/goal.go | 22 +-- .../cogentcore_org-core-base-datasize.go | 29 ---- .../cogentcore_org-core-base-elide.go | 17 --- .../cogentcore_org-core-base-errors.go | 25 ---- .../cogentcore_org-core-base-fsx.go | 27 ---- .../cogentcore_org-core-base-strcase.go | 54 ------- .../cogentcore_org-core-base-stringsx.go | 19 --- .../cogentcore_org-core-shell-cosh.go | 21 --- goal/interpreter/imports.go | 21 +-- goal/interpreter/interpreter.go | 96 ++++++------- goal/interpreter/make | 17 --- goal/math.go | 135 +++++++++++++++++- goal/run.go | 10 +- goal/token.go | 2 +- goal/transpile.go | 20 +-- goal/transpile_test.go | 128 ++++++++--------- spell/dict/dtool.go | 2 +- tensor/funcs_test.go | 2 +- tensor/indexed.go | 14 +- tensor/number.go | 2 +- tensor/stats/metric/metric_test.go | 4 +- tensor/stats/stats/quantiles.go | 6 +- tensor/stats/stats/stats_test.go | 4 +- tensor/tmath/math_test.go | 2 +- tensor/tmath/ops_test.go | 2 +- .../cogentcore_org-core-goal-goalib.go | 21 +++ .../cogentcore_org-core-tensor-stats-stats.go | 1 + .../symbols/cogentcore_org-core-tensor.go | 5 + yaegicore/symbols/make | 2 +- 37 files changed, 378 insertions(+), 417 deletions(-) delete mode 100644 goal/interpreter/cogentcore_org-core-base-datasize.go delete mode 100644 goal/interpreter/cogentcore_org-core-base-elide.go delete mode 100644 goal/interpreter/cogentcore_org-core-base-errors.go delete mode 100644 goal/interpreter/cogentcore_org-core-base-fsx.go delete mode 100644 goal/interpreter/cogentcore_org-core-base-strcase.go delete mode 100644 goal/interpreter/cogentcore_org-core-base-stringsx.go delete mode 100644 goal/interpreter/cogentcore_org-core-shell-cosh.go delete mode 100755 goal/interpreter/make create mode 100644 yaegicore/symbols/cogentcore_org-core-goal-goalib.go diff --git a/base/fileinfo/mimetype.go b/base/fileinfo/mimetype.go index d44bcc819f..f1e6c03891 100644 --- a/base/fileinfo/mimetype.go +++ b/base/fileinfo/mimetype.go @@ -316,7 +316,7 @@ var StandardMimes = []MimeType{ {"text/x-forth", []string{".frt"}, Code, Forth}, // note: ".fs" conflicts with fsharp {"text/x-fortran", []string{".f", ".F"}, Code, Fortran}, {"text/x-fsharp", []string{".fs", ".fsi"}, Code, FSharp}, - {"text/x-gosrc", []string{".go", ".mod", ".work", ".cosh"}, Code, Go}, + {"text/x-gosrc", []string{".go", ".mod", ".work", ".goal"}, Code, Go}, {"text/x-haskell", []string{".hs", ".lhs"}, Code, Haskell}, {"text/x-literate-haskell", nil, Code, Haskell}, // todo: not sure if same or not diff --git a/goal/README.md b/goal/README.md index 30c6fa752e..3660ea502b 100644 --- a/goal/README.md +++ b/goal/README.md @@ -29,7 +29,7 @@ for _, x := range $[1,2,3]$ { In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. -We henceforth refer to shell code as `exec` code (in reference to the Go & Cogent `exec` package that we use to execute programs), given the potential ambituity of the entire `cosh` language being the shell. There are different syntactic formatting rules for these two domains of Go and Exec, within cosh: +We henceforth refer to shell code as `exec` code (in reference to the Go & Cogent `exec` package that we use to execute programs), given the potential ambituity of the entire `goal` language being the shell. There are different syntactic formatting rules for these two domains of Go and Exec, within goal: * Go code is processed and formatted as usual (e.g., white space is irrelevant, etc). * Exec code is space separated, like normal command-line invocations. @@ -89,10 +89,10 @@ The command function name is registered so that the standard shell execution cod # Script Files and Makefile-like functionality -As with most scripting languages, a file of cosh code can be made directly executable by appending a "shebang" expression at the start of the file: +As with most scripting languages, a file of goal code can be made directly executable by appending a "shebang" expression at the start of the file: ```sh -#!/usr/bin/env cosh +#!/usr/bin/env goal ``` When executed this way, any additional args are available via an `args []any` variable, which can be passed to a command as follows: @@ -104,10 +104,10 @@ or by referring to specific arg indexes etc. To make a script behave like a standard Makefile, you can define different `command`s for each of the make commands, and then add the following at the end of the file to use the args to run commands: ```go -shell.RunCommands(args) +goal.RunCommands(args) ``` -See [make](cmd/cosh/testdata/make) for an example, in `cmd/cosh/testdata/make`, which can be run for example using: +See [make](cmd/goal/testdata/make) for an example, in `cmd/goal/testdata/make`, which can be run for example using: ```sh ./make build @@ -115,7 +115,7 @@ See [make](cmd/cosh/testdata/make) for an example, in `cmd/cosh/testdata/make`, Note that there is nothing special about the name `make` here, so this can be done with any file. -The `make` package defines a number of useful utility functions that accomplish the standard dependency and file timestamp checking functionality from the standard `make` command, as in the [magefile](https://magefile.org/dependencies/) system. Note that the cosh direct exec command syntax makes the resulting make files much closer to a standard bash-like Makefile, while still having all the benefits of Go control and expressions, compared to magefile. +The `make` package defines a number of useful utility functions that accomplish the standard dependency and file timestamp checking functionality from the standard `make` command, as in the [magefile](https://magefile.org/dependencies/) system. Note that the goal direct exec command syntax makes the resulting make files much closer to a standard bash-like Makefile, while still having all the benefits of Go control and expressions, compared to magefile. TODO: implement and document above. @@ -127,7 +127,7 @@ Each host maintains its own working directory and environment variables, which c * `cossh hostname.org [name]` establishes a connection, using given optional name to refer to this connection. If the name is not provided, a sequential number will be used, starting with 1, with 0 referring always to the local host. -* `@name` then refers to the given host in all subsequent commands, with `@0` referring to the local host where the cosh script is running. +* `@name` then refers to the given host in all subsequent commands, with `@0` referring to the local host where the goal script is running. ### Explicit per-command specification of host diff --git a/goal/builtins.go b/goal/builtins.go index 9251361663..7960a547a2 100644 --- a/goal/builtins.go +++ b/goal/builtins.go @@ -19,7 +19,7 @@ import ( "github.com/mitchellh/go-homedir" ) -// InstallBuiltins adds the builtin shell commands to [Shell.Builtins]. +// InstallBuiltins adds the builtin goal commands to [Goal.Builtins]. func (gl *Goal) InstallBuiltins() { gl.Builtins = make(map[string]func(cmdIO *exec.CmdIO, args ...string) error) gl.Builtins["cd"] = gl.Cd @@ -93,7 +93,7 @@ func (gl *Goal) JobsCmd(cmdIO *exec.CmdIO, args ...string) error { // Just expands the job id expressions %n into PIDs and calls system kill. func (gl *Goal) Kill(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { - return fmt.Errorf("cosh kill: expected at least one argument") + return fmt.Errorf("goal kill: expected at least one argument") } gl.JobIDExpand(args) gl.Config.RunIO(cmdIO, "kill", args...) @@ -103,12 +103,12 @@ func (gl *Goal) Kill(cmdIO *exec.CmdIO, args ...string) error { // Fg foregrounds a job by job number func (gl *Goal) Fg(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 1 { - return fmt.Errorf("cosh fg: requires exactly one job id argument") + return fmt.Errorf("goal fg: requires exactly one job id argument") } jid := args[0] exp := gl.JobIDExpand(args) if exp != 1 { - return fmt.Errorf("cosh fg: argument was not a job id in the form %%n") + return fmt.Errorf("goal fg: argument was not a job id in the form %%n") } jno, _ := strconv.Atoi(jid[1:]) // guaranteed good job := gl.Jobs[jno] @@ -125,7 +125,7 @@ func (gl *Goal) Fg(cmdIO *exec.CmdIO, args ...string) error { // AddPath adds the given path(s) to $PATH. func (gl *Goal) AddPath(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { - return fmt.Errorf("cosh add-path expected at least one argument") + return fmt.Errorf("goal add-path expected at least one argument") } path := os.Getenv("PATH") for _, arg := range args { @@ -143,7 +143,7 @@ func (gl *Goal) AddPath(cmdIO *exec.CmdIO, args ...string) error { // to exec which. func (gl *Goal) Which(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 1 { - return fmt.Errorf("cosh which: requires one argument") + return fmt.Errorf("goal which: requires one argument") } cmd := args[0] if _, hasCmd := gl.Commands[cmd]; hasCmd { @@ -151,7 +151,7 @@ func (gl *Goal) Which(cmdIO *exec.CmdIO, args ...string) error { return nil } if _, hasBlt := gl.Builtins[cmd]; hasBlt { - cmdIO.Println(cmd, "is a cosh builtin command") + cmdIO.Println(cmd, "is a goal builtin command") return nil } gl.Config.RunIO(cmdIO, "which", args...) @@ -161,7 +161,7 @@ func (gl *Goal) Which(cmdIO *exec.CmdIO, args ...string) error { // Source loads and evaluates the given file(s) func (gl *Goal) Source(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { - return fmt.Errorf("cosh source: requires at least one argument") + return fmt.Errorf("goal source: requires at least one argument") } for _, fn := range args { gl.TranspileCodeFromFile(fn) @@ -214,7 +214,7 @@ func (gl *Goal) CoSSH(cmdIO *exec.CmdIO, args ...string) error { gl.SSHActive = name cl := gl.ActiveSSH() if cl == nil { - err = fmt.Errorf("cosh: ssh connection named: %q not found", name) + err = fmt.Errorf("goal: ssh connection named: %q not found", name) } } } diff --git a/goal/cmd/goal/goal.go b/goal/cmd/goal/goal.go index 9ef4b3cd2d..ebfa689e1c 100644 --- a/goal/cmd/goal/goal.go +++ b/goal/cmd/goal/goal.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Command cosh is an interactive cli for running and compiling Cogent Shell (cosh). +// Command goal is an interactive cli for running and compiling Goal code. package main import ( @@ -14,14 +14,14 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" "cogentcore.org/core/cli" - "cogentcore.org/core/shell" - "cogentcore.org/core/shell/interpreter" + "cogentcore.org/core/goal" + "cogentcore.org/core/goal/interpreter" "github.com/cogentcore/yaegi/interp" ) //go:generate core generate -add-types -add-funcs -// Config is the configuration information for the cosh cli. +// Config is the configuration information for the goal cli. type Config struct { // Input is the input file to run/compile. @@ -37,7 +37,7 @@ type Config struct { Expr string `flag:"e,expr"` // Args is an optional list of arguments to pass in the run command. - // These arguments will be turned into an "args" local variable in the shell. + // These arguments will be turned into an "args" local variable in the goal. // These are automatically processed from any leftover arguments passed, so // you should not need to specify this flag manually. Args []string `cmd:"run" posarg:"leftover" required:"-"` @@ -49,17 +49,17 @@ type Config struct { } func main() { //types:skip - opts := cli.DefaultOptions("cosh", "An interactive tool for running and compiling Cogent Shell (cosh).") + opts := cli.DefaultOptions("goal", "An interactive tool for running and compiling Goal (Go augmented language).") cli.Run(opts, &Config{}, Run, Build) } -// Run runs the specified cosh file. If no file is specified, -// it runs an interactive shell that allows the user to input cosh. +// Run runs the specified goal file. If no file is specified, +// it runs an interactive shell that allows the user to input goal. func Run(c *Config) error { //cli:cmd -root in := interpreter.NewInterpreter(interp.Options{}) in.Config() if len(c.Args) > 0 { - in.Eval("args := cosh.StringsToAnys(" + fmt.Sprintf("%#v)", c.Args)) + in.Eval("args := goalib.StringsToAnys(" + fmt.Sprintf("%#v)", c.Args)) } if c.Input == "" { @@ -82,7 +82,7 @@ func Run(c *Config) error { //cli:cmd -root _, _, err := in.Eval(code) if err == nil { - err = in.Shell.DepthError() + err = in.Goal.DepthError() } if c.Interactive { return Interactive(c, in) @@ -90,7 +90,7 @@ func Run(c *Config) error { //cli:cmd -root return err } -// Interactive runs an interactive shell that allows the user to input cosh. +// Interactive runs an interactive shell that allows the user to input goal. func Interactive(c *Config, in *interpreter.Interpreter) error { if c.Expr != "" { in.Eval(c.Expr) @@ -99,7 +99,7 @@ func Interactive(c *Config, in *interpreter.Interpreter) error { return nil } -// Build builds the specified input cosh file, or all .cosh files in the current +// Build builds the specified input goal file, or all .goal files in the current // directory if no input is specified, to corresponding .go file name(s). // If the file does not already contain a "package" specification, then // "package main; func main()..." wrappers are added, which allows the same @@ -109,12 +109,12 @@ func Build(c *Config) error { if c.Input != "" { fns = []string{c.Input} } else { - fns = fsx.Filenames(".", ".cosh") + fns = fsx.Filenames(".", ".goal") } var errs []error for _, fn := range fns { ofn := strings.TrimSuffix(fn, filepath.Ext(fn)) + ".go" - err := shell.NewShell().TranspileFile(fn, ofn) + err := goal.NewGoal().TranspileFile(fn, ofn) if err != nil { errs = append(errs, err) } diff --git a/goal/cmd/goal/test.goal b/goal/cmd/goal/test.goal index dda7a55b21..e562da586b 100644 --- a/goal/cmd/goal/test.goal +++ b/goal/cmd/goal/test.goal @@ -1,8 +1,11 @@ -// test file for cosh cli +// test file for goal cli // todo: doesn't work: #1152 -echo {args} +// echo {args} for i, fn := range cosh.SplitLines(`/bin/ls -1`) { fmt.Println(i, fn) } + +$ x := 1 + diff --git a/goal/cmd/goal/typegen.go b/goal/cmd/goal/typegen.go index 9dcf995c36..966f3334df 100644 --- a/goal/cmd/goal/typegen.go +++ b/goal/cmd/goal/typegen.go @@ -6,10 +6,10 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "main.Config", IDName: "config", Doc: "Config is the configuration information for the cosh cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "Input", Doc: "Input is the input file to run/compile.\nIf this is provided as the first argument,\nthen the program will exit after running,\nunless the Interactive mode is flagged."}, {Name: "Expr", Doc: "Expr is an optional expression to evaluate, which can be used\nin addition to the Input file to run, to execute commands\ndefined within that file for example, or as a command to run\nprior to starting interactive mode if no Input is specified."}, {Name: "Args", Doc: "Args is an optional list of arguments to pass in the run command.\nThese arguments will be turned into an \"args\" local variable in the shell.\nThese are automatically processed from any leftover arguments passed, so\nyou should not need to specify this flag manually."}, {Name: "Interactive", Doc: "Interactive runs the interactive command line after processing any input file.\nInteractive mode is the default mode for the run command unless an input file\nis specified."}}}) +var _ = types.AddType(&types.Type{Name: "main.Config", IDName: "config", Doc: "Config is the configuration information for the goal cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "Input", Doc: "Input is the input file to run/compile.\nIf this is provided as the first argument,\nthen the program will exit after running,\nunless the Interactive mode is flagged."}, {Name: "Expr", Doc: "Expr is an optional expression to evaluate, which can be used\nin addition to the Input file to run, to execute commands\ndefined within that file for example, or as a command to run\nprior to starting interactive mode if no Input is specified."}, {Name: "Args", Doc: "Args is an optional list of arguments to pass in the run command.\nThese arguments will be turned into an \"args\" local variable in the goal.\nThese are automatically processed from any leftover arguments passed, so\nyou should not need to specify this flag manually."}, {Name: "Interactive", Doc: "Interactive runs the interactive command line after processing any input file.\nInteractive mode is the default mode for the run command unless an input file\nis specified."}}}) -var _ = types.AddFunc(&types.Func{Name: "main.Run", Doc: "Run runs the specified cosh file. If no file is specified,\nit runs an interactive shell that allows the user to input cosh.", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}}, Args: []string{"c"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "main.Run", Doc: "Run runs the specified goal file. If no file is specified,\nit runs an interactive shell that allows the user to input goal.", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}}, Args: []string{"c"}, Returns: []string{"error"}}) -var _ = types.AddFunc(&types.Func{Name: "main.Interactive", Doc: "Interactive runs an interactive shell that allows the user to input cosh.", Args: []string{"c", "in"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "main.Interactive", Doc: "Interactive runs an interactive shell that allows the user to input goal.", Args: []string{"c", "in"}, Returns: []string{"error"}}) -var _ = types.AddFunc(&types.Func{Name: "main.Build", Doc: "Build builds the specified input cosh file, or all .cosh files in the current\ndirectory if no input is specified, to corresponding .go file name(s).\nIf the file does not already contain a \"package\" specification, then\n\"package main; func main()...\" wrappers are added, which allows the same\ncode to be used in interactive and Go compiled modes.", Args: []string{"c"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "main.Build", Doc: "Build builds the specified input goal file, or all .goal files in the current\ndirectory if no input is specified, to corresponding .go file name(s).\nIf the file does not already contain a \"package\" specification, then\n\"package main; func main()...\" wrappers are added, which allows the same\ncode to be used in interactive and Go compiled modes.", Args: []string{"c"}, Returns: []string{"error"}}) diff --git a/goal/exec.go b/goal/exec.go index a8006d909d..0b746cc225 100644 --- a/goal/exec.go +++ b/goal/exec.go @@ -21,7 +21,7 @@ import ( // Exec handles command execution for all cases, parameterized by the args. // It executes the given command string, waiting for the command to finish, // handling the given arguments appropriately. -// If there is any error, it adds it to the shell, and triggers CancelExecution. +// If there is any error, it adds it to the goal, and triggers CancelExecution. // - errOk = don't call AddError so execution will not stop on error // - start = calls Start on the command, which then runs asynchronously, with // a goroutine forked to Wait for it and close its IO @@ -194,7 +194,7 @@ func (gl *Goal) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) (* newHost = hnm cl = scl } else { - gl.HandleArgErr(errOk, fmt.Errorf("cosh: ssh connection named: %q not found", hnm)) + gl.HandleArgErr(errOk, fmt.Errorf("goal: ssh connection named: %q not found", hnm)) } } if len(sargs) > 0 { @@ -260,7 +260,7 @@ func (gl *Goal) OutToFile(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, s narg = 1 } if fn == "" { - gl.HandleArgErr(errOk, fmt.Errorf("cosh: no output file specified")) + gl.HandleArgErr(errOk, fmt.Errorf("goal: no output file specified")) return sargs } if cl != nil { diff --git a/goal/execwords.go b/goal/execwords.go index 8d94db2e99..1acc8b6fc2 100644 --- a/goal/execwords.go +++ b/goal/execwords.go @@ -139,7 +139,7 @@ func ExecWords(ln string) ([]string, error) { } addWord() if dQuote || bQuote || brack > 0 { - return words, fmt.Errorf("cosh: exec command has unterminated quotes (\": %v, `: %v) or brackets [ %v ]", dQuote, bQuote, brack > 0) + return words, fmt.Errorf("goal: exec command has unterminated quotes (\": %v, `: %v) or brackets [ %v ]", dQuote, bQuote, brack > 0) } return words, nil } diff --git a/goal/goal.go b/goal/goal.go index d2d5b538ea..4af0c5cab7 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -92,7 +92,7 @@ type Goal struct { // Hist is the accumulated list of command-line input, // which is displayed with the history builtin command, - // and saved / restored from ~/.coshhist file + // and saved / restored from ~/.goalhist file Hist []string // FuncToVar translates function definitions into variable definitions, @@ -324,7 +324,7 @@ func (gl *Goal) TranspileCodeFromFile(file string) error { return nil } -// TranspileFile transpiles the given input cosh file to the +// TranspileFile transpiles the given input goal file to the // given output Go file. If no existing package declaration // is found, then package main and func main declarations are // added. This also affects how functions are interpreted. @@ -350,11 +350,11 @@ func (gl *Goal) TranspileFile(in string, out string) error { if err != nil { return err } - gen := "// Code generated by \"cosh build\"; DO NOT EDIT.\n\n" + gen := "// Code generated by \"goal build\"; DO NOT EDIT.\n\n" if hasPackage { gl.Lines = slices.Insert(gl.Lines, 0, gen) } else { - gl.Lines = slices.Insert(gl.Lines, 0, gen, "package main", "", "func main() {", "shell := shell.NewShell()") + gl.Lines = slices.Insert(gl.Lines, 0, gen, "package main", "", "func main() {", "goal := goal.NewGoal()") gl.Lines = append(gl.Lines, "}") } src := []byte(gl.Code()) @@ -371,7 +371,7 @@ func (gl *Goal) TranspileFile(in string, out string) error { // AddError adds the given error to the error stack if it is non-nil, // and calls the Cancel function if set, to stop execution. -// This is the main way that shell errors are handled. +// This is the main way that goal errors are handled. // It also prints the error. func (gl *Goal) AddError(err error) error { if err == nil { @@ -383,10 +383,10 @@ func (gl *Goal) AddError(err error) error { return err } -// TranspileConfig transpiles the .cosh startup config file in the user's +// TranspileConfig transpiles the .goal startup config file in the user's // home directory if it exists. func (gl *Goal) TranspileConfig() error { - path, err := homedir.Expand("~/.cosh") + path, err := homedir.Expand("~/.goal") if err != nil { return err } @@ -407,7 +407,7 @@ func (gl *Goal) AddHistory(line string) { } // SaveHistory saves up to the given number of lines of current history -// to given file, e.g., ~/.coshhist for the default cosh program. +// to given file, e.g., ~/.goalhist for the default goal program. // If n is <= 0 all lines are saved. n is typically 500 by default. func (gl *Goal) SaveHistory(n int, file string) error { path, err := homedir.Expand(file) @@ -428,7 +428,7 @@ func (gl *Goal) SaveHistory(n int, file string) error { } // OpenHistory opens Hist history lines from given file, -// e.g., ~/.coshhist +// e.g., ~/.goalhist func (gl *Goal) OpenHistory(file string) error { path, err := homedir.Expand(file) if err != nil { @@ -448,7 +448,7 @@ func (gl *Goal) AddCommand(name string, cmd func(args ...string)) { } // RunCommands runs the given command(s). This is typically called -// from a Makefile-style cosh script. +// from a Makefile-style goal script. func (gl *Goal) RunCommands(cmds []any) error { for _, cmd := range cmds { if cmdFun, hasCmd := gl.Commands[reflectx.ToString(cmd)]; hasCmd { @@ -485,7 +485,7 @@ func (gl *Goal) JobIDExpand(args []string) int { exp++ } } else { - gl.AddError(fmt.Errorf("cosh: job number out of range: %d", idx)) + gl.AddError(fmt.Errorf("goal: job number out of range: %d", idx)) } } } diff --git a/goal/interpreter/cogentcore_org-core-base-datasize.go b/goal/interpreter/cogentcore_org-core-base-datasize.go deleted file mode 100644 index cafb889317..0000000000 --- a/goal/interpreter/cogentcore_org-core-base-datasize.go +++ /dev/null @@ -1,29 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/base/datasize'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/base/datasize" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/base/datasize/datasize"] = map[string]reflect.Value{ - // function, constant and variable definitions - "B": reflect.ValueOf(datasize.B), - "EB": reflect.ValueOf(datasize.EB), - "ErrBits": reflect.ValueOf(&datasize.ErrBits).Elem(), - "GB": reflect.ValueOf(datasize.GB), - "KB": reflect.ValueOf(datasize.KB), - "MB": reflect.ValueOf(datasize.MB), - "MustParse": reflect.ValueOf(datasize.MustParse), - "MustParseString": reflect.ValueOf(datasize.MustParseString), - "PB": reflect.ValueOf(datasize.PB), - "Parse": reflect.ValueOf(datasize.Parse), - "ParseString": reflect.ValueOf(datasize.ParseString), - "TB": reflect.ValueOf(datasize.TB), - - // type definitions - "Size": reflect.ValueOf((*datasize.Size)(nil)), - } -} diff --git a/goal/interpreter/cogentcore_org-core-base-elide.go b/goal/interpreter/cogentcore_org-core-base-elide.go deleted file mode 100644 index c2cff7e20f..0000000000 --- a/goal/interpreter/cogentcore_org-core-base-elide.go +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/base/elide'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/base/elide" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/base/elide/elide"] = map[string]reflect.Value{ - // function, constant and variable definitions - "AppName": reflect.ValueOf(elide.AppName), - "End": reflect.ValueOf(elide.End), - "Middle": reflect.ValueOf(elide.Middle), - } -} diff --git a/goal/interpreter/cogentcore_org-core-base-errors.go b/goal/interpreter/cogentcore_org-core-base-errors.go deleted file mode 100644 index f595569dde..0000000000 --- a/goal/interpreter/cogentcore_org-core-base-errors.go +++ /dev/null @@ -1,25 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/base/errors'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/base/errors" - "github.com/cogentcore/yaegi/interp" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/base/errors/errors"] = map[string]reflect.Value{ - // function, constant and variable definitions - "As": reflect.ValueOf(errors.As), - "CallerInfo": reflect.ValueOf(errors.CallerInfo), - "ErrUnsupported": reflect.ValueOf(&errors.ErrUnsupported).Elem(), - "Is": reflect.ValueOf(errors.Is), - "Join": reflect.ValueOf(errors.Join), - "Log": reflect.ValueOf(errors.Log), - "Log1": reflect.ValueOf(interp.GenericFunc("func Log1[T any](v T, err error) T { //yaegi:add\n\tif err != nil {\n\t\tslog.Error(err.Error() + \" | \" + CallerInfo())\n\t}\n\treturn v\n}")), - "Must": reflect.ValueOf(errors.Must), - "New": reflect.ValueOf(errors.New), - "Unwrap": reflect.ValueOf(errors.Unwrap), - } -} diff --git a/goal/interpreter/cogentcore_org-core-base-fsx.go b/goal/interpreter/cogentcore_org-core-base-fsx.go deleted file mode 100644 index abc1208cb4..0000000000 --- a/goal/interpreter/cogentcore_org-core-base-fsx.go +++ /dev/null @@ -1,27 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/base/fsx'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/base/fsx" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/base/fsx/fsx"] = map[string]reflect.Value{ - // function, constant and variable definitions - "DirAndFile": reflect.ValueOf(fsx.DirAndFile), - "DirFS": reflect.ValueOf(fsx.DirFS), - "Dirs": reflect.ValueOf(fsx.Dirs), - "FileExists": reflect.ValueOf(fsx.FileExists), - "FileExistsFS": reflect.ValueOf(fsx.FileExistsFS), - "Filenames": reflect.ValueOf(fsx.Filenames), - "Files": reflect.ValueOf(fsx.Files), - "FindFilesOnPaths": reflect.ValueOf(fsx.FindFilesOnPaths), - "GoSrcDir": reflect.ValueOf(fsx.GoSrcDir), - "HasFile": reflect.ValueOf(fsx.HasFile), - "LatestMod": reflect.ValueOf(fsx.LatestMod), - "RelativeFilePath": reflect.ValueOf(fsx.RelativeFilePath), - "Sub": reflect.ValueOf(fsx.Sub), - } -} diff --git a/goal/interpreter/cogentcore_org-core-base-strcase.go b/goal/interpreter/cogentcore_org-core-base-strcase.go deleted file mode 100644 index 4c65387f68..0000000000 --- a/goal/interpreter/cogentcore_org-core-base-strcase.go +++ /dev/null @@ -1,54 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/base/strcase'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/base/strcase" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/base/strcase/strcase"] = map[string]reflect.Value{ - // function, constant and variable definitions - "CamelCase": reflect.ValueOf(strcase.CamelCase), - "CasesN": reflect.ValueOf(strcase.CasesN), - "CasesValues": reflect.ValueOf(strcase.CasesValues), - "FormatList": reflect.ValueOf(strcase.FormatList), - "KEBABCase": reflect.ValueOf(strcase.KEBABCase), - "KebabCase": reflect.ValueOf(strcase.KebabCase), - "LowerCamelCase": reflect.ValueOf(strcase.LowerCamelCase), - "LowerCase": reflect.ValueOf(strcase.LowerCase), - "Noop": reflect.ValueOf(strcase.Noop), - "SNAKECase": reflect.ValueOf(strcase.SNAKECase), - "SentenceCase": reflect.ValueOf(strcase.SentenceCase), - "Skip": reflect.ValueOf(strcase.Skip), - "SkipSplit": reflect.ValueOf(strcase.SkipSplit), - "SnakeCase": reflect.ValueOf(strcase.SnakeCase), - "Split": reflect.ValueOf(strcase.Split), - "TitleCase": reflect.ValueOf(strcase.TitleCase), - "To": reflect.ValueOf(strcase.To), - "ToCamel": reflect.ValueOf(strcase.ToCamel), - "ToKEBAB": reflect.ValueOf(strcase.ToKEBAB), - "ToKebab": reflect.ValueOf(strcase.ToKebab), - "ToLowerCamel": reflect.ValueOf(strcase.ToLowerCamel), - "ToSNAKE": reflect.ValueOf(strcase.ToSNAKE), - "ToSentence": reflect.ValueOf(strcase.ToSentence), - "ToSnake": reflect.ValueOf(strcase.ToSnake), - "ToTitle": reflect.ValueOf(strcase.ToTitle), - "ToWordCase": reflect.ValueOf(strcase.ToWordCase), - "UpperCase": reflect.ValueOf(strcase.UpperCase), - "WordCamelCase": reflect.ValueOf(strcase.WordCamelCase), - "WordCasesN": reflect.ValueOf(strcase.WordCasesN), - "WordCasesValues": reflect.ValueOf(strcase.WordCasesValues), - "WordLowerCase": reflect.ValueOf(strcase.WordLowerCase), - "WordOriginal": reflect.ValueOf(strcase.WordOriginal), - "WordSentenceCase": reflect.ValueOf(strcase.WordSentenceCase), - "WordTitleCase": reflect.ValueOf(strcase.WordTitleCase), - "WordUpperCase": reflect.ValueOf(strcase.WordUpperCase), - - // type definitions - "Cases": reflect.ValueOf((*strcase.Cases)(nil)), - "SplitAction": reflect.ValueOf((*strcase.SplitAction)(nil)), - "WordCases": reflect.ValueOf((*strcase.WordCases)(nil)), - } -} diff --git a/goal/interpreter/cogentcore_org-core-base-stringsx.go b/goal/interpreter/cogentcore_org-core-base-stringsx.go deleted file mode 100644 index f4f77cf0e5..0000000000 --- a/goal/interpreter/cogentcore_org-core-base-stringsx.go +++ /dev/null @@ -1,19 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/base/stringsx'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/base/stringsx" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/base/stringsx/stringsx"] = map[string]reflect.Value{ - // function, constant and variable definitions - "ByteSplitLines": reflect.ValueOf(stringsx.ByteSplitLines), - "ByteTrimCR": reflect.ValueOf(stringsx.ByteTrimCR), - "InsertFirstUnique": reflect.ValueOf(stringsx.InsertFirstUnique), - "SplitLines": reflect.ValueOf(stringsx.SplitLines), - "TrimCR": reflect.ValueOf(stringsx.TrimCR), - } -} diff --git a/goal/interpreter/cogentcore_org-core-shell-cosh.go b/goal/interpreter/cogentcore_org-core-shell-cosh.go deleted file mode 100644 index 68bfce6a75..0000000000 --- a/goal/interpreter/cogentcore_org-core-shell-cosh.go +++ /dev/null @@ -1,21 +0,0 @@ -// Code generated by 'yaegi extract cogentcore.org/core/shell/cosh'. DO NOT EDIT. - -package interpreter - -import ( - "cogentcore.org/core/shell/cosh" - "reflect" -) - -func init() { - Symbols["cogentcore.org/core/shell/cosh/cosh"] = map[string]reflect.Value{ - // function, constant and variable definitions - "AllFiles": reflect.ValueOf(cosh.AllFiles), - "FileExists": reflect.ValueOf(cosh.FileExists), - "ReadFile": reflect.ValueOf(cosh.ReadFile), - "ReplaceInFile": reflect.ValueOf(cosh.ReplaceInFile), - "SplitLines": reflect.ValueOf(cosh.SplitLines), - "StringsToAnys": reflect.ValueOf(cosh.StringsToAnys), - "WriteFile": reflect.ValueOf(cosh.WriteFile), - } -} diff --git a/goal/interpreter/imports.go b/goal/interpreter/imports.go index 27c79398fd..a6f7071a12 100644 --- a/goal/interpreter/imports.go +++ b/goal/interpreter/imports.go @@ -9,22 +9,23 @@ package interpreter import ( "reflect" + _ "cogentcore.org/core/yaegicore/symbols" "github.com/cogentcore/yaegi/interp" ) var Symbols = map[string]map[string]reflect.Value{} -// ImportShell imports special symbols from the shell package. -func (in *Interpreter) ImportShell() { +// ImportGoal imports special symbols from the goal package. +func (in *Interpreter) ImportGoal() { in.Interp.Use(interp.Exports{ - "cogentcore.org/core/shell/shell": map[string]reflect.Value{ - "Run": reflect.ValueOf(in.Shell.Run), - "RunErrOK": reflect.ValueOf(in.Shell.RunErrOK), - "Output": reflect.ValueOf(in.Shell.Output), - "OutputErrOK": reflect.ValueOf(in.Shell.OutputErrOK), - "Start": reflect.ValueOf(in.Shell.Start), - "AddCommand": reflect.ValueOf(in.Shell.AddCommand), - "RunCommands": reflect.ValueOf(in.Shell.RunCommands), + "cogentcore.org/core/goal/goal": map[string]reflect.Value{ + "Run": reflect.ValueOf(in.Goal.Run), + "RunErrOK": reflect.ValueOf(in.Goal.RunErrOK), + "Output": reflect.ValueOf(in.Goal.Output), + "OutputErrOK": reflect.ValueOf(in.Goal.OutputErrOK), + "Start": reflect.ValueOf(in.Goal.Start), + "AddCommand": reflect.ValueOf(in.Goal.AddCommand), + "RunCommands": reflect.ValueOf(in.Goal.RunCommands), }, }) } diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index a70c5079f5..3374f1956b 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -17,7 +17,7 @@ import ( "syscall" "cogentcore.org/core/base/errors" - "cogentcore.org/core/shell" + "cogentcore.org/core/goal" "github.com/cogentcore/yaegi/interp" "github.com/cogentcore/yaegi/stdlib" "github.com/ergochat/readline" @@ -25,11 +25,11 @@ import ( // Interpreter represents one running shell context type Interpreter struct { - // the cosh shell - Shell *shell.Shell + // the goal shell + Goal *goal.Goal // HistFile is the name of the history file to open / save. - // Defaults to ~/.cosh-history for the default cosh shell. + // Defaults to ~/.goal-history for the default goal shell. // Update this prior to running Config() to take effect. HistFile string @@ -46,34 +46,34 @@ func init() { // functions. End user app must call [Interp.Config] after importing any additional // symbols, prior to running the interpreter. func NewInterpreter(options interp.Options) *Interpreter { - in := &Interpreter{HistFile: "~/.cosh-history"} - in.Shell = shell.NewShell() + in := &Interpreter{HistFile: "~/.goal-history"} + in.Goal = goal.NewGoal() if options.Stdin != nil { - in.Shell.Config.StdIO.In = options.Stdin + in.Goal.Config.StdIO.In = options.Stdin } if options.Stdout != nil { - in.Shell.Config.StdIO.Out = options.Stdout + in.Goal.Config.StdIO.Out = options.Stdout } if options.Stderr != nil { - in.Shell.Config.StdIO.Err = options.Stderr + in.Goal.Config.StdIO.Err = options.Stderr } - in.Shell.SaveOrigStdIO() - options.Stdout = in.Shell.StdIOWrappers.Out - options.Stderr = in.Shell.StdIOWrappers.Err - options.Stdin = in.Shell.StdIOWrappers.In + in.Goal.SaveOrigStdIO() + options.Stdout = in.Goal.StdIOWrappers.Out + options.Stderr = in.Goal.StdIOWrappers.Err + options.Stdin = in.Goal.StdIOWrappers.In in.Interp = interp.New(options) errors.Log(in.Interp.Use(stdlib.Symbols)) errors.Log(in.Interp.Use(Symbols)) - in.ImportShell() + in.ImportGoal() go in.MonitorSignals() return in } // Prompt returns the appropriate REPL prompt to show the user. func (in *Interpreter) Prompt() string { - dp := in.Shell.TotalDepth() + dp := in.Goal.TotalDepth() if dp == 0 { - return in.Shell.HostAndDir() + " > " + return in.Goal.HostAndDir() + " > " } res := "> " for range dp { @@ -89,21 +89,21 @@ func (in *Interpreter) Prompt() string { // whether to print the result in interactive mode. // It automatically logs any error in addition to returning it. func (in *Interpreter) Eval(code string) (v reflect.Value, hasPrint bool, err error) { - in.Shell.TranspileCode(code) + in.Goal.TranspileCode(code) source := false - if in.Shell.SSHActive == "" { + if in.Goal.SSHActive == "" { source = strings.HasPrefix(code, "source") } - if in.Shell.TotalDepth() == 0 { - nl := len(in.Shell.Lines) + if in.Goal.TotalDepth() == 0 { + nl := len(in.Goal.Lines) if nl > 0 { - ln := in.Shell.Lines[nl-1] + ln := in.Goal.Lines[nl-1] if strings.Contains(strings.ToLower(ln), "print") { hasPrint = true } } v, err = in.RunCode() - in.Shell.Errors = nil + in.Goal.Errors = nil } if source { v, err = in.RunCode() // run accumulated code @@ -115,27 +115,27 @@ func (in *Interpreter) Eval(code string) (v reflect.Value, hasPrint bool, err er // and clears the stack of code lines. // It automatically logs any error in addition to returning it. func (in *Interpreter) RunCode() (reflect.Value, error) { - if len(in.Shell.Errors) > 0 { - return reflect.Value{}, errors.Join(in.Shell.Errors...) + if len(in.Goal.Errors) > 0 { + return reflect.Value{}, errors.Join(in.Goal.Errors...) } - in.Shell.AddChunk() - code := in.Shell.Chunks - in.Shell.ResetCode() + in.Goal.AddChunk() + code := in.Goal.Chunks + in.Goal.ResetCode() var v reflect.Value var err error for _, ch := range code { - ctx := in.Shell.StartContext() + ctx := in.Goal.StartContext() v, err = in.Interp.EvalWithContext(ctx, ch) - in.Shell.EndContext() + in.Goal.EndContext() if err != nil { cancelled := errors.Is(err, context.Canceled) // fmt.Println("cancelled:", cancelled) - in.Shell.RestoreOrigStdIO() - in.Shell.ResetDepth() + in.Goal.RestoreOrigStdIO() + in.Goal.ResetDepth() if !cancelled { - in.Shell.AddError(err) + in.Goal.AddError(err) } else { - in.Shell.Errors = nil + in.Goal.Errors = nil } break } @@ -143,10 +143,10 @@ func (in *Interpreter) RunCode() (reflect.Value, error) { return v, err } -// RunConfig runs the .cosh startup config file in the user's +// RunConfig runs the .goal startup config file in the user's // home directory if it exists. func (in *Interpreter) RunConfig() error { - err := in.Shell.TranspileConfig() + err := in.Goal.TranspileConfig() if err != nil { errors.Log(err) } @@ -162,7 +162,7 @@ func (in *Interpreter) MonitorSignals() { signal.Notify(c, os.Interrupt, syscall.SIGTERM) for { <-c - in.Shell.CancelExecution() + in.Goal.CancelExecution() } } @@ -175,9 +175,9 @@ func (in *Interpreter) Config() { // OpenHistory opens history from the current HistFile // and loads it into the readline history for given rl instance func (in *Interpreter) OpenHistory(rl *readline.Instance) error { - err := in.Shell.OpenHistory(in.HistFile) + err := in.Goal.OpenHistory(in.HistFile) if err == nil { - for _, h := range in.Shell.Hist { + for _, h := range in.Goal.Hist { rl.SaveToHistory(h) } } @@ -191,19 +191,19 @@ func (in *Interpreter) SaveHistory() error { if hfs := os.Getenv("HISTFILESIZE"); hfs != "" { en, err := strconv.Atoi(hfs) if err != nil { - in.Shell.Config.StdIO.ErrPrintf("SaveHistory: environment variable HISTFILESIZE: %q not a number: %s", hfs, err.Error()) + in.Goal.Config.StdIO.ErrPrintf("SaveHistory: environment variable HISTFILESIZE: %q not a number: %s", hfs, err.Error()) } else { n = en } } - return in.Shell.SaveHistory(n, in.HistFile) + return in.Goal.SaveHistory(n, in.HistFile) } -// Interactive runs an interactive shell that allows the user to input cosh. +// Interactive runs an interactive shell that allows the user to input goal. // Must have done in.Config() prior to calling. func (in *Interpreter) Interactive() error { rl, err := readline.NewFromConfig(&readline.Config{ - AutoComplete: &shell.ReadlineCompleter{Shell: in.Shell}, + AutoComplete: &goal.ReadlineCompleter{Goal: in.Goal}, Undo: true, }) if err != nil { @@ -229,21 +229,21 @@ func (in *Interpreter) Interactive() error { } if len(line) > 0 && line[0] == '!' { // history command hl, err := strconv.Atoi(line[1:]) - nh := len(in.Shell.Hist) + nh := len(in.Goal.Hist) if err != nil { - in.Shell.Config.StdIO.ErrPrintf("history number: %q not a number: %s", line[1:], err.Error()) + in.Goal.Config.StdIO.ErrPrintf("history number: %q not a number: %s", line[1:], err.Error()) line = "" } else if hl >= nh { - in.Shell.Config.StdIO.ErrPrintf("history number: %d not in range: [0:%d]", hl, nh) + in.Goal.Config.StdIO.ErrPrintf("history number: %d not in range: [0:%d]", hl, nh) line = "" } else { - line = in.Shell.Hist[hl] + line = in.Goal.Hist[hl] fmt.Printf("h:%d\t%s\n", hl, line) } } else if line != "" && !strings.HasPrefix(line, "history") && line != "h" { - in.Shell.AddHistory(line) + in.Goal.AddHistory(line) } - in.Shell.Errors = nil + in.Goal.Errors = nil v, hasPrint, err := in.Eval(line) if err == nil && !hasPrint && v.IsValid() && !v.IsZero() && v.Kind() != reflect.Func { fmt.Println(v.Interface()) diff --git a/goal/interpreter/make b/goal/interpreter/make deleted file mode 100755 index da045d6d28..0000000000 --- a/goal/interpreter/make +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env cosh -// add standard imports here; mostly base - -command base { - println("extracting base packages") - yaegi extract cogentcore.org/core/base/fsx cogentcore.org/core/base/errors cogentcore.org/core/base/strcase cogentcore.org/core/base/elide cogentcore.org/core/base/stringsx cogentcore.org/core/base/datasize -} - -command cosh { - println("extracting cosh packages") - yaegi extract cogentcore.org/core/shell/cosh -} - -// shell.RunCommands(args) -base -cosh - diff --git a/goal/math.go b/goal/math.go index 0c762c4123..ee96a0c272 100644 --- a/goal/math.go +++ b/goal/math.go @@ -4,10 +4,137 @@ package goal -import "fmt" +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" +) -func (gl *Goal) TranspileMath(toks Tokens) Tokens { - fmt.Printf("in math") +type mathParse struct { + toks Tokens // output tokens + lhs string + rhs string // or the only one + curToks Tokens // current source tokens we are parsing + curIdx int // current index in source tokens + lhsToks Tokens + rhsToks Tokens +} + +func (mp *mathParse) addCur() { + mp.toks.AddTokens(mp.curToks[mp.curIdx]) + mp.curIdx++ +} + +func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { + // fmt.Println("in math") + nt := len(toks) + + mp := mathParse{} + + // expr can't do statements, so we need to find those ourselves + assignIdx := -1 + for i, tk := range toks { + if tk.Tok == token.ASSIGN || tk.Tok == token.DEFINE { + assignIdx = i + break + } + } + if assignIdx >= 0 { + mp.lhsToks = toks[0:assignIdx] + mp.lhs = ln[toks[0].Pos-1 : toks[assignIdx].Pos-1] + fmt.Println("lhs:", mp.lhs) + mp.rhsToks = toks[assignIdx+1 : nt-1] + mp.rhs = ln[toks[assignIdx+1].Pos-1 : toks[nt-1].Pos] + fmt.Println("rhs:", mp.rhs) + lex, err := parser.ParseExpr(mp.lhs) + if err != nil { + fmt.Println("lhs parse err:", err) + } + rex, err := parser.ParseExpr(mp.rhs) + if err != nil { + fmt.Println("rhs parse err:", err) + } + mp.assignStmt(toks[assignIdx], lex, rex) + } else { + mp.rhsToks = toks[0 : nt-1] + mp.curToks = mp.rhsToks + mp.rhs = ln[toks[0].Pos-1 : toks[nt-1].Pos] + ex, err := parser.ParseExpr(mp.rhs) + if err != nil { + fmt.Println("expr parse err:", err) + } + fmt.Println("expr:", mp.rhs) + mp.expr(ex) + } + + return mp.toks +} + +func (mp *mathParse) assignStmt(tok *Token, lex, rex ast.Expr) { + mp.curToks = mp.lhsToks + mp.expr(lex) + mp.toks.AddTokens(tok) + mp.curToks = mp.rhsToks + mp.curIdx = 0 + mp.expr(rex) +} + +func (mp *mathParse) expr(ex ast.Expr) { + switch x := ex.(type) { + case *ast.BadExpr: + fmt.Println("bad!") + + case *ast.Ident: + // fmt.Println("ident:", x.Name) + mp.addCur() + + case *ast.BinaryExpr: + mp.binaryExpr(x) + + case *ast.BasicLit: + mp.basicLit(x) + + case *ast.FuncLit: + + case *ast.ParenExpr: + + case *ast.SelectorExpr: + mp.selectorExpr(x) + + case *ast.TypeAssertExpr: + + case *ast.IndexListExpr: + fmt.Println("index!") + + case *ast.SliceExpr: + mp.sliceExpr(x) + // todo: we'll need to work on this! + + case *ast.CallExpr: + + } +} + +func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { +} + +func (mp *mathParse) basicLit(lit *ast.BasicLit) { + switch lit.Kind { + case token.INT: + mp.toks.Add(token.IDENT, "tensor.NewIntScalar("+lit.Value+")") + mp.curIdx++ + case token.FLOAT: + mp.toks.Add(token.IDENT, "tensor.NewFloatScalar("+lit.Value+")") + mp.curIdx++ + case token.STRING: + mp.toks.Add(token.IDENT, "tensor.NewStringScalar("+lit.Value+")") + mp.curIdx++ + } +} + +func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { +} - return toks +func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { } diff --git a/goal/run.go b/goal/run.go index 1725f30eca..465e784310 100644 --- a/goal/run.go +++ b/goal/run.go @@ -6,7 +6,7 @@ package goal // Run executes the given command string, waiting for the command to finish, // handling the given arguments appropriately. -// If there is any error, it adds it to the shell, and triggers CancelExecution. +// If there is any error, it adds it to the goal, and triggers CancelExecution. // It forwards output to [exec.Config.Stdout] and [exec.Config.Stderr] appropriately. func (gl *Goal) Run(cmd any, args ...any) { gl.Exec(false, false, false, cmd, args...) @@ -15,7 +15,7 @@ func (gl *Goal) Run(cmd any, args ...any) { // RunErrOK executes the given command string, waiting for the command to finish, // handling the given arguments appropriately. // It does not stop execution if there is an error. -// If there is any error, it adds it to the shell. It forwards output to +// If there is any error, it adds it to the goal. It forwards output to // [exec.Config.Stdout] and [exec.Config.Stderr] appropriately. func (gl *Goal) RunErrOK(cmd any, args ...any) { gl.Exec(true, false, false, cmd, args...) @@ -23,21 +23,21 @@ func (gl *Goal) RunErrOK(cmd any, args ...any) { // Start starts the given command string for running in the background, // handling the given arguments appropriately. -// If there is any error, it adds it to the shell. It forwards output to +// If there is any error, it adds it to the goal. It forwards output to // [exec.Config.Stdout] and [exec.Config.Stderr] appropriately. func (gl *Goal) Start(cmd any, args ...any) { gl.Exec(false, true, false, cmd, args...) } // Output executes the given command string, handling the given arguments -// appropriately. If there is any error, it adds it to the shell. It returns +// appropriately. If there is any error, it adds it to the goal. It returns // the stdout as a string and forwards stderr to [exec.Config.Stderr] appropriately. func (gl *Goal) Output(cmd any, args ...any) string { return gl.Exec(false, false, true, cmd, args...) } // OutputErrOK executes the given command string, handling the given arguments -// appropriately. If there is any error, it adds it to the shell. It returns +// appropriately. If there is any error, it adds it to the goal. It returns // the stdout as a string and forwards stderr to [exec.Config.Stderr] appropriately. func (gl *Goal) OutputErrOK(cmd any, args ...any) string { return gl.Exec(true, false, true, cmd, args...) diff --git a/goal/token.go b/goal/token.go index e3adb6f39e..bf82fba465 100644 --- a/goal/token.go +++ b/goal/token.go @@ -48,7 +48,7 @@ func (tk *Tokens) Add(tok token.Token, str ...string) *Token { } // AddTokens adds given tokens to our list -func (tk *Tokens) AddTokens(toks Tokens) *Tokens { +func (tk *Tokens) AddTokens(toks ...*Token) *Tokens { *tk = append(*tk, toks...) return tk } diff --git a/goal/transpile.go b/goal/transpile.go index 642de5b93c..b2c89b15ff 100644 --- a/goal/transpile.go +++ b/goal/transpile.go @@ -12,7 +12,7 @@ import ( "cogentcore.org/core/base/logx" ) -// TranspileLine is the main function for parsing a single line of shell input, +// TranspileLine is the main function for parsing a single line of goal input, // returning a new transpiled line of code that converts Exec code into corresponding // Go function calls. func (gl *Goal) TranspileLine(ln string) string { @@ -76,7 +76,7 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { case t0.Tok == token.ILLEGAL: fmt.Println(t0.Str) if t0.Str != "" && t0.Str[:1] == "$" { - return gl.TranspileMath(toks[1:]) + return gl.TranspileMath(toks[1:], ln) } case t0.Tok == token.LBRACE: logx.PrintlnDebug("go: { } line") @@ -91,13 +91,13 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { logx.PrintlnDebug("exec: backquoted string") exe := gl.TranspileExecString(t0.Str, false) if n > 1 { // todo: is this an error? - exe.AddTokens(gl.TranspileGo(toks[1:])) + exe.AddTokens(gl.TranspileGo(toks[1:])...) } return exe case t0.Tok == token.IDENT && t0.Str == "command": gl.lastCommand = toks[1].Str // 1 is the name -- triggers AddCommand toks = toks[2:] // get rid of first - toks.Insert(0, token.IDENT, "shell.AddCommand") + toks.Insert(0, token.IDENT, "goal.AddCommand") toks.Insert(1, token.LPAREN) toks.Insert(2, token.STRING, `"`+gl.lastCommand+`"`) toks.Insert(3, token.COMMA) @@ -107,7 +107,7 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { toks.Insert(7, token.ELLIPSIS) toks.Insert(8, token.IDENT, "string") toks.Insert(9, token.RPAREN) - toks.AddTokens(gl.TranspileGo(toks[11:])) + toks.AddTokens(gl.TranspileGo(toks[11:])...) case t0.IsGo(): if t0.Tok == token.GO { if !toks.Contains(token.LPAREN) { @@ -201,7 +201,7 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { } startExec := func() { bgJob = false - etoks.Add(token.IDENT, "shell") + etoks.Add(token.IDENT, "goal") etoks.Add(token.PERIOD) switch { case output && noStop: @@ -230,10 +230,10 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { switch { case f == "{": // embedded go if n < i+3 { - gl.AddError(fmt.Errorf("cosh: no matching right brace } found in exec command line")) + gl.AddError(fmt.Errorf("goal: no matching right brace } found in exec command line")) } else { gstr := ewords[i+1] - etoks.AddTokens(gl.TranspileGo(gl.Tokens(gstr))) + etoks.AddTokens(gl.TranspileGo(gl.Tokens(gstr))...) etoks.Add(token.COMMA) i += 2 } @@ -250,12 +250,12 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { etoks.Add(token.COMMA) endExec() etoks.Add(token.SEMICOLON) - etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)) + etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)...) return etoks case f == ";": endExec() etoks.Add(token.SEMICOLON) - etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)) + etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)...) return etoks default: if f[0] == '"' || f[0] == '`' { diff --git a/goal/transpile_test.go b/goal/transpile_test.go index 45bae8364f..6fbaad971d 100644 --- a/goal/transpile_test.go +++ b/goal/transpile_test.go @@ -60,7 +60,7 @@ func TestPaths(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ {`fmt.Println("hi")`, `fmt.Println`}, - {"./cosh -i", `./cosh`}, + {"./goal -i", `./goal`}, {"main.go", `main.go`}, {"cogent/", `cogent/`}, {`./"Cogent Code"`, `./\"Cogent Code\"`}, @@ -86,78 +86,78 @@ func TestPaths(t *testing.T) { func TestTranspile(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"ls", `shell.Run("ls")`}, - {"`ls -la`", `shell.Run("ls", "-la")`}, - {"ls -la", `shell.Run("ls", "-la")`}, - {"ls --help", `shell.Run("ls", "--help")`}, - {"ls go", `shell.Run("ls", "go")`}, - {"cd go", `shell.Run("cd", "go")`}, + {"ls", `goal.Run("ls")`}, + {"`ls -la`", `goal.Run("ls", "-la")`}, + {"ls -la", `goal.Run("ls", "-la")`}, + {"ls --help", `goal.Run("ls", "--help")`}, + {"ls go", `goal.Run("ls", "go")`}, + {"cd go", `goal.Run("cd", "go")`}, {`var name string`, `var name string`}, {`name = "test"`, `name = "test"`}, - {`echo {name}`, `shell.Run("echo", name)`}, - {`echo "testing"`, `shell.Run("echo", "testing")`}, + {`echo {name}`, `goal.Run("echo", name)`}, + {`echo "testing"`, `goal.Run("echo", "testing")`}, {`number := 1.23`, `number := 1.23`}, {`res1, res2 := FunTwoRet()`, `res1, res2 := FunTwoRet()`}, {`res1, res2, res3 := FunThreeRet()`, `res1, res2, res3 := FunThreeRet()`}, {`println("hi")`, `println("hi")`}, {`fmt.Println("hi")`, `fmt.Println("hi")`}, {`for i := 0; i < 3; i++ { fmt.Println(i, "\n")`, `for i := 0; i < 3; i++ { fmt.Println(i, "\n")`}, - {"for i, v := range `ls -la` {", `for i, v := range shell.Output("ls", "-la") {`}, + {"for i, v := range `ls -la` {", `for i, v := range goal.Output("ls", "-la") {`}, {`// todo: fixit`, `// todo: fixit`}, - {"`go build`", `shell.Run("go", "build")`}, + {"`go build`", `goal.Run("go", "build")`}, {"{go build()}", `go build()`}, - {"go build", `shell.Run("go", "build")`}, + {"go build", `goal.Run("go", "build")`}, {"go build()", `go build()`}, - {"go build &", `shell.Start("go", "build")`}, - {"[mkdir subdir]", `shell.RunErrOK("mkdir", "subdir")`}, - {"set something hello-1", `shell.Run("set", "something", "hello-1")`}, - {"set something = hello", `shell.Run("set", "something", "=", "hello")`}, - {`set something = "hello"`, `shell.Run("set", "something", "=", "hello")`}, - {`set something=hello`, `shell.Run("set", "something=hello")`}, - {`set "something=hello"`, `shell.Run("set", "something=hello")`}, - {`set something="hello"`, `shell.Run("set", "something=\"hello\"")`}, - {`add-path /opt/sbin /opt/homebrew/bin`, `shell.Run("add-path", "/opt/sbin", "/opt/homebrew/bin")`}, - {`cat file > test.out`, `shell.Run("cat", "file", ">", "test.out")`}, - {`cat file | grep -v exe > test.out`, `shell.Start("cat", "file", "|"); shell.Run("grep", "-v", "exe", ">", "test.out")`}, - {`cd sub; pwd; ls -la`, `shell.Run("cd", "sub"); shell.Run("pwd"); shell.Run("ls", "-la")`}, - {`cd sub; [mkdir sub]; ls -la`, `shell.Run("cd", "sub"); shell.RunErrOK("mkdir", "sub"); shell.Run("ls", "-la")`}, - {`cd sub; mkdir names[4]`, `shell.Run("cd", "sub"); shell.Run("mkdir", "names[4]")`}, - {"ls -la > test.out", `shell.Run("ls", "-la", ">", "test.out")`}, - {"ls > test.out", `shell.Run("ls", ">", "test.out")`}, - {"ls -la >test.out", `shell.Run("ls", "-la", ">", "test.out")`}, - {"ls -la >> test.out", `shell.Run("ls", "-la", ">>", "test.out")`}, - {"ls -la >& test.out", `shell.Run("ls", "-la", ">&", "test.out")`}, - {"ls -la >>& test.out", `shell.Run("ls", "-la", ">>&", "test.out")`}, - {"@1 ls -la", `shell.Run("@1", "ls", "-la")`}, - {"git switch main", `shell.Run("git", "switch", "main")`}, - {"git checkout 123abc", `shell.Run("git", "checkout", "123abc")`}, - {"go get cogentcore.org/core@main", `shell.Run("go", "get", "cogentcore.org/core@main")`}, - {"ls *.go", `shell.Run("ls", "*.go")`}, - {"ls ??.go", `shell.Run("ls", "??.go")`}, + {"go build &", `goal.Start("go", "build")`}, + {"[mkdir subdir]", `goal.RunErrOK("mkdir", "subdir")`}, + {"set something hello-1", `goal.Run("set", "something", "hello-1")`}, + {"set something = hello", `goal.Run("set", "something", "=", "hello")`}, + {`set something = "hello"`, `goal.Run("set", "something", "=", "hello")`}, + {`set something=hello`, `goal.Run("set", "something=hello")`}, + {`set "something=hello"`, `goal.Run("set", "something=hello")`}, + {`set something="hello"`, `goal.Run("set", "something=\"hello\"")`}, + {`add-path /opt/sbin /opt/homebrew/bin`, `goal.Run("add-path", "/opt/sbin", "/opt/homebrew/bin")`}, + {`cat file > test.out`, `goal.Run("cat", "file", ">", "test.out")`}, + {`cat file | grep -v exe > test.out`, `goal.Start("cat", "file", "|"); goal.Run("grep", "-v", "exe", ">", "test.out")`}, + {`cd sub; pwd; ls -la`, `goal.Run("cd", "sub"); goal.Run("pwd"); goal.Run("ls", "-la")`}, + {`cd sub; [mkdir sub]; ls -la`, `goal.Run("cd", "sub"); goal.RunErrOK("mkdir", "sub"); goal.Run("ls", "-la")`}, + {`cd sub; mkdir names[4]`, `goal.Run("cd", "sub"); goal.Run("mkdir", "names[4]")`}, + {"ls -la > test.out", `goal.Run("ls", "-la", ">", "test.out")`}, + {"ls > test.out", `goal.Run("ls", ">", "test.out")`}, + {"ls -la >test.out", `goal.Run("ls", "-la", ">", "test.out")`}, + {"ls -la >> test.out", `goal.Run("ls", "-la", ">>", "test.out")`}, + {"ls -la >& test.out", `goal.Run("ls", "-la", ">&", "test.out")`}, + {"ls -la >>& test.out", `goal.Run("ls", "-la", ">>&", "test.out")`}, + {"@1 ls -la", `goal.Run("@1", "ls", "-la")`}, + {"git switch main", `goal.Run("git", "switch", "main")`}, + {"git checkout 123abc", `goal.Run("git", "checkout", "123abc")`}, + {"go get cogentcore.org/core@main", `goal.Run("go", "get", "cogentcore.org/core@main")`}, + {"ls *.go", `goal.Run("ls", "*.go")`}, + {"ls ??.go", `goal.Run("ls", "??.go")`}, {`fmt.Println("hi")`, `fmt.Println("hi")`}, - {"cosh -i", `shell.Run("cosh", "-i")`}, - {"./cosh -i", `shell.Run("./cosh", "-i")`}, - {"cat main.go", `shell.Run("cat", "main.go")`}, - {"cd cogent", `shell.Run("cd", "cogent")`}, - {"cd cogent/", `shell.Run("cd", "cogent/")`}, - {"echo $PATH", `shell.Run("echo", "$PATH")`}, - {`"./Cogent Code"`, `shell.Run("./Cogent Code")`}, - {`./"Cogent Code"`, `shell.Run("./\"Cogent Code\"")`}, - {`Cogent\ Code`, `shell.Run("Cogent Code")`}, - {`./Cogent\ Code`, `shell.Run("./Cogent Code")`}, - {`ios\ deploy -i`, `shell.Run("ios deploy", "-i")`}, - {"./ios-deploy -i", `shell.Run("./ios-deploy", "-i")`}, - {"ios_deploy -i tree_file", `shell.Run("ios_deploy", "-i", "tree_file")`}, - {"ios_deploy/sub -i tree_file", `shell.Run("ios_deploy/sub", "-i", "tree_file")`}, - {"C:/ios_deploy/sub -i tree_file", `shell.Run("C:/ios_deploy/sub", "-i", "tree_file")`}, - {"ios_deploy -i tree_file/path", `shell.Run("ios_deploy", "-i", "tree_file/path")`}, - {"ios-deploy -i", `shell.Run("ios-deploy", "-i")`}, - {"ios-deploy -i tree-file", `shell.Run("ios-deploy", "-i", "tree-file")`}, - {"ios-deploy -i tree-file/path/here", `shell.Run("ios-deploy", "-i", "tree-file/path/here")`}, - {"cd ..", `shell.Run("cd", "..")`}, - {"cd ../another/dir/to/go_to", `shell.Run("cd", "../another/dir/to/go_to")`}, - {"cd ../an-other/dir/", `shell.Run("cd", "../an-other/dir/")`}, - {"curl https://google.com/search?q=hello%20world#body", `shell.Run("curl", "https://google.com/search?q=hello%20world#body")`}, + {"goal -i", `goal.Run("goal", "-i")`}, + {"./goal -i", `goal.Run("./goal", "-i")`}, + {"cat main.go", `goal.Run("cat", "main.go")`}, + {"cd cogent", `goal.Run("cd", "cogent")`}, + {"cd cogent/", `goal.Run("cd", "cogent/")`}, + {"echo $PATH", `goal.Run("echo", "$PATH")`}, + {`"./Cogent Code"`, `goal.Run("./Cogent Code")`}, + {`./"Cogent Code"`, `goal.Run("./\"Cogent Code\"")`}, + {`Cogent\ Code`, `goal.Run("Cogent Code")`}, + {`./Cogent\ Code`, `goal.Run("./Cogent Code")`}, + {`ios\ deploy -i`, `goal.Run("ios deploy", "-i")`}, + {"./ios-deploy -i", `goal.Run("./ios-deploy", "-i")`}, + {"ios_deploy -i tree_file", `goal.Run("ios_deploy", "-i", "tree_file")`}, + {"ios_deploy/sub -i tree_file", `goal.Run("ios_deploy/sub", "-i", "tree_file")`}, + {"C:/ios_deploy/sub -i tree_file", `goal.Run("C:/ios_deploy/sub", "-i", "tree_file")`}, + {"ios_deploy -i tree_file/path", `goal.Run("ios_deploy", "-i", "tree_file/path")`}, + {"ios-deploy -i", `goal.Run("ios-deploy", "-i")`}, + {"ios-deploy -i tree-file", `goal.Run("ios-deploy", "-i", "tree-file")`}, + {"ios-deploy -i tree-file/path/here", `goal.Run("ios-deploy", "-i", "tree-file/path/here")`}, + {"cd ..", `goal.Run("cd", "..")`}, + {"cd ../another/dir/to/go_to", `goal.Run("cd", "../another/dir/to/go_to")`}, + {"cd ../an-other/dir/", `goal.Run("cd", "../an-other/dir/")`}, + {"curl https://google.com/search?q=hello%20world#body", `goal.Run("curl", "https://google.com/search?q=hello%20world#body")`}, {"func splitLines(str string) []string {", `splitLines := func(str string)[]string {`}, {"type Result struct {", `type Result struct {`}, {"var Jobs *table.Table", `var Jobs *table.Table`}, @@ -185,8 +185,8 @@ func TestCommand(t *testing.T) { `command list { ls -la args... }`, - `shell.AddCommand("list", func(args ...string) { -shell.Run("ls", "-la", "args...") + `goal.AddCommand("list", func(args ...string) { +goal.Run("ls", "-la", "args...") })`}, } @@ -201,7 +201,7 @@ shell.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"$x := 1", `x := tensor.NewFloat64Scalar(1)`}, + {"$x := 1", `x := tensor.NewIntScalar(1)`}, } gl := NewGoal() diff --git a/spell/dict/dtool.go b/spell/dict/dtool.go index 73d01cee20..84c66102d1 100644 --- a/spell/dict/dtool.go +++ b/spell/dict/dtool.go @@ -14,7 +14,7 @@ import ( //go:generate core generate -add-types -add-funcs -// Config is the configuration information for the cosh cli. +// Config is the configuration information for the dict cli. type Config struct { // InputA is the first input dictionary file diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 1f97e43161..cfe85e7e6c 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -31,7 +31,7 @@ func TestFuncs(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := NewIndexed(NewNumberFromSlice(vals)) + oned := NewIndexed(NewNumberFromSlice(vals...)) oneout := oned.Clone() err = Call("Abs", oned, oneout) diff --git a/tensor/indexed.go b/tensor/indexed.go index e4f8a87393..af8b1b5cc4 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -62,7 +62,19 @@ func NewFloat64Indexed(sizes ...int) *Indexed { // NewFloat64Scalar is a convenience method to quickly get an Indexed // representation of a single float64 scalar value, for use in math routines etc. func NewFloat64Scalar(val float64) *Indexed { - return &Indexed{Tensor: NewNumberFromSlice([]float64{val})} + return &Indexed{Tensor: NewNumberFromSlice(val)} +} + +// NewIntScalar is a convenience method to quickly get an Indexed +// representation of a single int scalar value, for use in math routines etc. +func NewIntScalar(val int) *Indexed { + return &Indexed{Tensor: NewNumberFromSlice(val)} +} + +// NewStringScalar is a convenience method to quickly get an Indexed +// representation of a single string scalar value, for use in math routines etc. +func NewStringScalar(val string) *Indexed { + return &Indexed{Tensor: NewStringFromSlice(val)} } // SetTensor sets as indexes into given tensor with sequential initial indexes diff --git a/tensor/number.go b/tensor/number.go index 6fa725009a..7a52fecde6 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -113,7 +113,7 @@ func NewNumberShape[T num.Number](shape *Shape) *Number[T] { // NewNumberFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewNumberFromSlice[T num.Number](vals []T) Tensor { +func NewNumberFromSlice[T num.Number](vals ...T) Tensor { n := len(vals) tsr := &Number[T]{} tsr.Values = vals diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index ea50cefe28..0ef95119cd 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -21,8 +21,8 @@ func TestFuncs(t *testing.T) { tol := 1.0e-8 - atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64)) - btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64)) + atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64...)) + btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64...)) out := tensor.NewFloat64(1) oix := tensor.NewIndexed(out) diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 3ab83217f7..421469a48b 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -61,17 +61,17 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { // MedianFunc computes the median (50% quantile) of tensor values. // See [StatsFunc] for general information. func MedianFunc(in, out *tensor.Indexed) { - QuantilesFunc(in, tensor.NewIndexed(tensor.NewNumberFromSlice([]float64{.5})), out) + QuantilesFunc(in, tensor.NewFloat64Scalar(.5), out) } // Q1Func computes the first quantile (25%) of tensor values. // See [StatsFunc] for general information. func Q1Func(in, out *tensor.Indexed) { - QuantilesFunc(in, tensor.NewIndexed(tensor.NewNumberFromSlice([]float64{.25})), out) + QuantilesFunc(in, tensor.NewFloat64Scalar(.25), out) } // Q3Func computes the third quantile (75%) of tensor values. // See [StatsFunc] for general information. func Q3Func(in, out *tensor.Indexed) { - QuantilesFunc(in, tensor.NewIndexed(tensor.NewNumberFromSlice([]float64{.75})), out) + QuantilesFunc(in, tensor.NewFloat64Scalar(.75), out) } diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index 33f2f2b6d1..ef21d13fbd 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -14,7 +14,7 @@ import ( func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) + ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) out := tensor.NewFloat64(1) oix := tensor.NewIndexed(out) @@ -90,7 +90,7 @@ func TestFuncs64(t *testing.T) { func TestFuncsInt(t *testing.T) { vals := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} - tsr := tensor.NewNumberFromSlice(vals) + tsr := tensor.NewNumberFromSlice(vals...) ix := tensor.NewIndexed(tsr) out := tensor.NewInt(1) oix := tensor.NewIndexed(out) diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 198b6e3f2c..8e381ea93e 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -32,7 +32,7 @@ func TestMath(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) + oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) oneout := oned.Clone() cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 166e289fb6..bd57440716 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -20,7 +20,7 @@ func TestAdd(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals)) + oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) oneout := oned.Clone() cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) diff --git a/yaegicore/symbols/cogentcore_org-core-goal-goalib.go b/yaegicore/symbols/cogentcore_org-core-goal-goalib.go new file mode 100644 index 0000000000..7f398c94a1 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-goal-goalib.go @@ -0,0 +1,21 @@ +// Code generated by 'yaegi extract cogentcore.org/core/goal/goalib'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/goal/goalib" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/goal/goalib/goalib"] = map[string]reflect.Value{ + // function, constant and variable definitions + "AllFiles": reflect.ValueOf(goalib.AllFiles), + "FileExists": reflect.ValueOf(goalib.FileExists), + "ReadFile": reflect.ValueOf(goalib.ReadFile), + "ReplaceInFile": reflect.ValueOf(goalib.ReplaceInFile), + "SplitLines": reflect.ValueOf(goalib.SplitLines), + "StringsToAnys": reflect.ValueOf(goalib.StringsToAnys), + "WriteFile": reflect.ValueOf(goalib.WriteFile), + } +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index a010efe98c..8df9b6bfd9 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -17,6 +17,7 @@ func init() { "DescribeTable": reflect.ValueOf(stats.DescribeTable), "DescribeTableAll": reflect.ValueOf(stats.DescribeTableAll), "DescriptiveStats": reflect.ValueOf(&stats.DescriptiveStats).Elem(), + "GroupAll": reflect.ValueOf(stats.GroupAll), "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), "GroupStats": reflect.ValueOf(stats.GroupStats), "Groups": reflect.ValueOf(stats.Groups), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index fcb1d8e00d..c98a7e2cb2 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -41,6 +41,7 @@ func init() { "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), "Include": reflect.ValueOf(tensor.Include), "IntToBool": reflect.ValueOf(tensor.IntToBool), + "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), "NFirstLen": reflect.ValueOf(tensor.NFirstLen), "NFirstRows": reflect.ValueOf(tensor.NFirstRows), "NMinLen": reflect.ValueOf(tensor.NMinLen), @@ -56,10 +57,12 @@ func init() { "NewIndexed": reflect.ValueOf(tensor.NewIndexed), "NewInt": reflect.ValueOf(tensor.NewInt), "NewInt32": reflect.ValueOf(tensor.NewInt32), + "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), "NewOfType": reflect.ValueOf(tensor.NewOfType), "NewShape": reflect.ValueOf(tensor.NewShape), "NewString": reflect.ValueOf(tensor.NewString), "NewStringFromSlice": reflect.ValueOf(tensor.NewStringFromSlice), + "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), "NewStringShape": reflect.ValueOf(tensor.NewStringShape), "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), "OddColumn": reflect.ValueOf(tensor.OddColumn), @@ -82,6 +85,8 @@ func init() { "SliceSet": reflect.ValueOf(tensor.SliceSet), "SliceSize": reflect.ValueOf(tensor.SliceSize), "Space": reflect.ValueOf(tensor.Space), + "Sprint": reflect.ValueOf(tensor.Sprint), + "SprintIndexed": reflect.ValueOf(tensor.SprintIndexed), "Stable": reflect.ValueOf(tensor.Stable), "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 8fa61ca4c8..3dfa36d8f7 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -8,4 +8,4 @@ command extract { } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo From 288adacc16e084ba2403e0cb95f259ba6bfba584 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 23:18:08 -0700 Subject: [PATCH 061/311] goal now actually using yaegicore imports -- do it once.. --- goal/cmd/goal/testdata/make | 2 +- goal/cmd/goal/{ => testdata}/test.goal | 2 +- goal/interpreter/imports.go | 1 - goal/interpreter/interpreter.go | 5 +++-- 4 files changed, 5 insertions(+), 5 deletions(-) rename goal/cmd/goal/{ => testdata}/test.goal (66%) diff --git a/goal/cmd/goal/testdata/make b/goal/cmd/goal/testdata/make index c19daef44c..79dc53408b 100755 --- a/goal/cmd/goal/testdata/make +++ b/goal/cmd/goal/testdata/make @@ -11,5 +11,5 @@ command test { println("running the test command") } -shell.RunCommands(args) +goal.RunCommands(args) diff --git a/goal/cmd/goal/test.goal b/goal/cmd/goal/testdata/test.goal similarity index 66% rename from goal/cmd/goal/test.goal rename to goal/cmd/goal/testdata/test.goal index e562da586b..8aeaafd913 100644 --- a/goal/cmd/goal/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -3,7 +3,7 @@ // todo: doesn't work: #1152 // echo {args} -for i, fn := range cosh.SplitLines(`/bin/ls -1`) { +for i, fn := range goalib.SplitLines(`/bin/ls -1`) { fmt.Println(i, fn) } diff --git a/goal/interpreter/imports.go b/goal/interpreter/imports.go index a6f7071a12..4ef7fd4635 100644 --- a/goal/interpreter/imports.go +++ b/goal/interpreter/imports.go @@ -9,7 +9,6 @@ package interpreter import ( "reflect" - _ "cogentcore.org/core/yaegicore/symbols" "github.com/cogentcore/yaegi/interp" ) diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index 3374f1956b..a9c83a2a1f 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -18,6 +18,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/goal" + "cogentcore.org/core/yaegicore/symbols" "github.com/cogentcore/yaegi/interp" "github.com/cogentcore/yaegi/stdlib" "github.com/ergochat/readline" @@ -62,8 +63,8 @@ func NewInterpreter(options interp.Options) *Interpreter { options.Stderr = in.Goal.StdIOWrappers.Err options.Stdin = in.Goal.StdIOWrappers.In in.Interp = interp.New(options) - errors.Log(in.Interp.Use(stdlib.Symbols)) - errors.Log(in.Interp.Use(Symbols)) + // errors.Log(in.Interp.Use(stdlib.Symbols)) + errors.Log(in.Interp.Use(symbols.Symbols)) in.ImportGoal() go in.MonitorSignals() return in From bfd2b8d2fb09aa658ef0755e7f5ffac346ef79a2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 17 Sep 2024 23:39:21 -0700 Subject: [PATCH 062/311] basic binary expr! --- goal/interpreter/interpreter.go | 2 +- goal/math.go | 27 ++++++++++++++++++++++++--- goal/transpile.go | 1 - goal/transpile_test.go | 1 + tensor/funcs.go | 9 +++++---- tensor/funcs_test.go | 4 ++-- tensor/stats/metric/metrics.go | 3 +-- tensor/stats/stats/stats.go | 3 +-- tensor/tmath/ops.go | 7 +++++++ 9 files changed, 42 insertions(+), 15 deletions(-) diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index a9c83a2a1f..2eaa66cf08 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -63,7 +63,7 @@ func NewInterpreter(options interp.Options) *Interpreter { options.Stderr = in.Goal.StdIOWrappers.Err options.Stdin = in.Goal.StdIOWrappers.In in.Interp = interp.New(options) - // errors.Log(in.Interp.Use(stdlib.Symbols)) + // errors.Log(in.Interp.Use(stdlib.Symbols)) // note: yaegicore/symbols gets just a few key things errors.Log(in.Interp.Use(symbols.Symbols)) in.ImportGoal() go in.MonitorSignals() diff --git a/goal/math.go b/goal/math.go index ee96a0c272..a7a730ef7c 100644 --- a/goal/math.go +++ b/goal/math.go @@ -43,16 +43,16 @@ func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { if assignIdx >= 0 { mp.lhsToks = toks[0:assignIdx] mp.lhs = ln[toks[0].Pos-1 : toks[assignIdx].Pos-1] - fmt.Println("lhs:", mp.lhs) mp.rhsToks = toks[assignIdx+1 : nt-1] mp.rhs = ln[toks[assignIdx+1].Pos-1 : toks[nt-1].Pos] - fmt.Println("rhs:", mp.rhs) lex, err := parser.ParseExpr(mp.lhs) if err != nil { + fmt.Println("lhs:", mp.lhs) fmt.Println("lhs parse err:", err) } rex, err := parser.ParseExpr(mp.rhs) if err != nil { + fmt.Println("rhs:", mp.rhs) fmt.Println("rhs parse err:", err) } mp.assignStmt(toks[assignIdx], lex, rex) @@ -62,9 +62,9 @@ func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { mp.rhs = ln[toks[0].Pos-1 : toks[nt-1].Pos] ex, err := parser.ParseExpr(mp.rhs) if err != nil { + fmt.Println("expr:", mp.rhs) fmt.Println("expr parse err:", err) } - fmt.Println("expr:", mp.rhs) mp.expr(ex) } @@ -117,6 +117,27 @@ func (mp *mathParse) expr(ex ast.Expr) { } func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { + fn := "" + switch ex.Op { + case token.ADD: + fn = "Add" + case token.SUB: + fn = "Sub" + case token.MUL: + fn = "Mul" + case token.QUO: + fn = "Div" + } + mp.toks.Add(token.IDENT, "tensor.CallOut") + mp.toks.Add(token.LPAREN) + mp.toks.Add(token.STRING, `"`+fn+`"`) + mp.toks.Add(token.COMMA) + mp.expr(ex.X) + mp.curIdx++ + mp.toks.Add(token.COMMA) + mp.expr(ex.Y) + mp.toks.Add(token.RPAREN) + // todo: need elipses if part of something bigger } func (mp *mathParse) basicLit(lit *ast.BasicLit) { diff --git a/goal/transpile.go b/goal/transpile.go index b2c89b15ff..80d9305c6c 100644 --- a/goal/transpile.go +++ b/goal/transpile.go @@ -74,7 +74,6 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { switch { case t0.Tok == token.ILLEGAL: - fmt.Println(t0.Str) if t0.Str != "" && t0.Str[:1] == "$" { return gl.TranspileMath(toks[1:], ln) } diff --git a/goal/transpile_test.go b/goal/transpile_test.go index 6fbaad971d..f9e8692abf 100644 --- a/goal/transpile_test.go +++ b/goal/transpile_test.go @@ -202,6 +202,7 @@ func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ {"$x := 1", `x := tensor.NewIntScalar(1)`}, + {"$x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, } gl := NewGoal() diff --git a/tensor/funcs.go b/tensor/funcs.go index 91f17dd151..9da0d4688e 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -122,16 +122,17 @@ func CallString(name, first string, tsr ...*Indexed) error { // CallOut calls function of given name, with given set of _input_ // arguments appropriate for the given function, returning newly created // output tensors. -// An error is returned if the function name has not been registered +// An error is logged if the function name has not been registered // in the Funcs global function registry, or the argument count // does not match. -func CallOut(name string, tsr ...*Indexed) ([]*Indexed, error) { +func CallOut(name string, tsr ...*Indexed) []*Indexed { nm := strings.ToLower(name) fn, ok := Funcs[nm] if !ok { - return nil, errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) + errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) + return nil } - return fn.CallOut(tsr...) + return errors.Log1(fn.CallOut(tsr...)) } // ArgCount returns the number of tensor arguments the function takes, diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index cfe85e7e6c..93248c8a12 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -42,7 +42,7 @@ func TestFuncs(t *testing.T) { err = Call("Abs", oned) assert.Error(t, err) - out, err := CallOut("Abs", oned) - assert.NoError(t, err) + out := CallOut("Abs", oned) + // assert.NoError(t, err) assert.Equal(t, oneout.Tensor.(*Float64).Values, out[0].Tensor.(*Float64).Values) } diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index ff04b8a1c6..c0a35e7e5d 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -7,7 +7,6 @@ package metric import ( - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -36,7 +35,7 @@ func Metric(metric Metrics, a, b, out *tensor.Indexed) { // MetricOut calls a standard Metrics enum function on given tensors, // returning output as a newly created tensor. func MetricOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { - return errors.Log1(tensor.CallOut(metric.FuncName(), a, b))[0] // note: error should never happen + return tensor.CallOut(metric.FuncName(), a, b)[0] } // Metrics are standard metric functions diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index d4eeca3e1d..1128a5677d 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -7,7 +7,6 @@ package stats import ( "strings" - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -132,5 +131,5 @@ func Stat(stat Stats, in, out *tensor.Indexed) { // StatOut calls a standard Stats enum function on given tensor, // returning output as a newly created tensor. func StatOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { - return errors.Log1(tensor.CallOut(stat.FuncName(), in))[0] // note: error should never happen + return tensor.CallOut(stat.FuncName(), in)[0] } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 091aa52f00..6bce47c6cd 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -8,6 +8,13 @@ import ( "cogentcore.org/core/tensor" ) +func init() { + tensor.AddFunc("Add", Add, 1) + tensor.AddFunc("Sub", Sub, 1) + tensor.AddFunc("Mul", Mul, 1) + tensor.AddFunc("Div", Div, 1) +} + // Add adds two tensors into output. // If one is a scalar, it is added to all elements. // If one is the same size as the Cell SubSpace of the other From 3dd431601fcb593ed229e47806504fe576f50371 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 00:59:09 -0700 Subject: [PATCH 063/311] probably need our own parser at this point for math: array expressiosn are not well handled. --- goal/README.md | 2 +- goal/cmd/goal/testdata/test.goal | 6 ++ goal/goal.go | 9 ++- goal/interpreter/interpreter.go | 3 + goal/math.go | 28 ++++++-- goal/transpile_test.go | 5 +- tensor/tmath/math_test.go | 10 +++ .../cogentcore_org-core-tensor-tmath.go | 69 +++++++++++++++++++ yaegicore/symbols/make | 2 +- 9 files changed, 123 insertions(+), 11 deletions(-) create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-tmath.go diff --git a/goal/README.md b/goal/README.md index 3660ea502b..5da7c5433f 100644 --- a/goal/README.md +++ b/goal/README.md @@ -218,7 +218,7 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of array `a` | | same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` array | | same: | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of array a | -| `tensor([[1., 2., 3.], [4., 5., 6.]])` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D array | +| `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D array | | same: | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks a, b, c, and d | | same: | `a[-1]` | `a(end)` | access last element | | same: | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D array a | diff --git a/goal/cmd/goal/testdata/test.goal b/goal/cmd/goal/testdata/test.goal index 8aeaafd913..ee83f73e38 100644 --- a/goal/cmd/goal/testdata/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -8,4 +8,10 @@ for i, fn := range goalib.SplitLines(`/bin/ls -1`) { } $ x := 1 +$ y := 4 +$ a := x * 2 +$ b := x + y +$ c := x * y + a * b + +fmt.Println(c) diff --git a/goal/goal.go b/goal/goal.go index 4af0c5cab7..5ba9a0afad 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -354,7 +354,14 @@ func (gl *Goal) TranspileFile(in string, out string) error { if hasPackage { gl.Lines = slices.Insert(gl.Lines, 0, gen) } else { - gl.Lines = slices.Insert(gl.Lines, 0, gen, "package main", "", "func main() {", "goal := goal.NewGoal()") + gl.Lines = slices.Insert(gl.Lines, 0, gen, "package main", "import (", + ` "cogentcore.org/core/goal"`, + ` "cogentcore.org/core/goal/goalib"`, + ` "cogentcore.org/core/tensor"`, + ` _ "cogentcore.org/core/tensor/tmath"`, + ` _ "cogentcore.org/core/tensor/stats/stats"`, + ` _ "cogentcore.org/core/tensor/stats/metric"`, + ")", "func main() {", "goal := goal.NewGoal()") gl.Lines = append(gl.Lines, "}") } src := []byte(gl.Code()) diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index 2eaa66cf08..ceebd88b05 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -18,6 +18,9 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/goal" + _ "cogentcore.org/core/tensor/stats/metric" + _ "cogentcore.org/core/tensor/stats/stats" + _ "cogentcore.org/core/tensor/tmath" "cogentcore.org/core/yaegicore/symbols" "github.com/cogentcore/yaegi/interp" "github.com/cogentcore/yaegi/stdlib" diff --git a/goal/math.go b/goal/math.go index a7a730ef7c..c52ca5c857 100644 --- a/goal/math.go +++ b/goal/math.go @@ -22,13 +22,17 @@ type mathParse struct { } func (mp *mathParse) addCur() { - mp.toks.AddTokens(mp.curToks[mp.curIdx]) - mp.curIdx++ + if len(mp.curToks) > mp.curIdx { + mp.toks.AddTokens(mp.curToks[mp.curIdx]) + mp.curIdx++ + return + } + fmt.Println("out of toks:", mp.curToks) } func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { - // fmt.Println("in math") nt := len(toks) + // fmt.Println(nt, toks) mp := mathParse{} @@ -43,7 +47,7 @@ func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { if assignIdx >= 0 { mp.lhsToks = toks[0:assignIdx] mp.lhs = ln[toks[0].Pos-1 : toks[assignIdx].Pos-1] - mp.rhsToks = toks[assignIdx+1 : nt-1] + mp.rhsToks = toks[assignIdx+1 : nt] mp.rhs = ln[toks[assignIdx+1].Pos-1 : toks[nt-1].Pos] lex, err := parser.ParseExpr(mp.lhs) if err != nil { @@ -54,10 +58,11 @@ func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { if err != nil { fmt.Println("rhs:", mp.rhs) fmt.Println("rhs parse err:", err) + fmt.Printf("%#v\n", rex) } mp.assignStmt(toks[assignIdx], lex, rex) } else { - mp.rhsToks = toks[0 : nt-1] + mp.rhsToks = toks[0:nt] mp.curToks = mp.rhsToks mp.rhs = ln[toks[0].Pos-1 : toks[nt-1].Pos] ex, err := parser.ParseExpr(mp.rhs) @@ -113,6 +118,12 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.CallExpr: + case *ast.ArrayType: + // basically at this point we have a bad expression and + // need to do our own parsing. + // it is unclear if perhaps we just need to do that from the start. + fmt.Println("array type:", x, x.Len) + fmt.Printf("%#v\n", x.Len) } } @@ -133,11 +144,13 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { mp.toks.Add(token.STRING, `"`+fn+`"`) mp.toks.Add(token.COMMA) mp.expr(ex.X) - mp.curIdx++ mp.toks.Add(token.COMMA) + mp.curIdx++ mp.expr(ex.Y) mp.toks.Add(token.RPAREN) - // todo: need elipses if part of something bigger + mp.toks.Add(token.LBRACK) + mp.toks.Add(token.INT, "0") + mp.toks.Add(token.RBRACK) } func (mp *mathParse) basicLit(lit *ast.BasicLit) { @@ -158,4 +171,5 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { } func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { + fmt.Println("slice expr", se) } diff --git a/goal/transpile_test.go b/goal/transpile_test.go index f9e8692abf..5acc413f01 100644 --- a/goal/transpile_test.go +++ b/goal/transpile_test.go @@ -202,7 +202,10 @@ func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ {"$x := 1", `x := tensor.NewIntScalar(1)`}, - {"$x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + {"$x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))[0]`}, + {"$x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))[0]`}, + {"$a = x + y", `a = tensor.CallOut("Add", x, y)[0]`}, + {"$a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, } gl := NewGoal() diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 8e381ea93e..48168f4940 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -67,3 +67,13 @@ func TestMath(t *testing.T) { } } } + +func TestOps(t *testing.T) { + x := tensor.NewIntScalar(1) + y := tensor.NewIntScalar(4) + a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2))[0] + b := tensor.CallOut("Add", x, y)[0] + c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y)[0], tensor.CallOut("Mul", a, b)[0])[0] + + assert.Equal(t, 14, c.IntRow(0)) +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go new file mode 100644 index 0000000000..4513200599 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -0,0 +1,69 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/tmath'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/tmath" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/tmath/tmath"] = map[string]reflect.Value{ + // function, constant and variable definitions + "Abs": reflect.ValueOf(tmath.Abs), + "Acos": reflect.ValueOf(tmath.Acos), + "Acosh": reflect.ValueOf(tmath.Acosh), + "Add": reflect.ValueOf(tmath.Add), + "AddScalar": reflect.ValueOf(tmath.AddScalar), + "AddSubSpace": reflect.ValueOf(tmath.AddSubSpace), + "Asin": reflect.ValueOf(tmath.Asin), + "Asinh": reflect.ValueOf(tmath.Asinh), + "Atan": reflect.ValueOf(tmath.Atan), + "Atanh": reflect.ValueOf(tmath.Atanh), + "Binarize": reflect.ValueOf(tmath.Binarize), + "Cbrt": reflect.ValueOf(tmath.Cbrt), + "Ceil": reflect.ValueOf(tmath.Ceil), + "Clamp": reflect.ValueOf(tmath.Clamp), + "Cos": reflect.ValueOf(tmath.Cos), + "Cosh": reflect.ValueOf(tmath.Cosh), + "Div": reflect.ValueOf(tmath.Div), + "DivScalar": reflect.ValueOf(tmath.DivScalar), + "DivScalarInv": reflect.ValueOf(tmath.DivScalarInv), + "DivSubSpace": reflect.ValueOf(tmath.DivSubSpace), + "DivSubSpaceInv": reflect.ValueOf(tmath.DivSubSpaceInv), + "Erf": reflect.ValueOf(tmath.Erf), + "Erfc": reflect.ValueOf(tmath.Erfc), + "Erfcinv": reflect.ValueOf(tmath.Erfcinv), + "Erfinv": reflect.ValueOf(tmath.Erfinv), + "Exp": reflect.ValueOf(tmath.Exp), + "Exp2": reflect.ValueOf(tmath.Exp2), + "Expm1": reflect.ValueOf(tmath.Expm1), + "Floor": reflect.ValueOf(tmath.Floor), + "Gamma": reflect.ValueOf(tmath.Gamma), + "J0": reflect.ValueOf(tmath.J0), + "J1": reflect.ValueOf(tmath.J1), + "Log": reflect.ValueOf(tmath.Log), + "Log10": reflect.ValueOf(tmath.Log10), + "Log1p": reflect.ValueOf(tmath.Log1p), + "Log2": reflect.ValueOf(tmath.Log2), + "Logb": reflect.ValueOf(tmath.Logb), + "Mul": reflect.ValueOf(tmath.Mul), + "MulScalar": reflect.ValueOf(tmath.MulScalar), + "MulSubSpace": reflect.ValueOf(tmath.MulSubSpace), + "Round": reflect.ValueOf(tmath.Round), + "RoundToEven": reflect.ValueOf(tmath.RoundToEven), + "Sin": reflect.ValueOf(tmath.Sin), + "Sinh": reflect.ValueOf(tmath.Sinh), + "Sqrt": reflect.ValueOf(tmath.Sqrt), + "Sub": reflect.ValueOf(tmath.Sub), + "SubScalar": reflect.ValueOf(tmath.SubScalar), + "SubSubSpace": reflect.ValueOf(tmath.SubSubSpace), + "Tan": reflect.ValueOf(tmath.Tan), + "Tanh": reflect.ValueOf(tmath.Tanh), + "Trunc": reflect.ValueOf(tmath.Trunc), + "UnitNorm": reflect.ValueOf(tmath.UnitNorm), + "Y0": reflect.ValueOf(tmath.Y0), + "Y1": reflect.ValueOf(tmath.Y1), + "ZScore": reflect.ValueOf(tmath.ZScore), + } +} diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 3dfa36d8f7..0c5d689d31 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -8,4 +8,4 @@ command extract { } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo From aa2b23833e2ee641b1a20c8b183aad68174dd32b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 08:41:17 -0700 Subject: [PATCH 064/311] most math funcs return only 1 value; CallOut does that, making yaegi work less hard. --- goal/math.go | 3 --- goal/transpile_test.go | 10 +++++----- tensor/funcs.go | 22 +++++++++++++++++++--- tensor/funcs_test.go | 2 +- tensor/stats/metric/metrics.go | 2 +- tensor/stats/stats/stats.go | 2 +- tensor/tmath/math_test.go | 6 +++--- 7 files changed, 30 insertions(+), 17 deletions(-) diff --git a/goal/math.go b/goal/math.go index c52ca5c857..589a946e5c 100644 --- a/goal/math.go +++ b/goal/math.go @@ -148,9 +148,6 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { mp.curIdx++ mp.expr(ex.Y) mp.toks.Add(token.RPAREN) - mp.toks.Add(token.LBRACK) - mp.toks.Add(token.INT, "0") - mp.toks.Add(token.RBRACK) } func (mp *mathParse) basicLit(lit *ast.BasicLit) { diff --git a/goal/transpile_test.go b/goal/transpile_test.go index 5acc413f01..c6c37e78db 100644 --- a/goal/transpile_test.go +++ b/goal/transpile_test.go @@ -201,11 +201,11 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"$x := 1", `x := tensor.NewIntScalar(1)`}, - {"$x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))[0]`}, - {"$x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))[0]`}, - {"$a = x + y", `a = tensor.CallOut("Add", x, y)[0]`}, - {"$a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, + {"$ x := 1", `x := tensor.NewIntScalar(1)`}, + {"$ x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + {"$ x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + {"$ a = x + y", `a = tensor.CallOut("Add", x, y)`}, + // {"$ a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, } gl := NewGoal() diff --git a/tensor/funcs.go b/tensor/funcs.go index 9da0d4688e..bd0e7f32bc 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -18,7 +18,7 @@ const StringFirstArg = true // Func represents a registered tensor function, which has // In number of input *tensor.Indexed arguments, and Out -// number of output arguments. There can also be an optional +// number of output arguments (typically 1). There can also be an optional // string first argument, which is used to specify the name of // another function in some cases (e.g., a stat or metric function). type Func struct { @@ -120,12 +120,28 @@ func CallString(name, first string, tsr ...*Indexed) error { } // CallOut calls function of given name, with given set of _input_ +// arguments appropriate for the given function, returning a created +// output tensor, for the common case with just one return value. +// An error is logged if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match. +func CallOut(name string, tsr ...*Indexed) *Indexed { + nm := strings.ToLower(name) + fn, ok := Funcs[nm] + if !ok { + errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) + return nil + } + return errors.Log1(fn.CallOut(tsr...))[0] +} + +// CallOutMulti calls function of given name, with given set of _input_ // arguments appropriate for the given function, returning newly created -// output tensors. +// output tensors, for the rare case of multiple return values. // An error is logged if the function name has not been registered // in the Funcs global function registry, or the argument count // does not match. -func CallOut(name string, tsr ...*Indexed) []*Indexed { +func CallOutMulti(name string, tsr ...*Indexed) []*Indexed { nm := strings.ToLower(name) fn, ok := Funcs[nm] if !ok { diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 93248c8a12..53f8c1ef82 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -44,5 +44,5 @@ func TestFuncs(t *testing.T) { out := CallOut("Abs", oned) // assert.NoError(t, err) - assert.Equal(t, oneout.Tensor.(*Float64).Values, out[0].Tensor.(*Float64).Values) + assert.Equal(t, oneout.Tensor.(*Float64).Values, out.Tensor.(*Float64).Values) } diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index c0a35e7e5d..57691415e8 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -35,7 +35,7 @@ func Metric(metric Metrics, a, b, out *tensor.Indexed) { // MetricOut calls a standard Metrics enum function on given tensors, // returning output as a newly created tensor. func MetricOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { - return tensor.CallOut(metric.FuncName(), a, b)[0] + return tensor.CallOut(metric.FuncName(), a, b) } // Metrics are standard metric functions diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 1128a5677d..b901649b09 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -131,5 +131,5 @@ func Stat(stat Stats, in, out *tensor.Indexed) { // StatOut calls a standard Stats enum function on given tensor, // returning output as a newly created tensor. func StatOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { - return tensor.CallOut(stat.FuncName(), in)[0] + return tensor.CallOut(stat.FuncName(), in) } diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 48168f4940..f04f56c5fb 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -71,9 +71,9 @@ func TestMath(t *testing.T) { func TestOps(t *testing.T) { x := tensor.NewIntScalar(1) y := tensor.NewIntScalar(4) - a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2))[0] - b := tensor.CallOut("Add", x, y)[0] - c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y)[0], tensor.CallOut("Mul", a, b)[0])[0] + a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2)) + b := tensor.CallOut("Add", x, y) + c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y), tensor.CallOut("Mul", a, b)) assert.Equal(t, 14, c.IntRow(0)) } From 3b49be86ef641c0ea6855f1e732a160194717bd4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 09:27:00 -0700 Subject: [PATCH 065/311] starting copy of go/parser for math parsing --- goal/mparse/mparse.go | 2917 +++++++++++++++++ .../symbols/cogentcore_org-core-tensor.go | 1 + yaegicore/symbols/make | 2 +- 3 files changed, 2919 insertions(+), 1 deletion(-) create mode 100644 goal/mparse/mparse.go diff --git a/goal/mparse/mparse.go b/goal/mparse/mparse.go new file mode 100644 index 0000000000..5dfbeb807a --- /dev/null +++ b/goal/mparse/mparse.go @@ -0,0 +1,2917 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// mparse is a hacked version of go/parser: +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package mparse is a hacked version of go/parser to support +// the additional math mode syntax for goal. +package mparse + +import ( + "fmt" + "go/ast" + "go/build/constraint" + "go/scanner" + "go/token" + "strings" +) + +// A Mode value is a set of flags (or 0). +// They control the amount of source code parsed and other optional +// parser functionality. +type Mode uint + +const ( + PackageClauseOnly Mode = 1 << iota // stop parsing after package clause + ImportsOnly // stop parsing after import declarations + ParseComments // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors + SpuriousErrors // same as AllErrors, for backward-compatibility + SkipObjectResolution // skip deprecated identifier resolution; see ParseFile + AllErrors = SpuriousErrors // report all errors (not just the first 10 on different lines) +) + +// The parser structure holds the parser's internal state. +type parser struct { + file *token.File + errors scanner.ErrorList + scanner scanner.Scanner + + // Tracing/debugging + mode Mode // parsing mode + trace bool // == (mode&Trace != 0) + indent int // indentation used for tracing output + + // Comments + comments []*ast.CommentGroup + leadComment *ast.CommentGroup // last lead comment + lineComment *ast.CommentGroup // last line comment + top bool // in top of file (before package clause) + goVersion string // minimum Go version found in //go:build comment + + // Next token + pos token.Pos // token position + tok token.Token // one token look-ahead + lit string // token literal + + // Error recovery + // (used to limit the number of calls to parser.advance + // w/o making scanning progress - avoids potential endless + // loops across multiple parser functions during error recovery) + syncPos token.Pos // last synchronization position + syncCnt int // number of parser.advance calls without progress + + // Non-syntactic parser control + exprLev int // < 0: in control clause, >= 0: in expression + inRhs bool // if set, the parser is parsing a rhs expression + + imports []*ast.ImportSpec // list of imports + + // nestLev is used to track and limit the recursion depth + // during parsing. + nestLev int +} + +func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) { + p.file = fset.AddFile(filename, -1, len(src)) + eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) } + p.scanner.Init(p.file, src, eh, scanner.ScanComments) + + p.top = true + p.mode = mode + p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) + p.next() +} + +// ---------------------------------------------------------------------------- +// Parsing support + +func (p *parser) printTrace(a ...any) { + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const n = len(dots) + pos := p.file.Position(p.pos) + fmt.Printf("%5d:%3d: ", pos.Line, pos.Column) + i := 2 * p.indent + for i > n { + fmt.Print(dots) + i -= n + } + // i <= n + fmt.Print(dots[0:i]) + fmt.Println(a...) +} + +func trace(p *parser, msg string) *parser { + p.printTrace(msg, "(") + p.indent++ + return p +} + +// Usage pattern: defer un(trace(p, "...")) +func un(p *parser) { + p.indent-- + p.printTrace(")") +} + +// maxNestLev is the deepest we're willing to recurse during parsing +const maxNestLev int = 1e5 + +func incNestLev(p *parser) *parser { + p.nestLev++ + if p.nestLev > maxNestLev { + p.error(p.pos, "exceeded max nesting depth") + panic(bailout{}) + } + return p +} + +// decNestLev is used to track nesting depth during parsing to prevent stack exhaustion. +// It is used along with incNestLev in a similar fashion to how un and trace are used. +func decNestLev(p *parser) { + p.nestLev-- +} + +// Advance to the next token. +func (p *parser) next0() { + // Because of one-token look-ahead, print the previous token + // when tracing as it provides a more readable output. The + // very first token (!p.pos.IsValid()) is not initialized + // (it is token.ILLEGAL), so don't print it. + if p.trace && p.pos.IsValid() { + s := p.tok.String() + switch { + case p.tok.IsLiteral(): + p.printTrace(s, p.lit) + case p.tok.IsOperator(), p.tok.IsKeyword(): + p.printTrace("\"" + s + "\"") + default: + p.printTrace(s) + } + } + + for { + p.pos, p.tok, p.lit = p.scanner.Scan() + if p.tok == token.COMMENT { + if p.top && strings.HasPrefix(p.lit, "//go:build") { + if x, err := constraint.Parse(p.lit); err == nil { + p.goVersion = constraint.GoVersion(x) + } + } + if p.mode&ParseComments == 0 { + continue + } + } else { + // Found a non-comment; top of file is over. + p.top = false + } + break + } +} + +// Consume a comment and return it and the line on which it ends. +func (p *parser) consumeComment() (comment *ast.Comment, endline int) { + // /*-style comments may end on a different line than where they start. + // Scan the comment for '\n' chars and adjust endline accordingly. + endline = p.file.Line(p.pos) + if p.lit[1] == '*' { + // don't use range here - no need to decode Unicode code points + for i := 0; i < len(p.lit); i++ { + if p.lit[i] == '\n' { + endline++ + } + } + } + + comment = &ast.Comment{Slash: p.pos, Text: p.lit} + p.next0() + + return +} + +// Consume a group of adjacent comments, add it to the parser's +// comments list, and return it together with the line at which +// the last comment in the group ends. A non-comment token or n +// empty lines terminate a comment group. +func (p *parser) consumeCommentGroup(n int) (comments *ast.CommentGroup, endline int) { + var list []*ast.Comment + endline = p.file.Line(p.pos) + for p.tok == token.COMMENT && p.file.Line(p.pos) <= endline+n { + var comment *ast.Comment + comment, endline = p.consumeComment() + list = append(list, comment) + } + + // add comment group to the comments list + comments = &ast.CommentGroup{List: list} + p.comments = append(p.comments, comments) + + return +} + +// Advance to the next non-comment token. In the process, collect +// any comment groups encountered, and remember the last lead and +// line comments. +// +// A lead comment is a comment group that starts and ends in a +// line without any other tokens and that is followed by a non-comment +// token on the line immediately after the comment group. +// +// A line comment is a comment group that follows a non-comment +// token on the same line, and that has no tokens after it on the line +// where it ends. +// +// Lead and line comments may be considered documentation that is +// stored in the AST. +func (p *parser) next() { + p.leadComment = nil + p.lineComment = nil + prev := p.pos + p.next0() + + if p.tok == token.COMMENT { + var comment *ast.CommentGroup + var endline int + + if p.file.Line(p.pos) == p.file.Line(prev) { + // The comment is on same line as the previous token; it + // cannot be a lead comment but may be a line comment. + comment, endline = p.consumeCommentGroup(0) + if p.file.Line(p.pos) != endline || p.tok == token.SEMICOLON || p.tok == token.EOF { + // The next token is on a different line, thus + // the last comment group is a line comment. + p.lineComment = comment + } + } + + // consume successor comments, if any + endline = -1 + for p.tok == token.COMMENT { + comment, endline = p.consumeCommentGroup(1) + } + + if endline+1 == p.file.Line(p.pos) { + // The next token is following on the line immediately after the + // comment group, thus the last comment group is a lead comment. + p.leadComment = comment + } + } +} + +// A bailout panic is raised to indicate early termination. pos and msg are +// only populated when bailing out of object resolution. +type bailout struct { + pos token.Pos + msg string +} + +func (p *parser) error(pos token.Pos, msg string) { + if p.trace { + defer un(trace(p, "error: "+msg)) + } + + epos := p.file.Position(pos) + + // If AllErrors is not set, discard errors reported on the same line + // as the last recorded error and stop parsing if there are more than + // 10 errors. + if p.mode&AllErrors == 0 { + n := len(p.errors) + if n > 0 && p.errors[n-1].Pos.Line == epos.Line { + return // discard - likely a spurious error + } + if n > 10 { + panic(bailout{}) + } + } + + p.errors.Add(epos, msg) +} + +func (p *parser) errorExpected(pos token.Pos, msg string) { + msg = "expected " + msg + if pos == p.pos { + // the error happened at the current position; + // make the error message more specific + switch { + case p.tok == token.SEMICOLON && p.lit == "\n": + msg += ", found newline" + case p.tok.IsLiteral(): + // print 123 rather than 'INT', etc. + msg += ", found " + p.lit + default: + msg += ", found '" + p.tok.String() + "'" + } + } + p.error(pos, msg) +} + +func (p *parser) expect(tok token.Token) token.Pos { + pos := p.pos + if p.tok != tok { + p.errorExpected(pos, "'"+tok.String()+"'") + } + p.next() // make progress + return pos +} + +// expect2 is like expect, but it returns an invalid position +// if the expected token is not found. +func (p *parser) expect2(tok token.Token) (pos token.Pos) { + if p.tok == tok { + pos = p.pos + } else { + p.errorExpected(p.pos, "'"+tok.String()+"'") + } + p.next() // make progress + return +} + +// expectClosing is like expect but provides a better error message +// for the common case of a missing comma before a newline. +func (p *parser) expectClosing(tok token.Token, context string) token.Pos { + if p.tok != tok && p.tok == token.SEMICOLON && p.lit == "\n" { + p.error(p.pos, "missing ',' before newline in "+context) + p.next() + } + return p.expect(tok) +} + +// expectSemi consumes a semicolon and returns the applicable line comment. +func (p *parser) expectSemi() (comment *ast.CommentGroup) { + // semicolon is optional before a closing ')' or '}' + if p.tok != token.RPAREN && p.tok != token.RBRACE { + switch p.tok { + case token.COMMA: + // permit a ',' instead of a ';' but complain + p.errorExpected(p.pos, "';'") + fallthrough + case token.SEMICOLON: + if p.lit == ";" { + // explicit semicolon + p.next() + comment = p.lineComment // use following comments + } else { + // artificial semicolon + comment = p.lineComment // use preceding comments + p.next() + } + return comment + default: + p.errorExpected(p.pos, "';'") + p.advance(stmtStart) + } + } + return nil +} + +func (p *parser) atComma(context string, follow token.Token) bool { + if p.tok == token.COMMA { + return true + } + if p.tok != follow { + msg := "missing ','" + if p.tok == token.SEMICOLON && p.lit == "\n" { + msg += " before newline" + } + p.error(p.pos, msg+" in "+context) + return true // "insert" comma and continue + } + return false +} + +func assert(cond bool, msg string) { + if !cond { + panic("go/parser internal error: " + msg) + } +} + +// advance consumes tokens until the current token p.tok +// is in the 'to' set, or token.EOF. For error recovery. +func (p *parser) advance(to map[token.Token]bool) { + for ; p.tok != token.EOF; p.next() { + if to[p.tok] { + // Return only if parser made some progress since last + // sync or if it has not reached 10 advance calls without + // progress. Otherwise consume at least one token to + // avoid an endless parser loop (it is possible that + // both parseOperand and parseStmt call advance and + // correctly do not advance, thus the need for the + // invocation limit p.syncCnt). + if p.pos == p.syncPos && p.syncCnt < 10 { + p.syncCnt++ + return + } + if p.pos > p.syncPos { + p.syncPos = p.pos + p.syncCnt = 0 + return + } + // Reaching here indicates a parser bug, likely an + // incorrect token list in this function, but it only + // leads to skipping of possibly correct code if a + // previous error is present, and thus is preferred + // over a non-terminating parse. + } + } +} + +var stmtStart = map[token.Token]bool{ + token.BREAK: true, + token.CONST: true, + token.CONTINUE: true, + token.DEFER: true, + token.FALLTHROUGH: true, + token.FOR: true, + token.GO: true, + token.GOTO: true, + token.IF: true, + token.RETURN: true, + token.SELECT: true, + token.SWITCH: true, + token.TYPE: true, + token.VAR: true, +} + +var declStart = map[token.Token]bool{ + token.IMPORT: true, + token.CONST: true, + token.TYPE: true, + token.VAR: true, +} + +var exprEnd = map[token.Token]bool{ + token.COMMA: true, + token.COLON: true, + token.SEMICOLON: true, + token.RPAREN: true, + token.RBRACK: true, + token.RBRACE: true, +} + +// safePos returns a valid file position for a given position: If pos +// is valid to begin with, safePos returns pos. If pos is out-of-range, +// safePos returns the EOF position. +// +// This is hack to work around "artificial" end positions in the AST which +// are computed by adding 1 to (presumably valid) token positions. If the +// token positions are invalid due to parse errors, the resulting end position +// may be past the file's EOF position, which would lead to panics if used +// later on. +func (p *parser) safePos(pos token.Pos) (res token.Pos) { + defer func() { + if recover() != nil { + res = token.Pos(p.file.Base() + p.file.Size()) // EOF position + } + }() + _ = p.file.Offset(pos) // trigger a panic if position is out-of-range + return pos +} + +// ---------------------------------------------------------------------------- +// Identifiers + +func (p *parser) parseIdent() *ast.Ident { + pos := p.pos + name := "_" + if p.tok == token.IDENT { + name = p.lit + p.next() + } else { + p.expect(token.IDENT) // use expect() error handling + } + return &ast.Ident{NamePos: pos, Name: name} +} + +func (p *parser) parseIdentList() (list []*ast.Ident) { + if p.trace { + defer un(trace(p, "IdentList")) + } + + list = append(list, p.parseIdent()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseIdent()) + } + + return +} + +// ---------------------------------------------------------------------------- +// Common productions + +// If lhs is set, result list elements which are identifiers are not resolved. +func (p *parser) parseExprList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ExpressionList")) + } + + list = append(list, p.parseExpr()) + for p.tok == token.COMMA { + p.next() + list = append(list, p.parseExpr()) + } + + return +} + +func (p *parser) parseList(inRhs bool) []ast.Expr { + old := p.inRhs + p.inRhs = inRhs + list := p.parseExprList() + p.inRhs = old + return list +} + +// ---------------------------------------------------------------------------- +// Types + +func (p *parser) parseType() ast.Expr { + if p.trace { + defer un(trace(p, "Type")) + } + + typ := p.tryIdentOrType() + + if typ == nil { + pos := p.pos + p.errorExpected(pos, "type") + p.advance(exprEnd) + return &ast.BadExpr{From: pos, To: p.pos} + } + + return typ +} + +func (p *parser) parseQualifiedIdent(ident *ast.Ident) ast.Expr { + if p.trace { + defer un(trace(p, "QualifiedIdent")) + } + + typ := p.parseTypeName(ident) + if p.tok == token.LBRACK { + typ = p.parseTypeInstance(typ) + } + + return typ +} + +// If the result is an identifier, it is not resolved. +func (p *parser) parseTypeName(ident *ast.Ident) ast.Expr { + if p.trace { + defer un(trace(p, "TypeName")) + } + + if ident == nil { + ident = p.parseIdent() + } + + if p.tok == token.PERIOD { + // ident is a package name + p.next() + sel := p.parseIdent() + return &ast.SelectorExpr{X: ident, Sel: sel} + } + + return ident +} + +// "[" has already been consumed, and lbrack is its position. +// If len != nil it is the already consumed array length. +func (p *parser) parseArrayType(lbrack token.Pos, len ast.Expr) *ast.ArrayType { + if p.trace { + defer un(trace(p, "ArrayType")) + } + + if len == nil { + p.exprLev++ + // always permit ellipsis for more fault-tolerant parsing + if p.tok == token.ELLIPSIS { + len = &ast.Ellipsis{Ellipsis: p.pos} + p.next() + } else if p.tok != token.RBRACK { + len = p.parseRhs() + } + p.exprLev-- + } + if p.tok == token.COMMA { + // Trailing commas are accepted in type parameter + // lists but not in array type declarations. + // Accept for better error handling but complain. + p.error(p.pos, "unexpected comma; expecting ]") + p.next() + } + p.expect(token.RBRACK) + elt := p.parseType() + return &ast.ArrayType{Lbrack: lbrack, Len: len, Elt: elt} +} + +func (p *parser) parseArrayFieldOrTypeInstance(x *ast.Ident) (*ast.Ident, ast.Expr) { + if p.trace { + defer un(trace(p, "ArrayFieldOrTypeInstance")) + } + + lbrack := p.expect(token.LBRACK) + trailingComma := token.NoPos // if valid, the position of a trailing comma preceding the ']' + var args []ast.Expr + if p.tok != token.RBRACK { + p.exprLev++ + args = append(args, p.parseRhs()) + for p.tok == token.COMMA { + comma := p.pos + p.next() + if p.tok == token.RBRACK { + trailingComma = comma + break + } + args = append(args, p.parseRhs()) + } + p.exprLev-- + } + rbrack := p.expect(token.RBRACK) + _ = rbrack + + if len(args) == 0 { + // x []E + elt := p.parseType() + return x, &ast.ArrayType{Lbrack: lbrack, Elt: elt} + } + + // x [P]E or x[P] + if len(args) == 1 { + elt := p.tryIdentOrType() + if elt != nil { + // x [P]E + if trailingComma.IsValid() { + // Trailing commas are invalid in array type fields. + p.error(trailingComma, "unexpected comma; expecting ]") + } + return x, &ast.ArrayType{Lbrack: lbrack, Len: args[0], Elt: elt} + } + } + + // x[P], x[P1, P2], ... + return nil, nil // typeparams.PackIndexExpr(x, lbrack, args, rbrack) +} + +func (p *parser) parseFieldDecl() *ast.Field { + if p.trace { + defer un(trace(p, "FieldDecl")) + } + + doc := p.leadComment + + var names []*ast.Ident + var typ ast.Expr + switch p.tok { + case token.IDENT: + name := p.parseIdent() + if p.tok == token.PERIOD || p.tok == token.STRING || p.tok == token.SEMICOLON || p.tok == token.RBRACE { + // embedded type + typ = name + if p.tok == token.PERIOD { + typ = p.parseQualifiedIdent(name) + } + } else { + // name1, name2, ... T + names = []*ast.Ident{name} + for p.tok == token.COMMA { + p.next() + names = append(names, p.parseIdent()) + } + // Careful dance: We don't know if we have an embedded instantiated + // type T[P1, P2, ...] or a field T of array type []E or [P]E. + if len(names) == 1 && p.tok == token.LBRACK { + name, typ = p.parseArrayFieldOrTypeInstance(name) + if name == nil { + names = nil + } + } else { + // T P + typ = p.parseType() + } + } + case token.MUL: + star := p.pos + p.next() + if p.tok == token.LPAREN { + // *(T) + p.error(p.pos, "cannot parenthesize embedded type") + p.next() + typ = p.parseQualifiedIdent(nil) + // expect closing ')' but no need to complain if missing + if p.tok == token.RPAREN { + p.next() + } + } else { + // *T + typ = p.parseQualifiedIdent(nil) + } + typ = &ast.StarExpr{Star: star, X: typ} + + case token.LPAREN: + p.error(p.pos, "cannot parenthesize embedded type") + p.next() + if p.tok == token.MUL { + // (*T) + star := p.pos + p.next() + typ = &ast.StarExpr{Star: star, X: p.parseQualifiedIdent(nil)} + } else { + // (T) + typ = p.parseQualifiedIdent(nil) + } + // expect closing ')' but no need to complain if missing + if p.tok == token.RPAREN { + p.next() + } + + default: + pos := p.pos + p.errorExpected(pos, "field name or embedded type") + p.advance(exprEnd) + typ = &ast.BadExpr{From: pos, To: p.pos} + } + + var tag *ast.BasicLit + if p.tok == token.STRING { + tag = &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + p.next() + } + + comment := p.expectSemi() + + field := &ast.Field{Doc: doc, Names: names, Type: typ, Tag: tag, Comment: comment} + return field +} + +func (p *parser) parseStructType() *ast.StructType { + if p.trace { + defer un(trace(p, "StructType")) + } + + pos := p.expect(token.STRUCT) + lbrace := p.expect(token.LBRACE) + var list []*ast.Field + for p.tok == token.IDENT || p.tok == token.MUL || p.tok == token.LPAREN { + // a field declaration cannot start with a '(' but we accept + // it here for more robust parsing and better error messages + // (parseFieldDecl will check and complain if necessary) + list = append(list, p.parseFieldDecl()) + } + rbrace := p.expect(token.RBRACE) + + return &ast.StructType{ + Struct: pos, + Fields: &ast.FieldList{ + Opening: lbrace, + List: list, + Closing: rbrace, + }, + } +} + +func (p *parser) parsePointerType() *ast.StarExpr { + if p.trace { + defer un(trace(p, "PointerType")) + } + + star := p.expect(token.MUL) + base := p.parseType() + + return &ast.StarExpr{Star: star, X: base} +} + +func (p *parser) parseDotsType() *ast.Ellipsis { + if p.trace { + defer un(trace(p, "DotsType")) + } + + pos := p.expect(token.ELLIPSIS) + elt := p.parseType() + + return &ast.Ellipsis{Ellipsis: pos, Elt: elt} +} + +type field struct { + name *ast.Ident + typ ast.Expr +} + +func (p *parser) parseParamDecl(name *ast.Ident, typeSetsOK bool) (f field) { + // TODO(rFindley) refactor to be more similar to paramDeclOrNil in the syntax + // package + if p.trace { + defer un(trace(p, "ParamDeclOrNil")) + } + + ptok := p.tok + if name != nil { + p.tok = token.IDENT // force token.IDENT case in switch below + } else if typeSetsOK && p.tok == token.TILDE { + // "~" ... + return field{nil, p.embeddedElem(nil)} + } + + switch p.tok { + case token.IDENT: + // name + if name != nil { + f.name = name + p.tok = ptok + } else { + f.name = p.parseIdent() + } + switch p.tok { + case token.IDENT, token.MUL, token.ARROW, token.FUNC, token.CHAN, token.MAP, token.STRUCT, token.INTERFACE, token.LPAREN: + // name type + f.typ = p.parseType() + + case token.LBRACK: + // name "[" type1, ..., typeN "]" or name "[" n "]" type + f.name, f.typ = p.parseArrayFieldOrTypeInstance(f.name) + + case token.ELLIPSIS: + // name "..." type + f.typ = p.parseDotsType() + return // don't allow ...type "|" ... + + case token.PERIOD: + // name "." ... + f.typ = p.parseQualifiedIdent(f.name) + f.name = nil + + case token.TILDE: + if typeSetsOK { + f.typ = p.embeddedElem(nil) + return + } + + case token.OR: + if typeSetsOK { + // name "|" typeset + f.typ = p.embeddedElem(f.name) + f.name = nil + return + } + } + + case token.MUL, token.ARROW, token.FUNC, token.LBRACK, token.CHAN, token.MAP, token.STRUCT, token.INTERFACE, token.LPAREN: + // type + f.typ = p.parseType() + + case token.ELLIPSIS: + // "..." type + // (always accepted) + f.typ = p.parseDotsType() + return // don't allow ...type "|" ... + + default: + // TODO(rfindley): this is incorrect in the case of type parameter lists + // (should be "']'" in that case) + p.errorExpected(p.pos, "')'") + p.advance(exprEnd) + } + + // [name] type "|" + if typeSetsOK && p.tok == token.OR && f.typ != nil { + f.typ = p.embeddedElem(f.typ) + } + + return +} + +func (p *parser) parseParameterList(name0 *ast.Ident, typ0 ast.Expr, closing token.Token) (params []*ast.Field) { + if p.trace { + defer un(trace(p, "ParameterList")) + } + + // Type parameters are the only parameter list closed by ']'. + tparams := closing == token.RBRACK + + pos0 := p.pos + if name0 != nil { + pos0 = name0.Pos() + } else if typ0 != nil { + pos0 = typ0.Pos() + } + + // Note: The code below matches the corresponding code in the syntax + // parser closely. Changes must be reflected in either parser. + // For the code to match, we use the local []field list that + // corresponds to []syntax.Field. At the end, the list must be + // converted into an []*ast.Field. + + var list []field + var named int // number of parameters that have an explicit name and type + var typed int // number of parameters that have an explicit type + + for name0 != nil || p.tok != closing && p.tok != token.EOF { + var par field + if typ0 != nil { + if tparams { + typ0 = p.embeddedElem(typ0) + } + par = field{name0, typ0} + } else { + par = p.parseParamDecl(name0, tparams) + } + name0 = nil // 1st name was consumed if present + typ0 = nil // 1st typ was consumed if present + if par.name != nil || par.typ != nil { + list = append(list, par) + if par.name != nil && par.typ != nil { + named++ + } + if par.typ != nil { + typed++ + } + } + if !p.atComma("parameter list", closing) { + break + } + p.next() + } + + if len(list) == 0 { + return // not uncommon + } + + // distribute parameter types (len(list) > 0) + if named == 0 { + // all unnamed => found names are type names + for i := 0; i < len(list); i++ { + par := &list[i] + if typ := par.name; typ != nil { + par.typ = typ + par.name = nil + } + } + if tparams { + // This is the same error handling as below, adjusted for type parameters only. + // See comment below for details. (go.dev/issue/64534) + var errPos token.Pos + var msg string + if named == typed /* same as typed == 0 */ { + errPos = p.pos // position error at closing ] + msg = "missing type constraint" + } else { + errPos = pos0 // position at opening [ or first name + msg = "missing type parameter name" + if len(list) == 1 { + msg += " or invalid array length" + } + } + p.error(errPos, msg) + } + } else if named != len(list) { + // some named or we're in a type parameter list => all must be named + var errPos token.Pos // left-most error position (or invalid) + var typ ast.Expr // current type (from right to left) + for i := len(list) - 1; i >= 0; i-- { + if par := &list[i]; par.typ != nil { + typ = par.typ + if par.name == nil { + errPos = typ.Pos() + n := ast.NewIdent("_") + n.NamePos = errPos // correct position + par.name = n + } + } else if typ != nil { + par.typ = typ + } else { + // par.typ == nil && typ == nil => we only have a par.name + errPos = par.name.Pos() + par.typ = &ast.BadExpr{From: errPos, To: p.pos} + } + } + if errPos.IsValid() { + var msg string + if tparams { + // Not all parameters are named because named != len(list). + // If named == typed we must have parameters that have no types, + // and they must be at the end of the parameter list, otherwise + // the types would have been filled in by the right-to-left sweep + // above and we wouldn't have an error. Since we are in a type + // parameter list, the missing types are constraints. + if named == typed { + errPos = p.pos // position error at closing ] + msg = "missing type constraint" + } else { + msg = "missing type parameter name" + // go.dev/issue/60812 + if len(list) == 1 { + msg += " or invalid array length" + } + } + } else { + msg = "mixed named and unnamed parameters" + } + p.error(errPos, msg) + } + } + + // Convert list to []*ast.Field. + // If list contains types only, each type gets its own ast.Field. + if named == 0 { + // parameter list consists of types only + for _, par := range list { + assert(par.typ != nil, "nil type in unnamed parameter list") + params = append(params, &ast.Field{Type: par.typ}) + } + return + } + + // If the parameter list consists of named parameters with types, + // collect all names with the same types into a single ast.Field. + var names []*ast.Ident + var typ ast.Expr + addParams := func() { + assert(typ != nil, "nil type in named parameter list") + field := &ast.Field{Names: names, Type: typ} + params = append(params, field) + names = nil + } + for _, par := range list { + if par.typ != typ { + if len(names) > 0 { + addParams() + } + typ = par.typ + } + names = append(names, par.name) + } + if len(names) > 0 { + addParams() + } + return +} + +func (p *parser) parseParameters(acceptTParams bool) (tparams, params *ast.FieldList) { + if p.trace { + defer un(trace(p, "Parameters")) + } + + if acceptTParams && p.tok == token.LBRACK { + opening := p.pos + p.next() + // [T any](params) syntax + list := p.parseParameterList(nil, nil, token.RBRACK) + rbrack := p.expect(token.RBRACK) + tparams = &ast.FieldList{Opening: opening, List: list, Closing: rbrack} + // Type parameter lists must not be empty. + if tparams.NumFields() == 0 { + p.error(tparams.Closing, "empty type parameter list") + tparams = nil // avoid follow-on errors + } + } + + opening := p.expect(token.LPAREN) + + var fields []*ast.Field + if p.tok != token.RPAREN { + fields = p.parseParameterList(nil, nil, token.RPAREN) + } + + rparen := p.expect(token.RPAREN) + params = &ast.FieldList{Opening: opening, List: fields, Closing: rparen} + + return +} + +func (p *parser) parseResult() *ast.FieldList { + if p.trace { + defer un(trace(p, "Result")) + } + + if p.tok == token.LPAREN { + _, results := p.parseParameters(false) + return results + } + + typ := p.tryIdentOrType() + if typ != nil { + list := make([]*ast.Field, 1) + list[0] = &ast.Field{Type: typ} + return &ast.FieldList{List: list} + } + + return nil +} + +func (p *parser) parseFuncType() *ast.FuncType { + if p.trace { + defer un(trace(p, "FuncType")) + } + + pos := p.expect(token.FUNC) + tparams, params := p.parseParameters(true) + if tparams != nil { + p.error(tparams.Pos(), "function type must have no type parameters") + } + results := p.parseResult() + + return &ast.FuncType{Func: pos, Params: params, Results: results} +} + +func (p *parser) parseMethodSpec() *ast.Field { + if p.trace { + defer un(trace(p, "MethodSpec")) + } + + doc := p.leadComment + var idents []*ast.Ident + var typ ast.Expr + x := p.parseTypeName(nil) + if ident, _ := x.(*ast.Ident); ident != nil { + switch { + case p.tok == token.LBRACK: + // generic method or embedded instantiated type + lbrack := p.pos + p.next() + p.exprLev++ + x := p.parseExpr() + p.exprLev-- + if name0, _ := x.(*ast.Ident); name0 != nil && p.tok != token.COMMA && p.tok != token.RBRACK { + // generic method m[T any] + // + // Interface methods do not have type parameters. We parse them for a + // better error message and improved error recovery. + _ = p.parseParameterList(name0, nil, token.RBRACK) + _ = p.expect(token.RBRACK) + p.error(lbrack, "interface method must have no type parameters") + + // TODO(rfindley) refactor to share code with parseFuncType. + _, params := p.parseParameters(false) + results := p.parseResult() + idents = []*ast.Ident{ident} + typ = &ast.FuncType{ + Func: token.NoPos, + Params: params, + Results: results, + } + } else { + // embedded instantiated type + // TODO(rfindley) should resolve all identifiers in x. + list := []ast.Expr{x} + if p.atComma("type argument list", token.RBRACK) { + p.exprLev++ + p.next() + for p.tok != token.RBRACK && p.tok != token.EOF { + list = append(list, p.parseType()) + if !p.atComma("type argument list", token.RBRACK) { + break + } + p.next() + } + p.exprLev-- + } + // rbrack := p.expectClosing(token.RBRACK, "type argument list") + // typ = typeparams.PackIndexExpr(ident, lbrack, list, rbrack) + } + case p.tok == token.LPAREN: + // ordinary method + // TODO(rfindley) refactor to share code with parseFuncType. + _, params := p.parseParameters(false) + results := p.parseResult() + idents = []*ast.Ident{ident} + typ = &ast.FuncType{Func: token.NoPos, Params: params, Results: results} + default: + // embedded type + typ = x + } + } else { + // embedded, possibly instantiated type + typ = x + if p.tok == token.LBRACK { + // embedded instantiated interface + typ = p.parseTypeInstance(typ) + } + } + + // Comment is added at the callsite: the field below may joined with + // additional type specs using '|'. + // TODO(rfindley) this should be refactored. + // TODO(rfindley) add more tests for comment handling. + return &ast.Field{Doc: doc, Names: idents, Type: typ} +} + +func (p *parser) embeddedElem(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "EmbeddedElem")) + } + if x == nil { + x = p.embeddedTerm() + } + for p.tok == token.OR { + t := new(ast.BinaryExpr) + t.OpPos = p.pos + t.Op = token.OR + p.next() + t.X = x + t.Y = p.embeddedTerm() + x = t + } + return x +} + +func (p *parser) embeddedTerm() ast.Expr { + if p.trace { + defer un(trace(p, "EmbeddedTerm")) + } + if p.tok == token.TILDE { + t := new(ast.UnaryExpr) + t.OpPos = p.pos + t.Op = token.TILDE + p.next() + t.X = p.parseType() + return t + } + + t := p.tryIdentOrType() + if t == nil { + pos := p.pos + p.errorExpected(pos, "~ term or type") + p.advance(exprEnd) + return &ast.BadExpr{From: pos, To: p.pos} + } + + return t +} + +func (p *parser) parseInterfaceType() *ast.InterfaceType { + if p.trace { + defer un(trace(p, "InterfaceType")) + } + + pos := p.expect(token.INTERFACE) + lbrace := p.expect(token.LBRACE) + + var list []*ast.Field + +parseElements: + for { + switch { + case p.tok == token.IDENT: + f := p.parseMethodSpec() + if f.Names == nil { + f.Type = p.embeddedElem(f.Type) + } + f.Comment = p.expectSemi() + list = append(list, f) + case p.tok == token.TILDE: + typ := p.embeddedElem(nil) + comment := p.expectSemi() + list = append(list, &ast.Field{Type: typ, Comment: comment}) + default: + if t := p.tryIdentOrType(); t != nil { + typ := p.embeddedElem(t) + comment := p.expectSemi() + list = append(list, &ast.Field{Type: typ, Comment: comment}) + } else { + break parseElements + } + } + } + + // TODO(rfindley): the error produced here could be improved, since we could + // accept an identifier, 'type', or a '}' at this point. + rbrace := p.expect(token.RBRACE) + + return &ast.InterfaceType{ + Interface: pos, + Methods: &ast.FieldList{ + Opening: lbrace, + List: list, + Closing: rbrace, + }, + } +} + +func (p *parser) parseMapType() *ast.MapType { + if p.trace { + defer un(trace(p, "MapType")) + } + + pos := p.expect(token.MAP) + p.expect(token.LBRACK) + key := p.parseType() + p.expect(token.RBRACK) + value := p.parseType() + + return &ast.MapType{Map: pos, Key: key, Value: value} +} + +func (p *parser) parseChanType() *ast.ChanType { + if p.trace { + defer un(trace(p, "ChanType")) + } + + pos := p.pos + dir := ast.SEND | ast.RECV + var arrow token.Pos + if p.tok == token.CHAN { + p.next() + if p.tok == token.ARROW { + arrow = p.pos + p.next() + dir = ast.SEND + } + } else { + arrow = p.expect(token.ARROW) + p.expect(token.CHAN) + dir = ast.RECV + } + value := p.parseType() + + return &ast.ChanType{Begin: pos, Arrow: arrow, Dir: dir, Value: value} +} + +func (p *parser) parseTypeInstance(typ ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeInstance")) + } + + opening := p.expect(token.LBRACK) + p.exprLev++ + var list []ast.Expr + for p.tok != token.RBRACK && p.tok != token.EOF { + list = append(list, p.parseType()) + if !p.atComma("type argument list", token.RBRACK) { + break + } + p.next() + } + p.exprLev-- + + closing := p.expectClosing(token.RBRACK, "type argument list") + + if len(list) == 0 { + p.errorExpected(closing, "type argument list") + return &ast.IndexExpr{ + X: typ, + Lbrack: opening, + Index: &ast.BadExpr{From: opening + 1, To: closing}, + Rbrack: closing, + } + } + + return nil // typeparams.PackIndexExpr(typ, opening, list, closing) +} + +func (p *parser) tryIdentOrType() ast.Expr { + defer decNestLev(incNestLev(p)) + + switch p.tok { + case token.IDENT: + typ := p.parseTypeName(nil) + if p.tok == token.LBRACK { + typ = p.parseTypeInstance(typ) + } + return typ + case token.LBRACK: + lbrack := p.expect(token.LBRACK) + return p.parseArrayType(lbrack, nil) + case token.STRUCT: + return p.parseStructType() + case token.MUL: + return p.parsePointerType() + case token.FUNC: + return p.parseFuncType() + case token.INTERFACE: + return p.parseInterfaceType() + case token.MAP: + return p.parseMapType() + case token.CHAN, token.ARROW: + return p.parseChanType() + case token.LPAREN: + lparen := p.pos + p.next() + typ := p.parseType() + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{Lparen: lparen, X: typ, Rparen: rparen} + } + + // no type found + return nil +} + +// ---------------------------------------------------------------------------- +// Blocks + +func (p *parser) parseStmtList() (list []ast.Stmt) { + if p.trace { + defer un(trace(p, "StatementList")) + } + + for p.tok != token.CASE && p.tok != token.DEFAULT && p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseStmt()) + } + + return +} + +func (p *parser) parseBody() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "Body")) + } + + lbrace := p.expect(token.LBRACE) + list := p.parseStmtList() + rbrace := p.expect2(token.RBRACE) + + return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} +} + +func (p *parser) parseBlockStmt() *ast.BlockStmt { + if p.trace { + defer un(trace(p, "BlockStmt")) + } + + lbrace := p.expect(token.LBRACE) + list := p.parseStmtList() + rbrace := p.expect2(token.RBRACE) + + return &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} +} + +// ---------------------------------------------------------------------------- +// Expressions + +func (p *parser) parseFuncTypeOrLit() ast.Expr { + if p.trace { + defer un(trace(p, "FuncTypeOrLit")) + } + + typ := p.parseFuncType() + if p.tok != token.LBRACE { + // function type only + return typ + } + + p.exprLev++ + body := p.parseBody() + p.exprLev-- + + return &ast.FuncLit{Type: typ, Body: body} +} + +// parseOperand may return an expression or a raw type (incl. array +// types of the form [...]T). Callers must verify the result. +func (p *parser) parseOperand() ast.Expr { + if p.trace { + defer un(trace(p, "Operand")) + } + + switch p.tok { + case token.IDENT: + x := p.parseIdent() + return x + + case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: + x := &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + p.next() + return x + + case token.LPAREN: + lparen := p.pos + p.next() + p.exprLev++ + x := p.parseRhs() // types may be parenthesized: (some type) + p.exprLev-- + rparen := p.expect(token.RPAREN) + return &ast.ParenExpr{Lparen: lparen, X: x, Rparen: rparen} + + case token.FUNC: + return p.parseFuncTypeOrLit() + } + + if typ := p.tryIdentOrType(); typ != nil { // do not consume trailing type parameters + // could be type for composite literal or conversion + _, isIdent := typ.(*ast.Ident) + assert(!isIdent, "type cannot be identifier") + return typ + } + + // we have an error + pos := p.pos + p.errorExpected(pos, "operand") + p.advance(stmtStart) + return &ast.BadExpr{From: pos, To: p.pos} +} + +func (p *parser) parseSelector(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "Selector")) + } + + sel := p.parseIdent() + + return &ast.SelectorExpr{X: x, Sel: sel} +} + +func (p *parser) parseTypeAssertion(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "TypeAssertion")) + } + + lparen := p.expect(token.LPAREN) + var typ ast.Expr + if p.tok == token.TYPE { + // type switch: typ == nil + p.next() + } else { + typ = p.parseType() + } + rparen := p.expect(token.RPAREN) + + return &ast.TypeAssertExpr{X: x, Type: typ, Lparen: lparen, Rparen: rparen} +} + +func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "parseIndexOrSliceOrInstance")) + } + + lbrack := p.expect(token.LBRACK) + if p.tok == token.RBRACK { + // empty index, slice or index expressions are not permitted; + // accept them for parsing tolerance, but complain + p.errorExpected(p.pos, "operand") + rbrack := p.pos + p.next() + return &ast.IndexExpr{ + X: x, + Lbrack: lbrack, + Index: &ast.BadExpr{From: rbrack, To: rbrack}, + Rbrack: rbrack, + } + } + p.exprLev++ + + const N = 3 // change the 3 to 2 to disable 3-index slices + var args []ast.Expr + var index [N]ast.Expr + var colons [N - 1]token.Pos + if p.tok != token.COLON { + // We can't know if we have an index expression or a type instantiation; + // so even if we see a (named) type we are not going to be in type context. + index[0] = p.parseRhs() + } + ncolons := 0 + switch p.tok { + case token.COLON: + // slice expression + for p.tok == token.COLON && ncolons < len(colons) { + colons[ncolons] = p.pos + ncolons++ + p.next() + if p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF { + index[ncolons] = p.parseRhs() + } + } + case token.COMMA: + // instance expression + args = append(args, index[0]) + for p.tok == token.COMMA { + p.next() + if p.tok != token.RBRACK && p.tok != token.EOF { + args = append(args, p.parseType()) + } + } + } + + p.exprLev-- + rbrack := p.expect(token.RBRACK) + + if ncolons > 0 { + // slice expression + slice3 := false + if ncolons == 2 { + slice3 = true + // Check presence of middle and final index here rather than during type-checking + // to prevent erroneous programs from passing through gofmt (was go.dev/issue/7305). + if index[1] == nil { + p.error(colons[0], "middle index required in 3-index slice") + index[1] = &ast.BadExpr{From: colons[0] + 1, To: colons[1]} + } + if index[2] == nil { + p.error(colons[1], "final index required in 3-index slice") + index[2] = &ast.BadExpr{From: colons[1] + 1, To: rbrack} + } + } + return &ast.SliceExpr{X: x, Lbrack: lbrack, Low: index[0], High: index[1], Max: index[2], Slice3: slice3, Rbrack: rbrack} + } + + if len(args) == 0 { + // index expression + return &ast.IndexExpr{X: x, Lbrack: lbrack, Index: index[0], Rbrack: rbrack} + } + + // instance expression + return nil // typeparams.PackIndexExpr(x, lbrack, args, rbrack) +} + +func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { + if p.trace { + defer un(trace(p, "CallOrConversion")) + } + + lparen := p.expect(token.LPAREN) + p.exprLev++ + var list []ast.Expr + var ellipsis token.Pos + for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() { + list = append(list, p.parseRhs()) // builtins may expect a type: make(some type, ...) + if p.tok == token.ELLIPSIS { + ellipsis = p.pos + p.next() + } + if !p.atComma("argument list", token.RPAREN) { + break + } + p.next() + } + p.exprLev-- + rparen := p.expectClosing(token.RPAREN, "argument list") + + return &ast.CallExpr{Fun: fun, Lparen: lparen, Args: list, Ellipsis: ellipsis, Rparen: rparen} +} + +func (p *parser) parseValue() ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + if p.tok == token.LBRACE { + return p.parseLiteralValue(nil) + } + + x := p.parseExpr() + + return x +} + +func (p *parser) parseElement() ast.Expr { + if p.trace { + defer un(trace(p, "Element")) + } + + x := p.parseValue() + if p.tok == token.COLON { + colon := p.pos + p.next() + x = &ast.KeyValueExpr{Key: x, Colon: colon, Value: p.parseValue()} + } + + return x +} + +func (p *parser) parseElementList() (list []ast.Expr) { + if p.trace { + defer un(trace(p, "ElementList")) + } + + for p.tok != token.RBRACE && p.tok != token.EOF { + list = append(list, p.parseElement()) + if !p.atComma("composite literal", token.RBRACE) { + break + } + p.next() + } + + return +} + +func (p *parser) parseLiteralValue(typ ast.Expr) ast.Expr { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "LiteralValue")) + } + + lbrace := p.expect(token.LBRACE) + var elts []ast.Expr + p.exprLev++ + if p.tok != token.RBRACE { + elts = p.parseElementList() + } + p.exprLev-- + rbrace := p.expectClosing(token.RBRACE, "composite literal") + return &ast.CompositeLit{Type: typ, Lbrace: lbrace, Elts: elts, Rbrace: rbrace} +} + +func (p *parser) parsePrimaryExpr(x ast.Expr) ast.Expr { + if p.trace { + defer un(trace(p, "PrimaryExpr")) + } + + if x == nil { + x = p.parseOperand() + } + // We track the nesting here rather than at the entry for the function, + // since it can iteratively produce a nested output, and we want to + // limit how deep a structure we generate. + var n int + defer func() { p.nestLev -= n }() + for n = 1; ; n++ { + incNestLev(p) + switch p.tok { + case token.PERIOD: + p.next() + switch p.tok { + case token.IDENT: + x = p.parseSelector(x) + case token.LPAREN: + x = p.parseTypeAssertion(x) + default: + pos := p.pos + p.errorExpected(pos, "selector or type assertion") + // TODO(rFindley) The check for token.RBRACE below is a targeted fix + // to error recovery sufficient to make the x/tools tests to + // pass with the new parsing logic introduced for type + // parameters. Remove this once error recovery has been + // more generally reconsidered. + if p.tok != token.RBRACE { + p.next() // make progress + } + sel := &ast.Ident{NamePos: pos, Name: "_"} + x = &ast.SelectorExpr{X: x, Sel: sel} + } + case token.LBRACK: + x = p.parseIndexOrSliceOrInstance(x) + case token.LPAREN: + x = p.parseCallOrConversion(x) + case token.LBRACE: + // operand may have returned a parenthesized complit + // type; accept it but complain if we have a complit + t := ast.Unparen(x) + // determine if '{' belongs to a composite literal or a block statement + switch t.(type) { + case *ast.BadExpr, *ast.Ident, *ast.SelectorExpr: + if p.exprLev < 0 { + return x + } + // x is possibly a composite literal type + case *ast.IndexExpr, *ast.IndexListExpr: + if p.exprLev < 0 { + return x + } + // x is possibly a composite literal type + case *ast.ArrayType, *ast.StructType, *ast.MapType: + // x is a composite literal type + default: + return x + } + if t != x { + p.error(t.Pos(), "cannot parenthesize type in composite literal") + // already progressed, no need to advance + } + x = p.parseLiteralValue(x) + default: + return x + } + } +} + +func (p *parser) parseUnaryExpr() ast.Expr { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "UnaryExpr")) + } + + switch p.tok { + case token.ADD, token.SUB, token.NOT, token.XOR, token.AND, token.TILDE: + pos, op := p.pos, p.tok + p.next() + x := p.parseUnaryExpr() + return &ast.UnaryExpr{OpPos: pos, Op: op, X: x} + + case token.ARROW: + // channel type or receive expression + arrow := p.pos + p.next() + + // If the next token is token.CHAN we still don't know if it + // is a channel type or a receive operation - we only know + // once we have found the end of the unary expression. There + // are two cases: + // + // <- type => (<-type) must be channel type + // <- expr => <-(expr) is a receive from an expression + // + // In the first case, the arrow must be re-associated with + // the channel type parsed already: + // + // <- (chan type) => (<-chan type) + // <- (chan<- type) => (<-chan (<-type)) + + x := p.parseUnaryExpr() + + // determine which case we have + if typ, ok := x.(*ast.ChanType); ok { + // (<-type) + + // re-associate position info and <- + dir := ast.SEND + for ok && dir == ast.SEND { + if typ.Dir == ast.RECV { + // error: (<-type) is (<-(<-chan T)) + p.errorExpected(typ.Arrow, "'chan'") + } + arrow, typ.Begin, typ.Arrow = typ.Arrow, arrow, arrow + dir, typ.Dir = typ.Dir, ast.RECV + typ, ok = typ.Value.(*ast.ChanType) + } + if dir == ast.SEND { + p.errorExpected(arrow, "channel type") + } + + return x + } + + // <-(expr) + return &ast.UnaryExpr{OpPos: arrow, Op: token.ARROW, X: x} + + case token.MUL: + // pointer type or unary "*" expression + pos := p.pos + p.next() + x := p.parseUnaryExpr() + return &ast.StarExpr{Star: pos, X: x} + } + + return p.parsePrimaryExpr(nil) +} + +func (p *parser) tokPrec() (token.Token, int) { + tok := p.tok + if p.inRhs && tok == token.ASSIGN { + tok = token.EQL + } + return tok, tok.Precedence() +} + +// parseBinaryExpr parses a (possibly) binary expression. +// If x is non-nil, it is used as the left operand. +// +// TODO(rfindley): parseBinaryExpr has become overloaded. Consider refactoring. +func (p *parser) parseBinaryExpr(x ast.Expr, prec1 int) ast.Expr { + if p.trace { + defer un(trace(p, "BinaryExpr")) + } + + if x == nil { + x = p.parseUnaryExpr() + } + // We track the nesting here rather than at the entry for the function, + // since it can iteratively produce a nested output, and we want to + // limit how deep a structure we generate. + var n int + defer func() { p.nestLev -= n }() + for n = 1; ; n++ { + incNestLev(p) + op, oprec := p.tokPrec() + if oprec < prec1 { + return x + } + pos := p.expect(op) + y := p.parseBinaryExpr(nil, oprec+1) + x = &ast.BinaryExpr{X: x, OpPos: pos, Op: op, Y: y} + } +} + +// The result may be a type or even a raw type ([...]int). +func (p *parser) parseExpr() ast.Expr { + if p.trace { + defer un(trace(p, "Expression")) + } + + return p.parseBinaryExpr(nil, token.LowestPrec+1) +} + +func (p *parser) parseRhs() ast.Expr { + old := p.inRhs + p.inRhs = true + x := p.parseExpr() + p.inRhs = old + return x +} + +// ---------------------------------------------------------------------------- +// Statements + +// Parsing modes for parseSimpleStmt. +const ( + basic = iota + labelOk + rangeOk +) + +// parseSimpleStmt returns true as 2nd result if it parsed the assignment +// of a range clause (with mode == rangeOk). The returned statement is an +// assignment with a right-hand side that is a single unary expression of +// the form "range x". No guarantees are given for the left-hand side. +func (p *parser) parseSimpleStmt(mode int) (ast.Stmt, bool) { + if p.trace { + defer un(trace(p, "SimpleStmt")) + } + + x := p.parseList(false) + + switch p.tok { + case + token.DEFINE, token.ASSIGN, token.ADD_ASSIGN, + token.SUB_ASSIGN, token.MUL_ASSIGN, token.QUO_ASSIGN, + token.REM_ASSIGN, token.AND_ASSIGN, token.OR_ASSIGN, + token.XOR_ASSIGN, token.SHL_ASSIGN, token.SHR_ASSIGN, token.AND_NOT_ASSIGN: + // assignment statement, possibly part of a range clause + pos, tok := p.pos, p.tok + p.next() + var y []ast.Expr + isRange := false + if mode == rangeOk && p.tok == token.RANGE && (tok == token.DEFINE || tok == token.ASSIGN) { + pos := p.pos + p.next() + y = []ast.Expr{&ast.UnaryExpr{OpPos: pos, Op: token.RANGE, X: p.parseRhs()}} + isRange = true + } else { + y = p.parseList(true) + } + return &ast.AssignStmt{Lhs: x, TokPos: pos, Tok: tok, Rhs: y}, isRange + } + + if len(x) > 1 { + p.errorExpected(x[0].Pos(), "1 expression") + // continue with first expression + } + + switch p.tok { + case token.COLON: + // labeled statement + colon := p.pos + p.next() + if label, isIdent := x[0].(*ast.Ident); mode == labelOk && isIdent { + // Go spec: The scope of a label is the body of the function + // in which it is declared and excludes the body of any nested + // function. + stmt := &ast.LabeledStmt{Label: label, Colon: colon, Stmt: p.parseStmt()} + return stmt, false + } + // The label declaration typically starts at x[0].Pos(), but the label + // declaration may be erroneous due to a token after that position (and + // before the ':'). If SpuriousErrors is not set, the (only) error + // reported for the line is the illegal label error instead of the token + // before the ':' that caused the problem. Thus, use the (latest) colon + // position for error reporting. + p.error(colon, "illegal label declaration") + return &ast.BadStmt{From: x[0].Pos(), To: colon + 1}, false + + case token.ARROW: + // send statement + arrow := p.pos + p.next() + y := p.parseRhs() + return &ast.SendStmt{Chan: x[0], Arrow: arrow, Value: y}, false + + case token.INC, token.DEC: + // increment or decrement + s := &ast.IncDecStmt{X: x[0], TokPos: p.pos, Tok: p.tok} + p.next() + return s, false + } + + // expression + return &ast.ExprStmt{X: x[0]}, false +} + +func (p *parser) parseCallExpr(callType string) *ast.CallExpr { + x := p.parseRhs() // could be a conversion: (some type)(x) + if t := ast.Unparen(x); t != x { + p.error(x.Pos(), fmt.Sprintf("expression in %s must not be parenthesized", callType)) + x = t + } + if call, isCall := x.(*ast.CallExpr); isCall { + return call + } + if _, isBad := x.(*ast.BadExpr); !isBad { + // only report error if it's a new one + p.error(p.safePos(x.End()), fmt.Sprintf("expression in %s must be function call", callType)) + } + return nil +} + +func (p *parser) parseGoStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "GoStmt")) + } + + pos := p.expect(token.GO) + call := p.parseCallExpr("go") + p.expectSemi() + if call == nil { + return &ast.BadStmt{From: pos, To: pos + 2} // len("go") + } + + return &ast.GoStmt{Go: pos, Call: call} +} + +func (p *parser) parseDeferStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "DeferStmt")) + } + + pos := p.expect(token.DEFER) + call := p.parseCallExpr("defer") + p.expectSemi() + if call == nil { + return &ast.BadStmt{From: pos, To: pos + 5} // len("defer") + } + + return &ast.DeferStmt{Defer: pos, Call: call} +} + +func (p *parser) parseReturnStmt() *ast.ReturnStmt { + if p.trace { + defer un(trace(p, "ReturnStmt")) + } + + pos := p.pos + p.expect(token.RETURN) + var x []ast.Expr + if p.tok != token.SEMICOLON && p.tok != token.RBRACE { + x = p.parseList(true) + } + p.expectSemi() + + return &ast.ReturnStmt{Return: pos, Results: x} +} + +func (p *parser) parseBranchStmt(tok token.Token) *ast.BranchStmt { + if p.trace { + defer un(trace(p, "BranchStmt")) + } + + pos := p.expect(tok) + var label *ast.Ident + if tok != token.FALLTHROUGH && p.tok == token.IDENT { + label = p.parseIdent() + } + p.expectSemi() + + return &ast.BranchStmt{TokPos: pos, Tok: tok, Label: label} +} + +func (p *parser) makeExpr(s ast.Stmt, want string) ast.Expr { + if s == nil { + return nil + } + if es, isExpr := s.(*ast.ExprStmt); isExpr { + return es.X + } + found := "simple statement" + if _, isAss := s.(*ast.AssignStmt); isAss { + found = "assignment" + } + p.error(s.Pos(), fmt.Sprintf("expected %s, found %s (missing parentheses around composite literal?)", want, found)) + return &ast.BadExpr{From: s.Pos(), To: p.safePos(s.End())} +} + +// parseIfHeader is an adjusted version of parser.header +// in cmd/compile/internal/syntax/parser.go, which has +// been tuned for better error handling. +func (p *parser) parseIfHeader() (init ast.Stmt, cond ast.Expr) { + if p.tok == token.LBRACE { + p.error(p.pos, "missing condition in if statement") + cond = &ast.BadExpr{From: p.pos, To: p.pos} + return + } + // p.tok != token.LBRACE + + prevLev := p.exprLev + p.exprLev = -1 + + if p.tok != token.SEMICOLON { + // accept potential variable declaration but complain + if p.tok == token.VAR { + p.next() + p.error(p.pos, "var declaration not allowed in if initializer") + } + init, _ = p.parseSimpleStmt(basic) + } + + var condStmt ast.Stmt + var semi struct { + pos token.Pos + lit string // ";" or "\n"; valid if pos.IsValid() + } + if p.tok != token.LBRACE { + if p.tok == token.SEMICOLON { + semi.pos = p.pos + semi.lit = p.lit + p.next() + } else { + p.expect(token.SEMICOLON) + } + if p.tok != token.LBRACE { + condStmt, _ = p.parseSimpleStmt(basic) + } + } else { + condStmt = init + init = nil + } + + if condStmt != nil { + cond = p.makeExpr(condStmt, "boolean expression") + } else if semi.pos.IsValid() { + if semi.lit == "\n" { + p.error(semi.pos, "unexpected newline, expecting { after if clause") + } else { + p.error(semi.pos, "missing condition in if statement") + } + } + + // make sure we have a valid AST + if cond == nil { + cond = &ast.BadExpr{From: p.pos, To: p.pos} + } + + p.exprLev = prevLev + return +} + +func (p *parser) parseIfStmt() *ast.IfStmt { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "IfStmt")) + } + + pos := p.expect(token.IF) + + init, cond := p.parseIfHeader() + body := p.parseBlockStmt() + + var else_ ast.Stmt + if p.tok == token.ELSE { + p.next() + switch p.tok { + case token.IF: + else_ = p.parseIfStmt() + case token.LBRACE: + else_ = p.parseBlockStmt() + p.expectSemi() + default: + p.errorExpected(p.pos, "if statement or block") + else_ = &ast.BadStmt{From: p.pos, To: p.pos} + } + } else { + p.expectSemi() + } + + return &ast.IfStmt{If: pos, Init: init, Cond: cond, Body: body, Else: else_} +} + +func (p *parser) parseCaseClause() *ast.CaseClause { + if p.trace { + defer un(trace(p, "CaseClause")) + } + + pos := p.pos + var list []ast.Expr + if p.tok == token.CASE { + p.next() + list = p.parseList(true) + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + + return &ast.CaseClause{Case: pos, List: list, Colon: colon, Body: body} +} + +func isTypeSwitchAssert(x ast.Expr) bool { + a, ok := x.(*ast.TypeAssertExpr) + return ok && a.Type == nil +} + +func (p *parser) isTypeSwitchGuard(s ast.Stmt) bool { + switch t := s.(type) { + case *ast.ExprStmt: + // x.(type) + return isTypeSwitchAssert(t.X) + case *ast.AssignStmt: + // v := x.(type) + if len(t.Lhs) == 1 && len(t.Rhs) == 1 && isTypeSwitchAssert(t.Rhs[0]) { + switch t.Tok { + case token.ASSIGN: + // permit v = x.(type) but complain + p.error(t.TokPos, "expected ':=', found '='") + fallthrough + case token.DEFINE: + return true + } + } + } + return false +} + +func (p *parser) parseSwitchStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "SwitchStmt")) + } + + pos := p.expect(token.SWITCH) + + var s1, s2 ast.Stmt + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + s2, _ = p.parseSimpleStmt(basic) + } + if p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.LBRACE { + // A TypeSwitchGuard may declare a variable in addition + // to the variable declared in the initial SimpleStmt. + // Introduce extra scope to avoid redeclaration errors: + // + // switch t := 0; t := x.(T) { ... } + // + // (this code is not valid Go because the first t + // cannot be accessed and thus is never used, the extra + // scope is needed for the correct error message). + // + // If we don't have a type switch, s2 must be an expression. + // Having the extra nested but empty scope won't affect it. + s2, _ = p.parseSimpleStmt(basic) + } + } + p.exprLev = prevLev + } + + typeSwitch := p.isTypeSwitchGuard(s2) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCaseClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} + + if typeSwitch { + return &ast.TypeSwitchStmt{Switch: pos, Init: s1, Assign: s2, Body: body} + } + + return &ast.SwitchStmt{Switch: pos, Init: s1, Tag: p.makeExpr(s2, "switch expression"), Body: body} +} + +func (p *parser) parseCommClause() *ast.CommClause { + if p.trace { + defer un(trace(p, "CommClause")) + } + + pos := p.pos + var comm ast.Stmt + if p.tok == token.CASE { + p.next() + lhs := p.parseList(false) + if p.tok == token.ARROW { + // SendStmt + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + arrow := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.SendStmt{Chan: lhs[0], Arrow: arrow, Value: rhs} + } else { + // RecvStmt + if tok := p.tok; tok == token.ASSIGN || tok == token.DEFINE { + // RecvStmt with assignment + if len(lhs) > 2 { + p.errorExpected(lhs[0].Pos(), "1 or 2 expressions") + // continue with first two expressions + lhs = lhs[0:2] + } + pos := p.pos + p.next() + rhs := p.parseRhs() + comm = &ast.AssignStmt{Lhs: lhs, TokPos: pos, Tok: tok, Rhs: []ast.Expr{rhs}} + } else { + // lhs must be single receive operation + if len(lhs) > 1 { + p.errorExpected(lhs[0].Pos(), "1 expression") + // continue with first expression + } + comm = &ast.ExprStmt{X: lhs[0]} + } + } + } else { + p.expect(token.DEFAULT) + } + + colon := p.expect(token.COLON) + body := p.parseStmtList() + + return &ast.CommClause{Case: pos, Comm: comm, Colon: colon, Body: body} +} + +func (p *parser) parseSelectStmt() *ast.SelectStmt { + if p.trace { + defer un(trace(p, "SelectStmt")) + } + + pos := p.expect(token.SELECT) + lbrace := p.expect(token.LBRACE) + var list []ast.Stmt + for p.tok == token.CASE || p.tok == token.DEFAULT { + list = append(list, p.parseCommClause()) + } + rbrace := p.expect(token.RBRACE) + p.expectSemi() + body := &ast.BlockStmt{Lbrace: lbrace, List: list, Rbrace: rbrace} + + return &ast.SelectStmt{Select: pos, Body: body} +} + +func (p *parser) parseForStmt() ast.Stmt { + if p.trace { + defer un(trace(p, "ForStmt")) + } + + pos := p.expect(token.FOR) + + var s1, s2, s3 ast.Stmt + var isRange bool + if p.tok != token.LBRACE { + prevLev := p.exprLev + p.exprLev = -1 + if p.tok != token.SEMICOLON { + if p.tok == token.RANGE { + // "for range x" (nil lhs in assignment) + pos := p.pos + p.next() + y := []ast.Expr{&ast.UnaryExpr{OpPos: pos, Op: token.RANGE, X: p.parseRhs()}} + s2 = &ast.AssignStmt{Rhs: y} + isRange = true + } else { + s2, isRange = p.parseSimpleStmt(rangeOk) + } + } + if !isRange && p.tok == token.SEMICOLON { + p.next() + s1 = s2 + s2 = nil + if p.tok != token.SEMICOLON { + s2, _ = p.parseSimpleStmt(basic) + } + p.expectSemi() + if p.tok != token.LBRACE { + s3, _ = p.parseSimpleStmt(basic) + } + } + p.exprLev = prevLev + } + + body := p.parseBlockStmt() + p.expectSemi() + + if isRange { + as := s2.(*ast.AssignStmt) + // check lhs + var key, value ast.Expr + switch len(as.Lhs) { + case 0: + // nothing to do + case 1: + key = as.Lhs[0] + case 2: + key, value = as.Lhs[0], as.Lhs[1] + default: + p.errorExpected(as.Lhs[len(as.Lhs)-1].Pos(), "at most 2 expressions") + return &ast.BadStmt{From: pos, To: p.safePos(body.End())} + } + // parseSimpleStmt returned a right-hand side that + // is a single unary expression of the form "range x" + x := as.Rhs[0].(*ast.UnaryExpr).X + return &ast.RangeStmt{ + For: pos, + Key: key, + Value: value, + TokPos: as.TokPos, + Tok: as.Tok, + Range: as.Rhs[0].Pos(), + X: x, + Body: body, + } + } + + // regular for statement + return &ast.ForStmt{ + For: pos, + Init: s1, + Cond: p.makeExpr(s2, "boolean or range expression"), + Post: s3, + Body: body, + } +} + +func (p *parser) parseStmt() (s ast.Stmt) { + defer decNestLev(incNestLev(p)) + + if p.trace { + defer un(trace(p, "Statement")) + } + + switch p.tok { + case token.CONST, token.TYPE, token.VAR: + s = &ast.DeclStmt{Decl: p.parseDecl(stmtStart)} + case + // tokens that may start an expression + token.IDENT, token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING, token.FUNC, token.LPAREN, // operands + token.LBRACK, token.STRUCT, token.MAP, token.CHAN, token.INTERFACE, // composite types + token.ADD, token.SUB, token.MUL, token.AND, token.XOR, token.ARROW, token.NOT: // unary operators + s, _ = p.parseSimpleStmt(labelOk) + // because of the required look-ahead, labeled statements are + // parsed by parseSimpleStmt - don't expect a semicolon after + // them + if _, isLabeledStmt := s.(*ast.LabeledStmt); !isLabeledStmt { + p.expectSemi() + } + case token.GO: + s = p.parseGoStmt() + case token.DEFER: + s = p.parseDeferStmt() + case token.RETURN: + s = p.parseReturnStmt() + case token.BREAK, token.CONTINUE, token.GOTO, token.FALLTHROUGH: + s = p.parseBranchStmt(p.tok) + case token.LBRACE: + s = p.parseBlockStmt() + p.expectSemi() + case token.IF: + s = p.parseIfStmt() + case token.SWITCH: + s = p.parseSwitchStmt() + case token.SELECT: + s = p.parseSelectStmt() + case token.FOR: + s = p.parseForStmt() + case token.SEMICOLON: + // Is it ever possible to have an implicit semicolon + // producing an empty statement in a valid program? + // (handle correctly anyway) + s = &ast.EmptyStmt{Semicolon: p.pos, Implicit: p.lit == "\n"} + p.next() + case token.RBRACE: + // a semicolon may be omitted before a closing "}" + s = &ast.EmptyStmt{Semicolon: p.pos, Implicit: true} + default: + // no statement found + pos := p.pos + p.errorExpected(pos, "statement") + p.advance(stmtStart) + s = &ast.BadStmt{From: pos, To: p.pos} + } + + return +} + +// ---------------------------------------------------------------------------- +// Declarations + +type parseSpecFunction func(doc *ast.CommentGroup, keyword token.Token, iota int) ast.Spec + +func (p *parser) parseImportSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "ImportSpec")) + } + + var ident *ast.Ident + switch p.tok { + case token.IDENT: + ident = p.parseIdent() + case token.PERIOD: + ident = &ast.Ident{NamePos: p.pos, Name: "."} + p.next() + } + + pos := p.pos + var path string + if p.tok == token.STRING { + path = p.lit + p.next() + } else if p.tok.IsLiteral() { + p.error(pos, "import path must be a string") + p.next() + } else { + p.error(pos, "missing import path") + p.advance(exprEnd) + } + comment := p.expectSemi() + + // collect imports + spec := &ast.ImportSpec{ + Doc: doc, + Name: ident, + Path: &ast.BasicLit{ValuePos: pos, Kind: token.STRING, Value: path}, + Comment: comment, + } + p.imports = append(p.imports, spec) + + return spec +} + +func (p *parser) parseValueSpec(doc *ast.CommentGroup, keyword token.Token, iota int) ast.Spec { + if p.trace { + defer un(trace(p, keyword.String()+"Spec")) + } + + idents := p.parseIdentList() + var typ ast.Expr + var values []ast.Expr + switch keyword { + case token.CONST: + // always permit optional type and initialization for more tolerant parsing + if p.tok != token.EOF && p.tok != token.SEMICOLON && p.tok != token.RPAREN { + typ = p.tryIdentOrType() + if p.tok == token.ASSIGN { + p.next() + values = p.parseList(true) + } + } + case token.VAR: + if p.tok != token.ASSIGN { + typ = p.parseType() + } + if p.tok == token.ASSIGN { + p.next() + values = p.parseList(true) + } + default: + panic("unreachable") + } + comment := p.expectSemi() + + spec := &ast.ValueSpec{ + Doc: doc, + Names: idents, + Type: typ, + Values: values, + Comment: comment, + } + return spec +} + +func (p *parser) parseGenericType(spec *ast.TypeSpec, openPos token.Pos, name0 *ast.Ident, typ0 ast.Expr) { + if p.trace { + defer un(trace(p, "parseGenericType")) + } + + list := p.parseParameterList(name0, typ0, token.RBRACK) + closePos := p.expect(token.RBRACK) + spec.TypeParams = &ast.FieldList{Opening: openPos, List: list, Closing: closePos} + // Let the type checker decide whether to accept type parameters on aliases: + // see go.dev/issue/46477. + if p.tok == token.ASSIGN { + // type alias + spec.Assign = p.pos + p.next() + } + spec.Type = p.parseType() +} + +func (p *parser) parseTypeSpec(doc *ast.CommentGroup, _ token.Token, _ int) ast.Spec { + if p.trace { + defer un(trace(p, "TypeSpec")) + } + + name := p.parseIdent() + spec := &ast.TypeSpec{Doc: doc, Name: name} + + if p.tok == token.LBRACK { + // spec.Name "[" ... + // array/slice type or type parameter list + lbrack := p.pos + p.next() + if p.tok == token.IDENT { + // We may have an array type or a type parameter list. + // In either case we expect an expression x (which may + // just be a name, or a more complex expression) which + // we can analyze further. + // + // A type parameter list may have a type bound starting + // with a "[" as in: P []E. In that case, simply parsing + // an expression would lead to an error: P[] is invalid. + // But since index or slice expressions are never constant + // and thus invalid array length expressions, if the name + // is followed by "[" it must be the start of an array or + // slice constraint. Only if we don't see a "[" do we + // need to parse a full expression. Notably, name <- x + // is not a concern because name <- x is a statement and + // not an expression. + var x ast.Expr = p.parseIdent() + if p.tok != token.LBRACK { + // To parse the expression starting with name, expand + // the call sequence we would get by passing in name + // to parser.expr, and pass in name to parsePrimaryExpr. + p.exprLev++ + lhs := p.parsePrimaryExpr(x) + x = p.parseBinaryExpr(lhs, token.LowestPrec+1) + p.exprLev-- + } + // Analyze expression x. If we can split x into a type parameter + // name, possibly followed by a type parameter type, we consider + // this the start of a type parameter list, with some caveats: + // a single name followed by "]" tilts the decision towards an + // array declaration; a type parameter type that could also be + // an ordinary expression but which is followed by a comma tilts + // the decision towards a type parameter list. + if pname, ptype := extractName(x, p.tok == token.COMMA); pname != nil && (ptype != nil || p.tok != token.RBRACK) { + // spec.Name "[" pname ... + // spec.Name "[" pname ptype ... + // spec.Name "[" pname ptype "," ... + p.parseGenericType(spec, lbrack, pname, ptype) // ptype may be nil + } else { + // spec.Name "[" pname "]" ... + // spec.Name "[" x ... + spec.Type = p.parseArrayType(lbrack, x) + } + } else { + // array type + spec.Type = p.parseArrayType(lbrack, nil) + } + } else { + // no type parameters + if p.tok == token.ASSIGN { + // type alias + spec.Assign = p.pos + p.next() + } + spec.Type = p.parseType() + } + + spec.Comment = p.expectSemi() + + return spec +} + +// extractName splits the expression x into (name, expr) if syntactically +// x can be written as name expr. The split only happens if expr is a type +// element (per the isTypeElem predicate) or if force is set. +// If x is just a name, the result is (name, nil). If the split succeeds, +// the result is (name, expr). Otherwise the result is (nil, x). +// Examples: +// +// x force name expr +// ------------------------------------ +// P*[]int T/F P *[]int +// P*E T P *E +// P*E F nil P*E +// P([]int) T/F P []int +// P(E) T P E +// P(E) F nil P(E) +// P*E|F|~G T/F P *E|F|~G +// P*E|F|G T P *E|F|G +// P*E|F|G F nil P*E|F|G +func extractName(x ast.Expr, force bool) (*ast.Ident, ast.Expr) { + switch x := x.(type) { + case *ast.Ident: + return x, nil + case *ast.BinaryExpr: + switch x.Op { + case token.MUL: + if name, _ := x.X.(*ast.Ident); name != nil && (force || isTypeElem(x.Y)) { + // x = name *x.Y + return name, &ast.StarExpr{Star: x.OpPos, X: x.Y} + } + case token.OR: + if name, lhs := extractName(x.X, force || isTypeElem(x.Y)); name != nil && lhs != nil { + // x = name lhs|x.Y + op := *x + op.X = lhs + return name, &op + } + } + case *ast.CallExpr: + if name, _ := x.Fun.(*ast.Ident); name != nil { + if len(x.Args) == 1 && x.Ellipsis == token.NoPos && (force || isTypeElem(x.Args[0])) { + // x = name "(" x.ArgList[0] ")" + return name, x.Args[0] + } + } + } + return nil, x +} + +// isTypeElem reports whether x is a (possibly parenthesized) type element expression. +// The result is false if x could be a type element OR an ordinary (value) expression. +func isTypeElem(x ast.Expr) bool { + switch x := x.(type) { + case *ast.ArrayType, *ast.StructType, *ast.FuncType, *ast.InterfaceType, *ast.MapType, *ast.ChanType: + return true + case *ast.BinaryExpr: + return isTypeElem(x.X) || isTypeElem(x.Y) + case *ast.UnaryExpr: + return x.Op == token.TILDE + case *ast.ParenExpr: + return isTypeElem(x.X) + } + return false +} + +func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.GenDecl { + if p.trace { + defer un(trace(p, "GenDecl("+keyword.String()+")")) + } + + doc := p.leadComment + pos := p.expect(keyword) + var lparen, rparen token.Pos + var list []ast.Spec + if p.tok == token.LPAREN { + lparen = p.pos + p.next() + for iota := 0; p.tok != token.RPAREN && p.tok != token.EOF; iota++ { + list = append(list, f(p.leadComment, keyword, iota)) + } + rparen = p.expect(token.RPAREN) + p.expectSemi() + } else { + list = append(list, f(nil, keyword, 0)) + } + + return &ast.GenDecl{ + Doc: doc, + TokPos: pos, + Tok: keyword, + Lparen: lparen, + Specs: list, + Rparen: rparen, + } +} + +func (p *parser) parseFuncDecl() *ast.FuncDecl { + if p.trace { + defer un(trace(p, "FunctionDecl")) + } + + doc := p.leadComment + pos := p.expect(token.FUNC) + + var recv *ast.FieldList + if p.tok == token.LPAREN { + _, recv = p.parseParameters(false) + } + + ident := p.parseIdent() + + tparams, params := p.parseParameters(true) + if recv != nil && tparams != nil { + // Method declarations do not have type parameters. We parse them for a + // better error message and improved error recovery. + p.error(tparams.Opening, "method must have no type parameters") + tparams = nil + } + results := p.parseResult() + + var body *ast.BlockStmt + switch p.tok { + case token.LBRACE: + body = p.parseBody() + p.expectSemi() + case token.SEMICOLON: + p.next() + if p.tok == token.LBRACE { + // opening { of function declaration on next line + p.error(p.pos, "unexpected semicolon or newline before {") + body = p.parseBody() + p.expectSemi() + } + default: + p.expectSemi() + } + + decl := &ast.FuncDecl{ + Doc: doc, + Recv: recv, + Name: ident, + Type: &ast.FuncType{ + Func: pos, + TypeParams: tparams, + Params: params, + Results: results, + }, + Body: body, + } + return decl +} + +func (p *parser) parseDecl(sync map[token.Token]bool) ast.Decl { + if p.trace { + defer un(trace(p, "Declaration")) + } + + var f parseSpecFunction + switch p.tok { + case token.IMPORT: + f = p.parseImportSpec + + case token.CONST, token.VAR: + f = p.parseValueSpec + + case token.TYPE: + f = p.parseTypeSpec + + case token.FUNC: + return p.parseFuncDecl() + + default: + pos := p.pos + p.errorExpected(pos, "declaration") + p.advance(sync) + return &ast.BadDecl{From: pos, To: p.pos} + } + + return p.parseGenDecl(p.tok, f) +} + +// ---------------------------------------------------------------------------- +// Source files + +func (p *parser) parseFile() *ast.File { + if p.trace { + defer un(trace(p, "File")) + } + + // Don't bother parsing the rest if we had errors scanning the first token. + // Likely not a Go source file at all. + if p.errors.Len() != 0 { + return nil + } + + // package clause + doc := p.leadComment + pos := p.expect(token.PACKAGE) + // Go spec: The package clause is not a declaration; + // the package name does not appear in any scope. + ident := p.parseIdent() + if ident.Name == "_" && p.mode&DeclarationErrors != 0 { + p.error(p.pos, "invalid package name _") + } + p.expectSemi() + + // Don't bother parsing the rest if we had errors parsing the package clause. + // Likely not a Go source file at all. + if p.errors.Len() != 0 { + return nil + } + + var decls []ast.Decl + if p.mode&PackageClauseOnly == 0 { + // import decls + for p.tok == token.IMPORT { + decls = append(decls, p.parseGenDecl(token.IMPORT, p.parseImportSpec)) + } + + if p.mode&ImportsOnly == 0 { + // rest of package body + prev := token.IMPORT + for p.tok != token.EOF { + // Continue to accept import declarations for error tolerance, but complain. + if p.tok == token.IMPORT && prev != token.IMPORT { + p.error(p.pos, "imports must appear before other declarations") + } + prev = p.tok + + decls = append(decls, p.parseDecl(declStart)) + } + } + } + + f := &ast.File{ + Doc: doc, + Package: pos, + Name: ident, + Decls: decls, + FileStart: token.Pos(p.file.Base()), + FileEnd: token.Pos(p.file.Base() + p.file.Size()), + Imports: p.imports, + Comments: p.comments, + GoVersion: p.goVersion, + } + return f +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index c98a7e2cb2..52f1041901 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -22,6 +22,7 @@ func init() { "Calc": reflect.ValueOf(tensor.Calc), "Call": reflect.ValueOf(tensor.Call), "CallOut": reflect.ValueOf(tensor.CallOut), + "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), "CallString": reflect.ValueOf(tensor.CallString), "ColMajorStrides": reflect.ValueOf(tensor.ColMajorStrides), "Comma": reflect.ValueOf(tensor.Comma), diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 0c5d689d31..1a78ad064f 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -1,4 +1,4 @@ -#!/usr/bin/env cosh +#!/usr/bin/env goal yaegi extract fmt strconv strings image image/color image/draw time log/slog reflect From 759e27917e57e8539c23f5ab3b673ac4fa82906e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 11:34:35 -0700 Subject: [PATCH 066/311] goal: switch over to $ = shell, # = math --- goal/README.md | 97 ++++++++++++++++++++++++------------------ goal/execwords.go | 15 +++++++ goal/math.go | 8 ++-- goal/token.go | 20 +++++++++ goal/transpile.go | 95 ++++++++++++++++++++++++----------------- goal/transpile_test.go | 16 +++---- 6 files changed, 159 insertions(+), 92 deletions(-) diff --git a/goal/README.md b/goal/README.md index 5da7c5433f..57a4585c5f 100644 --- a/goal/README.md +++ b/goal/README.md @@ -1,38 +1,55 @@ # Goal: Go augmented language -_Goal_ is an augmented version of the _Go_ language, which combines the best parts of _Go_, `bash`, and Python, to provide and integrated shell and numerical expression processing experience. +_Goal_ is an augmented version of the _Go_ language, which combines the best parts of _Go_, `bash`, and Python, to provide and integrated shell and numerical expression processing experience, which can be combined with the [yaegi](https://github.com/traefik/yaegi) interpreter to provide an interactive "REPL" (read, evaluate, print loop). _Goal_ transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: * Shell scripting, where you want to be able to directly call other executable programs with arguments, without having to navigate all the complexity of the standard [os.exec](https://pkg.go.dev/os/exec) package. -* Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and iterators etc. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. +* Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and need extended indexing and slicing expressions. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. The main goal of _Goal_ is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where the Go language has been a barrier. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, _Goal_ retains in its _Go_ foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. For the shell scripting aspect of _Goal_, the simple idea is that each line of code is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below). If a line starts off with something like `ls -la...` then it is clear that it is not valid Go code, and it is therefore processed as a shell command. -You can intermix Go within a shell line by wrapping an expression with `{ }` braces, and a Go expression can contain shell code by using backticks (\`) (raw quoted strings use double-backticks \`\`). Here's an example: +You can intermix Go within a shell line by wrapping an expression with `{ }` braces, and a Go expression can contain shell code by using `$`. Here's an example: ```go -for i, f := range goal.SplitLines(`ls -la`) { // `ls` executes, returns string - echo {i} {strings.ToLower(f)} // {} surrounds Go within shell +for i, f := range goalib.SplitLines($ls -la$) { // ls executes, returns string + echo {i} {strings.ToLower(f)} // {} surrounds Go within shell } ``` -where `goal.SplitLines` is a function that runs `strings.Split(arg, "\n")`, defined in the `goal` standard library of such frequently-used helper functions. +where `goalib.SplitLines` is a function that runs `strings.Split(arg, "\n")`, defined in the `goalib` standard library of such frequently-used helper functions. + +For cases where most of the code is standard Go with relatively infrequent use of shell expressions, or in the rare cases where the default interpretation doesn't work, you can explicitly tag a line as shell code using `$`: -For the mathematical expressions, we use `$` symbols (as in markdown) to demarcate such expressions: ```go -for _, x := range $[1,2,3]$ { - fmt.Println($x^2$) +$ chmod +x *.goal +``` + +For mathematical expressions, we use `#` symbols (`#` = number) to demarcate such expressions. Often you will write entire lines of such expressions: +```go +# x := 1. / (1. + exp(-wts[:, :, :n] * acts[:])) +``` +You can also intermix within Go code: +```go +for _, x := range #[1,2,3]# { + fmt.Println(#x^2#) } ``` -In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. +In general, the math mode syntax in _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. + +The rationale and mnemonics for using `$` and `#` are as follows: + +* These are two of the three symbols that are not part of standard Go syntax (`@` being the other). + +* `$` can be thought of as "S" in _S_hell, and is often used for a `bash` prompt, and many bash examples use it as a prefix. Furthermore, in bash, `$( )` is used to wrap shell expressions. -We henceforth refer to shell code as `exec` code (in reference to the Go & Cogent `exec` package that we use to execute programs), given the potential ambituity of the entire `goal` language being the shell. There are different syntactic formatting rules for these two domains of Go and Exec, within goal: +* `#` is commonly used to refer to numbers. It is also often used as a comment syntax, but on balance the number semantics and uniqueness relative to Go syntax outweigh that issue. -* Go code is processed and formatted as usual (e.g., white space is irrelevant, etc). -* Exec code is space separated, like normal command-line invocations. +# Examples + +Here are a few useful examples of _Goal_ code: You can easily perform handy duration and data size formatting: @@ -41,16 +58,11 @@ You can easily perform handy duration and data size formatting: datasize.Size(44610930) // 42.5 MB ``` -# Special syntax - -## Multiple statements per line - -* Multiple statements can be combined on one line, separated by `;` as in regular Go and shell languages. Critically, the language determination for the first statement determines the language for the remaining statements; you cannot intermix the two on one line, when using `;` -# Exec mode +# Shell mode ## Environment variables -* `set ` (space delimited as in all exec mode, no equals) +* `set ` (space delimited as in all shell mode, no equals) ## Output redirction @@ -68,9 +80,9 @@ cd some; [mkdir sub]; cd sub * `jobs`, `fg`, `bg`, and `kill` builtin commands function as in usual bash. -## Exec functions (aliases) +## Shell functions (aliases) -Use the `command` keyword to define new functions for Exec mode execution, which can then be used like any other command, for example: +Use the `command` keyword to define new functions for Shell mode execution, which can then be used like any other command, for example: ```sh command list { @@ -87,7 +99,7 @@ The `command` is transpiled into a Go function that takes `args ...string`. In The command function name is registered so that the standard shell execution code can run the function, passing the args. You can also call it directly from Go code using the standard parentheses expression. -# Script Files and Makefile-like functionality +## Script Files and Makefile-like functionality As with most scripting languages, a file of goal code can be made directly executable by appending a "shebang" expression at the start of the file: @@ -115,11 +127,11 @@ See [make](cmd/goal/testdata/make) for an example, in `cmd/goal/testdata/make`, Note that there is nothing special about the name `make` here, so this can be done with any file. -The `make` package defines a number of useful utility functions that accomplish the standard dependency and file timestamp checking functionality from the standard `make` command, as in the [magefile](https://magefile.org/dependencies/) system. Note that the goal direct exec command syntax makes the resulting make files much closer to a standard bash-like Makefile, while still having all the benefits of Go control and expressions, compared to magefile. +The `make` package defines a number of useful utility functions that accomplish the standard dependency and file timestamp checking functionality from the standard `make` command, as in the [magefile](https://magefile.org/dependencies/) system. Note that the goal direct shell command syntax makes the resulting make files much closer to a standard bash-like Makefile, while still having all the benefits of Go control and expressions, compared to magefile. TODO: implement and document above. -# SSH connections to remote hosts +## SSH connections to remote hosts Any number of active SSH connections can be maintained and used dynamically within a script, including simple ways of copying data among the different hosts (including the local host). The Go mode execution is always on the local host in one running process, and only the shell commands are executed remotely, enabling a unique ability to easily coordinate and distribute processing and data across various hosts. @@ -167,7 +179,7 @@ The builtin `scp` function allows easy copying of files across hosts, using the scp @name:hostfile.tsv localfile.tsv ``` -TODO: Importantly, file wildcard globbing works as expected: +Importantly, file wildcard globbing works as expected: ```sh scp @name:*.tsv @0:data/ ``` @@ -182,29 +194,32 @@ cossh close Will close all active connections and return the default host to @0. All active connections are also automatically closed when the shell terminates. -# Other Utilties +## Other Utilties -** need a replacement for findnm -- very powerful but garbage.. +** TODO: need a replacement for findnm -- very powerful but garbage.. -# Rules for Go vs. Shell determination +## Rules for Go vs. Shell determination -The critical extension from standard Go syntax is for lines that are processed by the `Exec` functions, used for running arbitrary programs on the user's executable path. Here are the rules (word = IDENT token): +These are the rules used to determine whether a line is Go vs. Shell (word = IDENT token): -* Backticks "``" anywhere: Exec. Returns a `string`. -* Within Exec, `{}`: Go -* Line starts with `Go` Keyword: Go -* Line is one word: Exec -* Line starts with `path`: Exec -* Line starts with `"string"`: Exec -* Line starts with `word word`: Exec -* Line starts with `word {`: Exec +* `$` at the start: Shell. +* Within Shell, `{}`: Go +* Within Go, `$ $`: Shell +* Line starts with `go` keyword: if no `( )` then Shell, else Go +* Line is one word: Shell +* Line starts with `path` expression (e.g., `./myexec`) : Shell +* Line starts with `"string"`: Shell +* Line starts with `word word`: Shell +* Line starts with `word {`: Shell * Otherwise: Go -# TODO: +TODO: update aboven + +## Multiple statements per line -* likewise, need to run everything effectively as a bg job with our own explicit Wait, which we can then communicate with to move from fg to bg. +* Multiple statements can be combined on one line, separated by `;` as in regular Go and shell languages. Critically, the language determination for the first statement determines the language for the remaining statements; you cannot intermix the two on one line, when using `;` -# Math Expression Details +# Math mode In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. The `np.` prefix on numpy global functions is optional, and corresponding field-like properties of tensors turn into the appropriate methods during the transpiling process. diff --git a/goal/execwords.go b/goal/execwords.go index 1acc8b6fc2..54e04830e6 100644 --- a/goal/execwords.go +++ b/goal/execwords.go @@ -17,6 +17,21 @@ func ExecWords(ln string) ([]string, error) { return nil, nil } + if ln[0] == '$' { + ln = strings.TrimSpace(ln[1:]) + n = len(ln) + if n == 0 { + return nil, nil + } + if ln[n-1] == '$' { + ln = strings.TrimSpace(ln[:n-1]) + n = len(ln) + if n == 0 { + return nil, nil + } + } + } + word := "" esc := false dQuote := false diff --git a/goal/math.go b/goal/math.go index 589a946e5c..e0ab44b9b4 100644 --- a/goal/math.go +++ b/goal/math.go @@ -30,7 +30,7 @@ func (mp *mathParse) addCur() { fmt.Println("out of toks:", mp.curToks) } -func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { +func (gl *Goal) TranspileMath(toks Tokens, code string) Tokens { nt := len(toks) // fmt.Println(nt, toks) @@ -46,9 +46,9 @@ func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { } if assignIdx >= 0 { mp.lhsToks = toks[0:assignIdx] - mp.lhs = ln[toks[0].Pos-1 : toks[assignIdx].Pos-1] + mp.lhs = code[toks[0].Pos-1 : toks[assignIdx].Pos-1] mp.rhsToks = toks[assignIdx+1 : nt] - mp.rhs = ln[toks[assignIdx+1].Pos-1 : toks[nt-1].Pos] + mp.rhs = code[toks[assignIdx+1].Pos-1 : toks[nt-1].Pos] lex, err := parser.ParseExpr(mp.lhs) if err != nil { fmt.Println("lhs:", mp.lhs) @@ -64,7 +64,7 @@ func (gl *Goal) TranspileMath(toks Tokens, ln string) Tokens { } else { mp.rhsToks = toks[0:nt] mp.curToks = mp.rhsToks - mp.rhs = ln[toks[0].Pos-1 : toks[nt-1].Pos] + mp.rhs = code[toks[0].Pos-1 : toks[nt-1].Pos] ex, err := parser.ParseExpr(mp.rhs) if err != nil { fmt.Println("expr:", mp.rhs) diff --git a/goal/token.go b/goal/token.go index bf82fba465..0a2f1bf67d 100644 --- a/goal/token.go +++ b/goal/token.go @@ -308,6 +308,26 @@ func (tk Tokens) BracketDepths() (paren, brace, brack int) { return } +// ModeEnd returns the position (or -1 if not found) for the +// next ILLEGAL mode token ($ or #) given the starting one that +// is at the 0 position of the current set of tokens. +func (tk Tokens) ModeEnd() int { + n := len(tk) + if n == 0 { + return -1 + } + st := tk[0].Str + for i := 1; i < n; i++ { + if tk[i].Tok != token.ILLEGAL { + continue + } + if tk[i].Str == st { + return i + } + } + return -1 +} + // Tokens converts the string into tokens func (gl *Goal) Tokens(ln string) Tokens { fset := token.NewFileSet() diff --git a/goal/transpile.go b/goal/transpile.go index 80d9305c6c..3c94affa9c 100644 --- a/goal/transpile.go +++ b/goal/transpile.go @@ -15,14 +15,14 @@ import ( // TranspileLine is the main function for parsing a single line of goal input, // returning a new transpiled line of code that converts Exec code into corresponding // Go function calls. -func (gl *Goal) TranspileLine(ln string) string { - if len(ln) == 0 { - return ln +func (gl *Goal) TranspileLine(code string) string { + if len(code) == 0 { + return code } - if strings.HasPrefix(ln, "#!") { + if strings.HasPrefix(code, "#!") { return "" } - toks := gl.TranspileLineTokens(ln) + toks := gl.TranspileLineTokens(code) paren, brace, brack := toks.BracketDepths() gl.ParenDepth += paren gl.BraceDepth += brace @@ -38,21 +38,21 @@ func (gl *Goal) TranspileLine(ln string) string { } // TranspileLineTokens returns the tokens for the full line -func (gl *Goal) TranspileLineTokens(ln string) Tokens { - if ln == "" { +func (gl *Goal) TranspileLineTokens(code string) Tokens { + if code == "" { return nil } - toks := gl.Tokens(ln) + toks := gl.Tokens(code) n := len(toks) if n == 0 { return toks } - ewords, err := ExecWords(ln) + ewords, err := ExecWords(code) if err != nil { gl.AddError(err) return nil } - logx.PrintlnDebug("\n########## line:\n", ln, "\nTokens:\n", toks.String(), "\nWords:\n", ewords) + logx.PrintlnDebug("\n########## line:\n", code, "\nTokens:\n", toks.String(), "\nWords:\n", ewords) if toks[0].Tok == token.TYPE { gl.TypeDepth++ @@ -63,7 +63,7 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { if gl.TypeDepth > 0 || gl.DeclDepth > 0 { logx.PrintlnDebug("go: type / decl defn") - return gl.TranspileGo(toks) + return gl.TranspileGo(toks, code) } t0 := toks[0] @@ -74,25 +74,17 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { switch { case t0.Tok == token.ILLEGAL: - if t0.Str != "" && t0.Str[:1] == "$" { - return gl.TranspileMath(toks[1:], ln) + if t0.Str == "#" { + logx.PrintlnDebug("math #") + return gl.TranspileMath(toks[1:], code) } + return gl.TranspileExec(ewords, false) case t0.Tok == token.LBRACE: logx.PrintlnDebug("go: { } line") - return gl.TranspileGo(toks[1 : n-1]) + return gl.TranspileGo(toks[1:n-1], code[toks[1].Pos-1:toks[n-1].Pos-1]) case t0.Tok == token.LBRACK: logx.PrintlnDebug("exec: [ ] line") return gl.TranspileExec(ewords, false) // it processes the [ ] - case t0.Tok == token.ILLEGAL: - logx.PrintlnDebug("exec: illegal") - return gl.TranspileExec(ewords, false) - case t0.IsBacktickString(): - logx.PrintlnDebug("exec: backquoted string") - exe := gl.TranspileExecString(t0.Str, false) - if n > 1 { // todo: is this an error? - exe.AddTokens(gl.TranspileGo(toks[1:])...) - } - return exe case t0.Tok == token.IDENT && t0.Str == "command": gl.lastCommand = toks[1].Str // 1 is the name -- triggers AddCommand toks = toks[2:] // get rid of first @@ -106,7 +98,7 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { toks.Insert(7, token.ELLIPSIS) toks.Insert(8, token.IDENT, "string") toks.Insert(9, token.RPAREN) - toks.AddTokens(gl.TranspileGo(toks[11:])...) + toks.AddTokens(gl.TranspileGo(toks[11:], code)...) case t0.IsGo(): if t0.Tok == token.GO { if !toks.Contains(token.LPAREN) { @@ -115,9 +107,9 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { } } logx.PrintlnDebug("go keyword") - return gl.TranspileGo(toks) + return gl.TranspileGo(toks, code) case toks[n-1].Tok == token.INC: - return gl.TranspileGo(toks) + return gl.TranspileGo(toks, code) case t0pn > 0: // path expr logx.PrintlnDebug("exec: path...") return gl.TranspileExec(ewords, false) @@ -129,23 +121,23 @@ func (gl *Goal) TranspileLineTokens(ln string) Tokens { return gl.TranspileExec(ewords, false) case !f0exec: // exec must be IDENT logx.PrintlnDebug("go: not ident") - return gl.TranspileGo(toks) + return gl.TranspileGo(toks, code) case f0exec && en > 1 && (ewords[1][0] == '=' || ewords[1][0] == ':' || ewords[1][0] == '+' || toks[1].Tok == token.COMMA): logx.PrintlnDebug("go: assignment or defn") - return gl.TranspileGo(toks) + return gl.TranspileGo(toks, code) case f0exec: // now any ident logx.PrintlnDebug("exec: ident..") return gl.TranspileExec(ewords, false) default: logx.PrintlnDebug("go: default") - return gl.TranspileGo(toks) + return gl.TranspileGo(toks, code) } return toks } // TranspileGo returns transpiled tokens assuming Go code. -// Unpacks any backtick encapsulated shell commands. -func (gl *Goal) TranspileGo(toks Tokens) Tokens { +// Unpacks any encapsulated shell or math expressions. +func (gl *Goal) TranspileGo(toks Tokens, code string) Tokens { n := len(toks) if n == 0 { return toks @@ -155,12 +147,25 @@ func (gl *Goal) TranspileGo(toks Tokens) Tokens { toks[0] = toks[1] toks.Insert(1, token.DEFINE) toks[2] = &Token{Tok: token.FUNC} + n = len(toks) } } gtoks := make(Tokens, 0, len(toks)) // return tokens - for _, tok := range toks { - if gl.TypeDepth == 0 && tok.IsBacktickString() { - gtoks = append(gtoks, gl.TranspileExecString(tok.Str, true)...) + for i := 0; i < n; i++ { + tok := toks[i] + if tok.Tok == token.ILLEGAL { + et := toks[i:].ModeEnd() + if et > 0 { + if tok.Str == "#" { + gtoks.AddTokens(gl.TranspileMath(toks[i+1:i+et], code)...) + } else { + gtoks.AddTokens(gl.TranspileExecTokens(toks[i+1:i+et+1], code, true)...) + } + i += et + continue + } else { + gtoks = append(gtoks, tok) + } } else { gtoks = append(gtoks, tok) } @@ -169,19 +174,30 @@ func (gl *Goal) TranspileGo(toks Tokens) Tokens { } // TranspileExecString returns transpiled tokens assuming Exec code, -// from a backtick-encoded string, with the given bool indicating -// whether [Output] is needed. +// from a string, with the given bool indicating whether [Output] is needed. func (gl *Goal) TranspileExecString(str string, output bool) Tokens { if len(str) <= 1 { return nil } - ewords, err := ExecWords(str[1 : len(str)-1]) // enclosed string + ewords, err := ExecWords(str) if err != nil { gl.AddError(err) } return gl.TranspileExec(ewords, output) } +// TranspileExecTokens returns transpiled tokens assuming Exec code, +// from given tokens, with the given bool indicating +// whether [Output] is needed. +func (gl *Goal) TranspileExecTokens(toks Tokens, code string, output bool) Tokens { + nt := len(toks) + if nt == 0 { + return nil + } + str := code[toks[0].Pos-1 : toks[nt-1].Pos-1] + return gl.TranspileExecString(str, output) +} + // TranspileExec returns transpiled tokens assuming Exec code, // with the given bools indicating the type of run to execute. func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { @@ -227,12 +243,13 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { for i := 0; i < n; i++ { f := ewords[i] switch { + // case f == "#": // embedded math TODO case f == "{": // embedded go if n < i+3 { gl.AddError(fmt.Errorf("goal: no matching right brace } found in exec command line")) } else { gstr := ewords[i+1] - etoks.AddTokens(gl.TranspileGo(gl.Tokens(gstr))...) + etoks.AddTokens(gl.TranspileGo(gl.Tokens(gstr), gstr)...) etoks.Add(token.COMMA) i += 2 } diff --git a/goal/transpile_test.go b/goal/transpile_test.go index c6c37e78db..cd54f3038d 100644 --- a/goal/transpile_test.go +++ b/goal/transpile_test.go @@ -87,7 +87,7 @@ func TestTranspile(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ {"ls", `goal.Run("ls")`}, - {"`ls -la`", `goal.Run("ls", "-la")`}, + {"$ls -la$", `goal.Run("ls", "-la")`}, {"ls -la", `goal.Run("ls", "-la")`}, {"ls --help", `goal.Run("ls", "--help")`}, {"ls go", `goal.Run("ls", "go")`}, @@ -102,9 +102,9 @@ func TestTranspile(t *testing.T) { {`println("hi")`, `println("hi")`}, {`fmt.Println("hi")`, `fmt.Println("hi")`}, {`for i := 0; i < 3; i++ { fmt.Println(i, "\n")`, `for i := 0; i < 3; i++ { fmt.Println(i, "\n")`}, - {"for i, v := range `ls -la` {", `for i, v := range goal.Output("ls", "-la") {`}, + {"for i, v := range $ls -la$ {", `for i, v := range goal.Output("ls", "-la") {`}, {`// todo: fixit`, `// todo: fixit`}, - {"`go build`", `goal.Run("go", "build")`}, + {"$go build$", `goal.Run("go", "build")`}, {"{go build()}", `go build()`}, {"go build", `goal.Run("go", "build")`}, {"go build()", `go build()`}, @@ -201,11 +201,11 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"$ x := 1", `x := tensor.NewIntScalar(1)`}, - {"$ x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - {"$ x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - {"$ a = x + y", `a = tensor.CallOut("Add", x, y)`}, - // {"$ a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, + {"# x := 1", `x := tensor.NewIntScalar(1)`}, + {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + // {"# a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, } gl := NewGoal() From e715ba2961983751113ba2464fed38f177a55a11 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 13:03:47 -0700 Subject: [PATCH 067/311] using our own parser now, deals with statements and exprs; now just need to extend parsing to array syntax. --- goal/math.go | 220 +++++++++++++++++++++++++++++------------- goal/mparse/mparse.go | 142 +++++++++++++-------------- goal/transpile.go | 4 +- 3 files changed, 222 insertions(+), 144 deletions(-) diff --git a/goal/math.go b/goal/math.go index e0ab44b9b4..f24f2ca6f9 100644 --- a/goal/math.go +++ b/goal/math.go @@ -7,92 +7,150 @@ package goal import ( "fmt" "go/ast" - "go/parser" "go/token" + + "cogentcore.org/core/goal/mparse" ) type mathParse struct { - toks Tokens // output tokens - lhs string - rhs string // or the only one - curToks Tokens // current source tokens we are parsing - curIdx int // current index in source tokens - lhsToks Tokens - rhsToks Tokens + code string // code string + toks Tokens // source tokens we are parsing + idx int // current index in source tokens + out Tokens // output tokens we generate +} + +// addToken adds output token and increments idx +func (mp *mathParse) addToken(tok token.Token) { + mp.out.Add(tok) + mp.idx++ } func (mp *mathParse) addCur() { - if len(mp.curToks) > mp.curIdx { - mp.toks.AddTokens(mp.curToks[mp.curIdx]) - mp.curIdx++ + if len(mp.toks) > mp.idx { + mp.out.AddTokens(mp.toks[mp.idx]) + mp.idx++ return } - fmt.Println("out of toks:", mp.curToks) + fmt.Println("out of tokens!", mp.idx, mp.toks) } -func (gl *Goal) TranspileMath(toks Tokens, code string) Tokens { +// TranspileMath does math mode transpiling. fullLine indicates code should be +// full statement(s). +func (gl *Goal) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { nt := len(toks) + if nt == 0 { + return nil + } // fmt.Println(nt, toks) - mp := mathParse{} + str := code[toks[0].Pos-1 : toks[nt-1].Pos] + mp := mathParse{toks: toks, code: code} - // expr can't do statements, so we need to find those ourselves - assignIdx := -1 - for i, tk := range toks { - if tk.Tok == token.ASSIGN || tk.Tok == token.DEFINE { - assignIdx = i - break - } - } - if assignIdx >= 0 { - mp.lhsToks = toks[0:assignIdx] - mp.lhs = code[toks[0].Pos-1 : toks[assignIdx].Pos-1] - mp.rhsToks = toks[assignIdx+1 : nt] - mp.rhs = code[toks[assignIdx+1].Pos-1 : toks[nt-1].Pos] - lex, err := parser.ParseExpr(mp.lhs) + if fullLine { + stmts, err := mparse.ParseLine(str, mparse.AllErrors) if err != nil { - fmt.Println("lhs:", mp.lhs) - fmt.Println("lhs parse err:", err) + fmt.Println("line code:", str) + fmt.Println("parse err:", err) } - rex, err := parser.ParseExpr(mp.rhs) - if err != nil { - fmt.Println("rhs:", mp.rhs) - fmt.Println("rhs parse err:", err) - fmt.Printf("%#v\n", rex) - } - mp.assignStmt(toks[assignIdx], lex, rex) + mp.stmtList(stmts) } else { - mp.rhsToks = toks[0:nt] - mp.curToks = mp.rhsToks - mp.rhs = code[toks[0].Pos-1 : toks[nt-1].Pos] - ex, err := parser.ParseExpr(mp.rhs) + ex, err := mparse.ParseExpr(str, mparse.AllErrors) if err != nil { - fmt.Println("expr:", mp.rhs) - fmt.Println("expr parse err:", err) + fmt.Println("expr:", str) + fmt.Println("parse err:", err) } mp.expr(ex) } - return mp.toks + return mp.out } -func (mp *mathParse) assignStmt(tok *Token, lex, rex ast.Expr) { - mp.curToks = mp.lhsToks - mp.expr(lex) - mp.toks.AddTokens(tok) - mp.curToks = mp.rhsToks - mp.curIdx = 0 - mp.expr(rex) +func (mp *mathParse) stmtList(sts []ast.Stmt) { + for _, st := range sts { + mp.stmt(st) + } +} + +func (mp *mathParse) stmt(st ast.Stmt) { + if st == nil { + return + } + switch x := st.(type) { + case *ast.BadStmt: + fmt.Println("bad stmt!") + + case *ast.DeclStmt: + + case *ast.ExprStmt: + mp.expr(x.X) + + case *ast.SendStmt: + mp.expr(x.Chan) + mp.addToken(token.ARROW) + mp.expr(x.Value) + + case *ast.IncDecStmt: + mp.expr(x.X) + mp.addToken(x.Tok) + + case *ast.AssignStmt: + mp.exprList(x.Lhs) + mp.addToken(x.Tok) + mp.exprList(x.Rhs) + + case *ast.GoStmt: + mp.addToken(token.GO) + mp.callExpr(x.Call) + + case *ast.DeferStmt: + mp.addToken(token.DEFER) + mp.callExpr(x.Call) + + case *ast.ReturnStmt: + mp.addToken(token.RETURN) + mp.exprList(x.Results) + + case *ast.BranchStmt: + mp.addToken(x.Tok) + mp.ident(x.Label) + + case *ast.BlockStmt: + mp.addToken(token.LBRACE) + mp.stmtList(x.List) + mp.addToken(token.RBRACE) + + case *ast.IfStmt: + mp.addToken(token.IF) + mp.stmt(x.Init) + if x.Init != nil { + mp.addToken(token.SEMICOLON) + } + mp.expr(x.Cond) + if x.Body != nil { + mp.addToken(token.LBRACE) + mp.stmtList(x.Body.List) + mp.addToken(token.RBRACE) + } + if x.Else != nil { + mp.addToken(token.ELSE) + mp.stmt(x.Else) + } + + // TODO + // CaseClause: SwitchStmt:, TypeSwitchStmt:, CommClause:, SelectStmt:, ForStmt:, RangeStmt: + } } func (mp *mathParse) expr(ex ast.Expr) { + if ex == nil { + return + } switch x := ex.(type) { case *ast.BadExpr: - fmt.Println("bad!") + fmt.Println("bad expr!") case *ast.Ident: - // fmt.Println("ident:", x.Name) - mp.addCur() + mp.ident(x) case *ast.BinaryExpr: mp.binaryExpr(x) @@ -127,6 +185,23 @@ func (mp *mathParse) expr(ex ast.Expr) { } } +func (mp *mathParse) exprList(ex []ast.Expr) { + n := len(ex) + if n == 0 { + return + } + if n == 1 { + mp.expr(ex[0]) + return + } + for i := range n { + mp.expr(ex[i]) + if i < n-1 { + mp.addToken(token.COMMA) + } + } +} + func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn := "" switch ex.Op { @@ -139,28 +214,28 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { case token.QUO: fn = "Div" } - mp.toks.Add(token.IDENT, "tensor.CallOut") - mp.toks.Add(token.LPAREN) - mp.toks.Add(token.STRING, `"`+fn+`"`) - mp.toks.Add(token.COMMA) + mp.out.Add(token.IDENT, "tensor.CallOut") + mp.out.Add(token.LPAREN) + mp.out.Add(token.STRING, `"`+fn+`"`) + mp.out.Add(token.COMMA) mp.expr(ex.X) - mp.toks.Add(token.COMMA) - mp.curIdx++ + mp.out.Add(token.COMMA) + mp.idx++ mp.expr(ex.Y) - mp.toks.Add(token.RPAREN) + mp.out.Add(token.RPAREN) } func (mp *mathParse) basicLit(lit *ast.BasicLit) { switch lit.Kind { case token.INT: - mp.toks.Add(token.IDENT, "tensor.NewIntScalar("+lit.Value+")") - mp.curIdx++ + mp.out.Add(token.IDENT, "tensor.NewIntScalar("+lit.Value+")") + mp.idx++ case token.FLOAT: - mp.toks.Add(token.IDENT, "tensor.NewFloatScalar("+lit.Value+")") - mp.curIdx++ + mp.out.Add(token.IDENT, "tensor.NewFloatScalar("+lit.Value+")") + mp.idx++ case token.STRING: - mp.toks.Add(token.IDENT, "tensor.NewStringScalar("+lit.Value+")") - mp.curIdx++ + mp.out.Add(token.IDENT, "tensor.NewStringScalar("+lit.Value+")") + mp.idx++ } } @@ -170,3 +245,14 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { fmt.Println("slice expr", se) } + +func (mp *mathParse) callExpr(ex *ast.CallExpr) { +} + +func (mp *mathParse) ident(id *ast.Ident) { + if id == nil { + return + } + // fmt.Println("ident:", x.Name) + mp.addCur() +} diff --git a/goal/mparse/mparse.go b/goal/mparse/mparse.go index 5dfbeb807a..91432099e1 100644 --- a/goal/mparse/mparse.go +++ b/goal/mparse/mparse.go @@ -20,20 +20,79 @@ import ( "strings" ) +// ParseLine parses a line of code that could contain one or more statements +func ParseLine(code string, mode Mode) (stmts []ast.Stmt, err error) { + fset := token.NewFileSet() + var p parser + defer func() { + if e := recover(); e != nil { + // resume same panic if it's not a bailout + bail, ok := e.(bailout) + if !ok { + panic(e) + } else if bail.msg != "" { + p.errors.Add(p.file.Position(bail.pos), bail.msg) + } + } + p.errors.Sort() + err = p.errors.Err() + }() + p.init(fset, "", []byte(code), mode) + + stmts = p.parseStmtList() + + // If a semicolon was inserted, consume it; + // report an error if there's more tokens. + if p.tok == token.SEMICOLON && p.lit == "\n" { + p.next() + } + p.expect(token.EOF) + + return +} + +// ParseExpr parses an expression +func ParseExpr(code string, mode Mode) (expr ast.Expr, err error) { + fset := token.NewFileSet() + var p parser + defer func() { + if e := recover(); e != nil { + // resume same panic if it's not a bailout + bail, ok := e.(bailout) + if !ok { + panic(e) + } else if bail.msg != "" { + p.errors.Add(p.file.Position(bail.pos), bail.msg) + } + } + p.errors.Sort() + err = p.errors.Err() + }() + p.init(fset, "", []byte(code), mode) + + expr = p.parseRhs() + + // If a semicolon was inserted, consume it; + // report an error if there's more tokens. + if p.tok == token.SEMICOLON && p.lit == "\n" { + p.next() + } + p.expect(token.EOF) + + return +} + // A Mode value is a set of flags (or 0). // They control the amount of source code parsed and other optional // parser functionality. type Mode uint const ( - PackageClauseOnly Mode = 1 << iota // stop parsing after package clause - ImportsOnly // stop parsing after import declarations - ParseComments // parse comments and add them to AST - Trace // print a trace of parsed productions - DeclarationErrors // report declaration errors - SpuriousErrors // same as AllErrors, for backward-compatibility - SkipObjectResolution // skip deprecated identifier resolution; see ParseFile - AllErrors = SpuriousErrors // report all errors (not just the first 10 on different lines) + ParseComments Mode = 1 << iota // parse comments and add them to AST + Trace // print a trace of parsed productions + DeclarationErrors // report declaration errors + SpuriousErrors // same as AllErrors, for backward-compatibility + AllErrors = SpuriousErrors // report all errors (not just the first 10 on different lines) ) // The parser structure holds the parser's internal state. @@ -2848,70 +2907,3 @@ func (p *parser) parseDecl(sync map[token.Token]bool) ast.Decl { return p.parseGenDecl(p.tok, f) } - -// ---------------------------------------------------------------------------- -// Source files - -func (p *parser) parseFile() *ast.File { - if p.trace { - defer un(trace(p, "File")) - } - - // Don't bother parsing the rest if we had errors scanning the first token. - // Likely not a Go source file at all. - if p.errors.Len() != 0 { - return nil - } - - // package clause - doc := p.leadComment - pos := p.expect(token.PACKAGE) - // Go spec: The package clause is not a declaration; - // the package name does not appear in any scope. - ident := p.parseIdent() - if ident.Name == "_" && p.mode&DeclarationErrors != 0 { - p.error(p.pos, "invalid package name _") - } - p.expectSemi() - - // Don't bother parsing the rest if we had errors parsing the package clause. - // Likely not a Go source file at all. - if p.errors.Len() != 0 { - return nil - } - - var decls []ast.Decl - if p.mode&PackageClauseOnly == 0 { - // import decls - for p.tok == token.IMPORT { - decls = append(decls, p.parseGenDecl(token.IMPORT, p.parseImportSpec)) - } - - if p.mode&ImportsOnly == 0 { - // rest of package body - prev := token.IMPORT - for p.tok != token.EOF { - // Continue to accept import declarations for error tolerance, but complain. - if p.tok == token.IMPORT && prev != token.IMPORT { - p.error(p.pos, "imports must appear before other declarations") - } - prev = p.tok - - decls = append(decls, p.parseDecl(declStart)) - } - } - } - - f := &ast.File{ - Doc: doc, - Package: pos, - Name: ident, - Decls: decls, - FileStart: token.Pos(p.file.Base()), - FileEnd: token.Pos(p.file.Base() + p.file.Size()), - Imports: p.imports, - Comments: p.comments, - GoVersion: p.goVersion, - } - return f -} diff --git a/goal/transpile.go b/goal/transpile.go index 3c94affa9c..ce4bb8a8c8 100644 --- a/goal/transpile.go +++ b/goal/transpile.go @@ -76,7 +76,7 @@ func (gl *Goal) TranspileLineTokens(code string) Tokens { case t0.Tok == token.ILLEGAL: if t0.Str == "#" { logx.PrintlnDebug("math #") - return gl.TranspileMath(toks[1:], code) + return gl.TranspileMath(toks[1:], code, true) } return gl.TranspileExec(ewords, false) case t0.Tok == token.LBRACE: @@ -157,7 +157,7 @@ func (gl *Goal) TranspileGo(toks Tokens, code string) Tokens { et := toks[i:].ModeEnd() if et > 0 { if tok.Str == "#" { - gtoks.AddTokens(gl.TranspileMath(toks[i+1:i+et], code)...) + gtoks.AddTokens(gl.TranspileMath(toks[i+1:i+et], code, false)...) } else { gtoks.AddTokens(gl.TranspileExecTokens(toks[i+1:i+et+1], code, true)...) } From 75ab7e1e1b69e74e645d1b0c003f3cab5501ab5c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 16:33:22 -0700 Subject: [PATCH 068/311] moved all transpiling into separate package -- getting kinda big and should be separated from higher-level goal api --- goal/cmd/goal/goal.go | 2 +- goal/cmd/goal/testdata/test.goal | 10 +- goal/goal.go | 167 +------------- goal/interpreter/interpreter.go | 16 +- goal/{ => transpile}/execwords.go | 2 +- goal/{ => transpile}/math.go | 76 ++++--- .../{mparse/mparse.go => transpile/parser.go} | 28 ++- goal/{ => transpile}/paths.go | 2 +- goal/transpile/state.go | 207 ++++++++++++++++++ goal/{ => transpile}/token.go | 50 ++--- goal/{ => transpile}/transpile.go | 102 +++++---- goal/{ => transpile}/transpile_test.go | 29 ++- 12 files changed, 386 insertions(+), 305 deletions(-) rename goal/{ => transpile}/execwords.go (99%) rename goal/{ => transpile}/math.go (89%) rename goal/{mparse/mparse.go => transpile/parser.go} (99%) rename goal/{ => transpile}/paths.go (99%) create mode 100644 goal/transpile/state.go rename goal/{ => transpile}/token.go (97%) rename goal/{ => transpile}/transpile.go (70%) rename goal/{ => transpile}/transpile_test.go (93%) diff --git a/goal/cmd/goal/goal.go b/goal/cmd/goal/goal.go index ebfa689e1c..a692a21990 100644 --- a/goal/cmd/goal/goal.go +++ b/goal/cmd/goal/goal.go @@ -82,7 +82,7 @@ func Run(c *Config) error { //cli:cmd -root _, _, err := in.Eval(code) if err == nil { - err = in.Goal.DepthError() + err = in.Goal.TrState.DepthError() } if c.Interactive { return Interactive(c, in) diff --git a/goal/cmd/goal/testdata/test.goal b/goal/cmd/goal/testdata/test.goal index ee83f73e38..79956ffe66 100644 --- a/goal/cmd/goal/testdata/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -7,11 +7,11 @@ for i, fn := range goalib.SplitLines(`/bin/ls -1`) { fmt.Println(i, fn) } -$ x := 1 -$ y := 4 -$ a := x * 2 -$ b := x + y -$ c := x * y + a * b +# x := 1 +# y := 4 +# a := x * 2 +# b := x + y +# c := x * y + a * b fmt.Println(c) diff --git a/goal/goal.go b/goal/goal.go index 5ba9a0afad..7e57e493ad 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -11,7 +11,6 @@ import ( "context" "fmt" "io/fs" - "log/slog" "os" "path/filepath" "slices" @@ -21,13 +20,11 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/base/exec" "cogentcore.org/core/base/logx" - "cogentcore.org/core/base/num" "cogentcore.org/core/base/reflectx" "cogentcore.org/core/base/sshclient" "cogentcore.org/core/base/stack" - "cogentcore.org/core/base/stringsx" + "cogentcore.org/core/goal/transpile" "github.com/mitchellh/go-homedir" - "golang.org/x/tools/imports" ) // Goal represents one running Goal language context. @@ -50,21 +47,6 @@ type Goal struct { // SSHActive is the name of the active SSH client SSHActive string - // depth of delim at the end of the current line. if 0, was complete. - ParenDepth, BraceDepth, BrackDepth, TypeDepth, DeclDepth int - - // Chunks of code lines that are accumulated during Transpile, - // each of which should be evaluated separately, to avoid - // issues with contextual effects from import, package etc. - Chunks []string - - // current stack of transpiled lines, that are accumulated into - // code Chunks - Lines []string - - // stack of runtime errors - Errors []error - // Builtins are all the builtin shell commands Builtins map[string]func(cmdIO *exec.CmdIO, args ...string) error @@ -81,6 +63,9 @@ type Goal struct { // Both can be nil. Cancel func() + // Errors is a stack of runtime errors + Errors []error + // Ctx is the context used for cancelling current shell running // a single chunk of code, typically from the interpreter. // We are not able to pass the context around so it is set here, @@ -95,12 +80,8 @@ type Goal struct { // and saved / restored from ~/.goalhist file Hist []string - // FuncToVar translates function definitions into variable definitions, - // which is the default for interactive use of random code fragments - // without the complete go formatting. - // For pure transpiling of a complete codebase with full proper Go formatting - // this should be turned off. - FuncToVar bool + // transpiling state + TrState transpile.State // commandArgs is a stack of args passed to a command, used for simplified // processing of args expressions. @@ -109,10 +90,6 @@ type Goal struct { // isCommand is a stack of bools indicating whether the _immediate_ run context // is a command, which affects the way that args are processed. isCommand stack.Stack[bool] - - // if this is non-empty, it is the name of the last command defined. - // triggers insertion of the AddCommand call to add to list of defined commands. - lastCommand string } // NewGoal returns a new [Goal] with default options. @@ -124,7 +101,7 @@ func NewGoal() *Goal { Buffer: false, }, } - gl.FuncToVar = true + gl.TrState.FuncToVar = true gl.Config.StdIO.SetFromOS() gl.SSH = sshclient.NewConfig(&gl.Config) gl.SSHClients = make(map[string]*sshclient.Client) @@ -226,92 +203,10 @@ func (gl *Goal) SSHByHost(host string) (*sshclient.Client, error) { return nil, fmt.Errorf("ssh connection named: %q not found", host) } -// TotalDepth returns the sum of any unresolved paren, brace, or bracket depths. -func (gl *Goal) TotalDepth() int { - return num.Abs(gl.ParenDepth) + num.Abs(gl.BraceDepth) + num.Abs(gl.BrackDepth) -} - -// ResetCode resets the stack of transpiled code -func (gl *Goal) ResetCode() { - gl.Chunks = nil - gl.Lines = nil -} - -// ResetDepth resets the current depths to 0 -func (gl *Goal) ResetDepth() { - gl.ParenDepth, gl.BraceDepth, gl.BrackDepth, gl.TypeDepth, gl.DeclDepth = 0, 0, 0, 0, 0 -} - -// DepthError reports an error if any of the parsing depths are not zero, -// to be called at the end of transpiling a complete block of code. -func (gl *Goal) DepthError() error { - if gl.TotalDepth() == 0 { - return nil - } - str := "" - if gl.ParenDepth != 0 { - str += fmt.Sprintf("Incomplete parentheses (), remaining depth: %d\n", gl.ParenDepth) - } - if gl.BraceDepth != 0 { - str += fmt.Sprintf("Incomplete braces [], remaining depth: %d\n", gl.BraceDepth) - } - if gl.BrackDepth != 0 { - str += fmt.Sprintf("Incomplete brackets {}, remaining depth: %d\n", gl.BrackDepth) - } - if str != "" { - slog.Error(str) - return errors.New(str) - } - return nil -} - -// AddLine adds line on the stack -func (gl *Goal) AddLine(ln string) { - gl.Lines = append(gl.Lines, ln) -} - -// Code returns the current transpiled lines, -// split into chunks that should be compiled separately. -func (gl *Goal) Code() string { - gl.AddChunk() - if len(gl.Chunks) == 0 { - return "" - } - return strings.Join(gl.Chunks, "\n") -} - -// AddChunk adds current lines into a chunk of code -// that should be compiled separately. -func (gl *Goal) AddChunk() { - if len(gl.Lines) == 0 { - return - } - gl.Chunks = append(gl.Chunks, strings.Join(gl.Lines, "\n")) - gl.Lines = nil -} - // TranspileCode processes each line of given code, // adding the results to the LineStack func (gl *Goal) TranspileCode(code string) { - lns := strings.Split(code, "\n") - n := len(lns) - if n == 0 { - return - } - for _, ln := range lns { - hasDecl := gl.DeclDepth > 0 - tl := gl.TranspileLine(ln) - gl.AddLine(tl) - if gl.BraceDepth == 0 && gl.BrackDepth == 0 && gl.ParenDepth == 1 && gl.lastCommand != "" { - gl.lastCommand = "" - nl := len(gl.Lines) - gl.Lines[nl-1] = gl.Lines[nl-1] + ")" - gl.ParenDepth-- - } - if hasDecl && gl.DeclDepth == 0 { // break at decl - gl.AddChunk() - } - } + gl.TrState.TranspileCode(code) } // TranspileCodeFromFile transpiles the code in given file @@ -329,51 +224,7 @@ func (gl *Goal) TranspileCodeFromFile(file string) error { // is found, then package main and func main declarations are // added. This also affects how functions are interpreted. func (gl *Goal) TranspileFile(in string, out string) error { - b, err := os.ReadFile(in) - if err != nil { - return err - } - code := string(b) - lns := stringsx.SplitLines(code) - hasPackage := false - for _, ln := range lns { - if strings.HasPrefix(ln, "package ") { - hasPackage = true - break - } - } - if hasPackage { - gl.FuncToVar = false // use raw functions - } - gl.TranspileCode(code) - gl.FuncToVar = true - if err != nil { - return err - } - gen := "// Code generated by \"goal build\"; DO NOT EDIT.\n\n" - if hasPackage { - gl.Lines = slices.Insert(gl.Lines, 0, gen) - } else { - gl.Lines = slices.Insert(gl.Lines, 0, gen, "package main", "import (", - ` "cogentcore.org/core/goal"`, - ` "cogentcore.org/core/goal/goalib"`, - ` "cogentcore.org/core/tensor"`, - ` _ "cogentcore.org/core/tensor/tmath"`, - ` _ "cogentcore.org/core/tensor/stats/stats"`, - ` _ "cogentcore.org/core/tensor/stats/metric"`, - ")", "func main() {", "goal := goal.NewGoal()") - gl.Lines = append(gl.Lines, "}") - } - src := []byte(gl.Code()) - res, err := imports.Process(out, src, nil) - if err != nil { - res = src - slog.Error(err.Error()) - } else { - err = gl.DepthError() - } - werr := os.WriteFile(out, res, 0666) - return errors.Join(err, werr) + return gl.TrState.TranspileFile(in, out) } // AddError adds the given error to the error stack if it is non-nil, diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index ceebd88b05..191087644e 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -75,7 +75,7 @@ func NewInterpreter(options interp.Options) *Interpreter { // Prompt returns the appropriate REPL prompt to show the user. func (in *Interpreter) Prompt() string { - dp := in.Goal.TotalDepth() + dp := in.Goal.TrState.TotalDepth() if dp == 0 { return in.Goal.HostAndDir() + " > " } @@ -98,10 +98,10 @@ func (in *Interpreter) Eval(code string) (v reflect.Value, hasPrint bool, err er if in.Goal.SSHActive == "" { source = strings.HasPrefix(code, "source") } - if in.Goal.TotalDepth() == 0 { - nl := len(in.Goal.Lines) + if in.Goal.TrState.TotalDepth() == 0 { + nl := len(in.Goal.TrState.Lines) if nl > 0 { - ln := in.Goal.Lines[nl-1] + ln := in.Goal.TrState.Lines[nl-1] if strings.Contains(strings.ToLower(ln), "print") { hasPrint = true } @@ -122,9 +122,9 @@ func (in *Interpreter) RunCode() (reflect.Value, error) { if len(in.Goal.Errors) > 0 { return reflect.Value{}, errors.Join(in.Goal.Errors...) } - in.Goal.AddChunk() - code := in.Goal.Chunks - in.Goal.ResetCode() + in.Goal.TrState.AddChunk() + code := in.Goal.TrState.Chunks + in.Goal.TrState.ResetCode() var v reflect.Value var err error for _, ch := range code { @@ -135,7 +135,7 @@ func (in *Interpreter) RunCode() (reflect.Value, error) { cancelled := errors.Is(err, context.Canceled) // fmt.Println("cancelled:", cancelled) in.Goal.RestoreOrigStdIO() - in.Goal.ResetDepth() + in.Goal.TrState.ResetDepth() if !cancelled { in.Goal.AddError(err) } else { diff --git a/goal/execwords.go b/goal/transpile/execwords.go similarity index 99% rename from goal/execwords.go rename to goal/transpile/execwords.go index 54e04830e6..74bbca6241 100644 --- a/goal/execwords.go +++ b/goal/transpile/execwords.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package goal +package transpile import ( "fmt" diff --git a/goal/math.go b/goal/transpile/math.go similarity index 89% rename from goal/math.go rename to goal/transpile/math.go index f24f2ca6f9..df30521243 100644 --- a/goal/math.go +++ b/goal/transpile/math.go @@ -2,41 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package goal +package transpile import ( "fmt" "go/ast" "go/token" - - "cogentcore.org/core/goal/mparse" ) -type mathParse struct { - code string // code string - toks Tokens // source tokens we are parsing - idx int // current index in source tokens - out Tokens // output tokens we generate -} - -// addToken adds output token and increments idx -func (mp *mathParse) addToken(tok token.Token) { - mp.out.Add(tok) - mp.idx++ -} - -func (mp *mathParse) addCur() { - if len(mp.toks) > mp.idx { - mp.out.AddTokens(mp.toks[mp.idx]) - mp.idx++ - return - } - fmt.Println("out of tokens!", mp.idx, mp.toks) -} - -// TranspileMath does math mode transpiling. fullLine indicates code should be -// full statement(s). -func (gl *Goal) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { +func MathParse(toks Tokens, code string, fullLine bool) Tokens { nt := len(toks) if nt == 0 { return nil @@ -46,15 +20,17 @@ func (gl *Goal) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { str := code[toks[0].Pos-1 : toks[nt-1].Pos] mp := mathParse{toks: toks, code: code} + mods := AllErrors | Trace + if fullLine { - stmts, err := mparse.ParseLine(str, mparse.AllErrors) + stmts, err := ParseLine(str, mods) if err != nil { fmt.Println("line code:", str) fmt.Println("parse err:", err) } mp.stmtList(stmts) } else { - ex, err := mparse.ParseExpr(str, mparse.AllErrors) + ex, err := ParseExpr(str, mods) if err != nil { fmt.Println("expr:", str) fmt.Println("parse err:", err) @@ -65,6 +41,28 @@ func (gl *Goal) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { return mp.out } +type mathParse struct { + code string // code string + toks Tokens // source tokens we are parsing + idx int // current index in source tokens + out Tokens // output tokens we generate +} + +// addToken adds output token and increments idx +func (mp *mathParse) addToken(tok token.Token) { + mp.out.Add(tok) + mp.idx++ +} + +func (mp *mathParse) addCur() { + if len(mp.toks) > mp.idx { + mp.out.AddTokens(mp.toks[mp.idx]) + mp.idx++ + return + } + fmt.Println("out of tokens!", mp.idx, mp.toks) +} + func (mp *mathParse) stmtList(sts []ast.Stmt) { for _, st := range sts { mp.stmt(st) @@ -168,11 +166,15 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.TypeAssertExpr: case *ast.IndexListExpr: - fmt.Println("index!") + if x.X == nil { // array literal + mp.arrayLiteral(x) + } else { + mp.indexListExpr(x) + } case *ast.SliceExpr: - mp.sliceExpr(x) - // todo: we'll need to work on this! + // mp.sliceExpr(x) + // todo: probably this is subsumed in indexListExpr case *ast.CallExpr: @@ -242,8 +244,12 @@ func (mp *mathParse) basicLit(lit *ast.BasicLit) { func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { } -func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { - fmt.Println("slice expr", se) +func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { + // fmt.Println("slice expr", se) +} + +func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { + } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/mparse/mparse.go b/goal/transpile/parser.go similarity index 99% rename from goal/mparse/mparse.go rename to goal/transpile/parser.go index 91432099e1..7320ffc128 100644 --- a/goal/mparse/mparse.go +++ b/goal/transpile/parser.go @@ -7,9 +7,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package mparse is a hacked version of go/parser to support -// the additional math mode syntax for goal. -package mparse +package transpile import ( "fmt" @@ -443,7 +441,7 @@ func (p *parser) atComma(context string, follow token.Token) bool { return false } -func assert(cond bool, msg string) { +func passert(cond bool, msg string) { if !cond { panic("go/parser internal error: " + msg) } @@ -586,6 +584,19 @@ func (p *parser) parseList(inRhs bool) []ast.Expr { return list } +// math: allow full array list expressions +func (p *parser) parseArrayList(lbrack token.Pos) *ast.IndexListExpr { + if p.trace { + defer un(trace(p, "ArrayList")) + } + p.exprLev++ + // x := p.parseRhs() + x := p.parseExprList() + p.exprLev-- + rbrack := p.expect(token.RBRACK) + return &ast.IndexListExpr{Lbrack: lbrack, Indices: x, Rbrack: rbrack} +} + // ---------------------------------------------------------------------------- // Types @@ -1079,7 +1090,7 @@ func (p *parser) parseParameterList(name0 *ast.Ident, typ0 ast.Expr, closing tok if named == 0 { // parameter list consists of types only for _, par := range list { - assert(par.typ != nil, "nil type in unnamed parameter list") + passert(par.typ != nil, "nil type in unnamed parameter list") params = append(params, &ast.Field{Type: par.typ}) } return @@ -1090,7 +1101,7 @@ func (p *parser) parseParameterList(name0 *ast.Ident, typ0 ast.Expr, closing tok var names []*ast.Ident var typ ast.Expr addParams := func() { - assert(typ != nil, "nil type in named parameter list") + passert(typ != nil, "nil type in named parameter list") field := &ast.Field{Names: names, Type: typ} params = append(params, field) names = nil @@ -1434,7 +1445,8 @@ func (p *parser) tryIdentOrType() ast.Expr { return typ case token.LBRACK: lbrack := p.expect(token.LBRACK) - return p.parseArrayType(lbrack, nil) + return p.parseArrayList(lbrack) // math: full array exprs + // return p.parseArrayType(lbrack, nil) case token.STRUCT: return p.parseStructType() case token.MUL: @@ -1552,7 +1564,7 @@ func (p *parser) parseOperand() ast.Expr { if typ := p.tryIdentOrType(); typ != nil { // do not consume trailing type parameters // could be type for composite literal or conversion _, isIdent := typ.(*ast.Ident) - assert(!isIdent, "type cannot be identifier") + passert(!isIdent, "type cannot be identifier") return typ } diff --git a/goal/paths.go b/goal/transpile/paths.go similarity index 99% rename from goal/paths.go rename to goal/transpile/paths.go index e9a6ac73ff..0803d5f227 100644 --- a/goal/paths.go +++ b/goal/transpile/paths.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package goal +package transpile import ( "go/token" diff --git a/goal/transpile/state.go b/goal/transpile/state.go new file mode 100644 index 0000000000..a1e8c24afe --- /dev/null +++ b/goal/transpile/state.go @@ -0,0 +1,207 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package transpile + +import ( + "errors" + "fmt" + "log/slog" + "os" + "slices" + "strings" + + "cogentcore.org/core/base/logx" + "cogentcore.org/core/base/num" + "cogentcore.org/core/base/stringsx" + "golang.org/x/tools/imports" +) + +// State holds the transpiling state +type State struct { + // FuncToVar translates function definitions into variable definitions, + // which is the default for interactive use of random code fragments + // without the complete go formatting. + // For pure transpiling of a complete codebase with full proper Go formatting + // this should be turned off. + FuncToVar bool + + // depth of delim at the end of the current line. if 0, was complete. + ParenDepth, BraceDepth, BrackDepth, TypeDepth, DeclDepth int + + // Chunks of code lines that are accumulated during Transpile, + // each of which should be evaluated separately, to avoid + // issues with contextual effects from import, package etc. + Chunks []string + + // current stack of transpiled lines, that are accumulated into + // code Chunks + Lines []string + + // stack of runtime errors + Errors []error + + // if this is non-empty, it is the name of the last command defined. + // triggers insertion of the AddCommand call to add to list of defined commands. + lastCommand string +} + +// NewState returns a new transpiling state; mostly for testing +func NewState() *State { + st := &State{FuncToVar: true} + return st +} + +// TranspileCode processes each line of given code, +// adding the results to the LineStack +func (st *State) TranspileCode(code string) { + lns := strings.Split(code, "\n") + n := len(lns) + if n == 0 { + return + } + for _, ln := range lns { + hasDecl := st.DeclDepth > 0 + tl := st.TranspileLine(ln) + st.AddLine(tl) + if st.BraceDepth == 0 && st.BrackDepth == 0 && st.ParenDepth == 1 && st.lastCommand != "" { + st.lastCommand = "" + nl := len(st.Lines) + st.Lines[nl-1] = st.Lines[nl-1] + ")" + st.ParenDepth-- + } + if hasDecl && st.DeclDepth == 0 { // break at decl + st.AddChunk() + } + } +} + +// TranspileFile transpiles the given input goal file to the +// given output Go file. If no existing package declaration +// is found, then package main and func main declarations are +// added. This also affects how functions are interpreted. +func (st *State) TranspileFile(in string, out string) error { + b, err := os.ReadFile(in) + if err != nil { + return err + } + code := string(b) + lns := stringsx.SplitLines(code) + hasPackage := false + for _, ln := range lns { + if strings.HasPrefix(ln, "package ") { + hasPackage = true + break + } + } + if hasPackage { + st.FuncToVar = false // use raw functions + } + st.TranspileCode(code) + st.FuncToVar = true + if err != nil { + return err + } + gen := "// Code generated by \"goal build\"; DO NOT EDIT.\n\n" + if hasPackage { + st.Lines = slices.Insert(st.Lines, 0, gen) + } else { + st.Lines = slices.Insert(st.Lines, 0, gen, "package main", "import (", + ` "cogentcore.org/core/goal"`, + ` "cogentcore.org/core/goal/goalib"`, + ` "cogentcore.org/core/tensor"`, + ` _ "cogentcore.org/core/tensor/tmath"`, + ` _ "cogentcore.org/core/tensor/stats/stats"`, + ` _ "cogentcore.org/core/tensor/stats/metric"`, + ")", "func main() {", "goal := goal.NewGoal()") + st.Lines = append(st.Lines, "}") + } + src := []byte(st.Code()) + res, err := imports.Process(out, src, nil) + if err != nil { + res = src + slog.Error(err.Error()) + } else { + err = st.DepthError() + } + werr := os.WriteFile(out, res, 0666) + return errors.Join(err, werr) +} + +// TotalDepth returns the sum of any unresolved paren, brace, or bracket depths. +func (st *State) TotalDepth() int { + return num.Abs(st.ParenDepth) + num.Abs(st.BraceDepth) + num.Abs(st.BrackDepth) +} + +// ResetCode resets the stack of transpiled code +func (st *State) ResetCode() { + st.Chunks = nil + st.Lines = nil +} + +// ResetDepth resets the current depths to 0 +func (st *State) ResetDepth() { + st.ParenDepth, st.BraceDepth, st.BrackDepth, st.TypeDepth, st.DeclDepth = 0, 0, 0, 0, 0 +} + +// DepthError reports an error if any of the parsing depths are not zero, +// to be called at the end of transpiling a complete block of code. +func (st *State) DepthError() error { + if st.TotalDepth() == 0 { + return nil + } + str := "" + if st.ParenDepth != 0 { + str += fmt.Sprintf("Incomplete parentheses (), remaining depth: %d\n", st.ParenDepth) + } + if st.BraceDepth != 0 { + str += fmt.Sprintf("Incomplete braces [], remaining depth: %d\n", st.BraceDepth) + } + if st.BrackDepth != 0 { + str += fmt.Sprintf("Incomplete brackets {}, remaining depth: %d\n", st.BrackDepth) + } + if str != "" { + slog.Error(str) + return errors.New(str) + } + return nil +} + +// AddLine adds line on the stack +func (st *State) AddLine(ln string) { + st.Lines = append(st.Lines, ln) +} + +// Code returns the current transpiled lines, +// split into chunks that should be compiled separately. +func (st *State) Code() string { + st.AddChunk() + if len(st.Chunks) == 0 { + return "" + } + return strings.Join(st.Chunks, "\n") +} + +// AddChunk adds current lines into a chunk of code +// that should be compiled separately. +func (st *State) AddChunk() { + if len(st.Lines) == 0 { + return + } + st.Chunks = append(st.Chunks, strings.Join(st.Lines, "\n")) + st.Lines = nil +} + +// AddError adds the given error to the error stack if it is non-nil, +// and calls the Cancel function if set, to stop execution. +// This is the main way that goal errors are handled. +// It also prints the error. +func (st *State) AddError(err error) error { + if err == nil { + return nil + } + st.Errors = append(st.Errors, err) + logx.PrintlnError(err) + return err +} diff --git a/goal/token.go b/goal/transpile/token.go similarity index 97% rename from goal/token.go rename to goal/transpile/token.go index 0a2f1bf67d..a409ac7487 100644 --- a/goal/token.go +++ b/goal/transpile/token.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package goal +package transpile import ( "go/scanner" @@ -28,6 +28,30 @@ type Token struct { Pos token.Pos } +// Tokens converts the string into tokens +func TokensFromString(ln string) Tokens { + fset := token.NewFileSet() + f := fset.AddFile("", fset.Base(), len(ln)) + var sc scanner.Scanner + sc.Init(f, []byte(ln), errHandler, scanner.ScanComments|2) // 2 is non-exported dontInsertSemis + // note to Go team: just export this stuff. seriously. + + var toks Tokens + for { + pos, tok, lit := sc.Scan() + if tok == token.EOF { + break + } + // logx.PrintfDebug(" token: %s\t%s\t%q\n", fset.Position(pos), tok, lit) + toks = append(toks, &Token{Tok: tok, Pos: pos, Str: lit}) + } + return toks +} + +func errHandler(pos token.Position, msg string) { + logx.PrintlnDebug("Scan Error:", pos, msg) +} + // Tokens is a slice of Token type Tokens []*Token @@ -327,27 +351,3 @@ func (tk Tokens) ModeEnd() int { } return -1 } - -// Tokens converts the string into tokens -func (gl *Goal) Tokens(ln string) Tokens { - fset := token.NewFileSet() - f := fset.AddFile("", fset.Base(), len(ln)) - var sc scanner.Scanner - sc.Init(f, []byte(ln), gl.errHandler, scanner.ScanComments|2) // 2 is non-exported dontInsertSemis - // note to Go team: just export this stuff. seriously. - - var toks Tokens - for { - pos, tok, lit := sc.Scan() - if tok == token.EOF { - break - } - // logx.PrintfDebug(" token: %s\t%s\t%q\n", fset.Position(pos), tok, lit) - toks = append(toks, &Token{Tok: tok, Pos: pos, Str: lit}) - } - return toks -} - -func (gl *Goal) errHandler(pos token.Position, msg string) { - logx.PrintlnDebug("Scan Error:", pos, msg) -} diff --git a/goal/transpile.go b/goal/transpile/transpile.go similarity index 70% rename from goal/transpile.go rename to goal/transpile/transpile.go index ce4bb8a8c8..246c70267b 100644 --- a/goal/transpile.go +++ b/goal/transpile/transpile.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package goal +package transpile import ( "fmt" @@ -15,55 +15,55 @@ import ( // TranspileLine is the main function for parsing a single line of goal input, // returning a new transpiled line of code that converts Exec code into corresponding // Go function calls. -func (gl *Goal) TranspileLine(code string) string { +func (st *State) TranspileLine(code string) string { if len(code) == 0 { return code } if strings.HasPrefix(code, "#!") { return "" } - toks := gl.TranspileLineTokens(code) + toks := st.TranspileLineTokens(code) paren, brace, brack := toks.BracketDepths() - gl.ParenDepth += paren - gl.BraceDepth += brace - gl.BrackDepth += brack - if gl.TypeDepth > 0 && gl.BraceDepth == 0 { - gl.TypeDepth = 0 + st.ParenDepth += paren + st.BraceDepth += brace + st.BrackDepth += brack + if st.TypeDepth > 0 && st.BraceDepth == 0 { + st.TypeDepth = 0 } - if gl.DeclDepth > 0 && gl.ParenDepth == 0 { - gl.DeclDepth = 0 + if st.DeclDepth > 0 && st.ParenDepth == 0 { + st.DeclDepth = 0 } // logx.PrintlnDebug("depths: ", sh.ParenDepth, sh.BraceDepth, sh.BrackDepth) return toks.Code() } // TranspileLineTokens returns the tokens for the full line -func (gl *Goal) TranspileLineTokens(code string) Tokens { +func (st *State) TranspileLineTokens(code string) Tokens { if code == "" { return nil } - toks := gl.Tokens(code) + toks := TokensFromString(code) n := len(toks) if n == 0 { return toks } ewords, err := ExecWords(code) if err != nil { - gl.AddError(err) + st.AddError(err) return nil } logx.PrintlnDebug("\n########## line:\n", code, "\nTokens:\n", toks.String(), "\nWords:\n", ewords) if toks[0].Tok == token.TYPE { - gl.TypeDepth++ + st.TypeDepth++ } if toks[0].Tok == token.IMPORT || toks[0].Tok == token.VAR || toks[0].Tok == token.CONST { - gl.DeclDepth++ + st.DeclDepth++ } - if gl.TypeDepth > 0 || gl.DeclDepth > 0 { + if st.TypeDepth > 0 || st.DeclDepth > 0 { logx.PrintlnDebug("go: type / decl defn") - return gl.TranspileGo(toks, code) + return st.TranspileGo(toks, code) } t0 := toks[0] @@ -76,21 +76,21 @@ func (gl *Goal) TranspileLineTokens(code string) Tokens { case t0.Tok == token.ILLEGAL: if t0.Str == "#" { logx.PrintlnDebug("math #") - return gl.TranspileMath(toks[1:], code, true) + return st.TranspileMath(toks[1:], code, true) } - return gl.TranspileExec(ewords, false) + return st.TranspileExec(ewords, false) case t0.Tok == token.LBRACE: logx.PrintlnDebug("go: { } line") - return gl.TranspileGo(toks[1:n-1], code[toks[1].Pos-1:toks[n-1].Pos-1]) + return st.TranspileGo(toks[1:n-1], code[toks[1].Pos-1:toks[n-1].Pos-1]) case t0.Tok == token.LBRACK: logx.PrintlnDebug("exec: [ ] line") - return gl.TranspileExec(ewords, false) // it processes the [ ] + return st.TranspileExec(ewords, false) // it processes the [ ] case t0.Tok == token.IDENT && t0.Str == "command": - gl.lastCommand = toks[1].Str // 1 is the name -- triggers AddCommand + st.lastCommand = toks[1].Str // 1 is the name -- triggers AddCommand toks = toks[2:] // get rid of first toks.Insert(0, token.IDENT, "goal.AddCommand") toks.Insert(1, token.LPAREN) - toks.Insert(2, token.STRING, `"`+gl.lastCommand+`"`) + toks.Insert(2, token.STRING, `"`+st.lastCommand+`"`) toks.Insert(3, token.COMMA) toks.Insert(4, token.FUNC) toks.Insert(5, token.LPAREN) @@ -98,51 +98,51 @@ func (gl *Goal) TranspileLineTokens(code string) Tokens { toks.Insert(7, token.ELLIPSIS) toks.Insert(8, token.IDENT, "string") toks.Insert(9, token.RPAREN) - toks.AddTokens(gl.TranspileGo(toks[11:], code)...) + toks.AddTokens(st.TranspileGo(toks[11:], code)...) case t0.IsGo(): if t0.Tok == token.GO { if !toks.Contains(token.LPAREN) { logx.PrintlnDebug("exec: go command") - return gl.TranspileExec(ewords, false) + return st.TranspileExec(ewords, false) } } logx.PrintlnDebug("go keyword") - return gl.TranspileGo(toks, code) + return st.TranspileGo(toks, code) case toks[n-1].Tok == token.INC: - return gl.TranspileGo(toks, code) + return st.TranspileGo(toks, code) case t0pn > 0: // path expr logx.PrintlnDebug("exec: path...") - return gl.TranspileExec(ewords, false) + return st.TranspileExec(ewords, false) case t0.Tok == token.STRING: logx.PrintlnDebug("exec: string...") - return gl.TranspileExec(ewords, false) + return st.TranspileExec(ewords, false) case f0exec && en == 1: logx.PrintlnDebug("exec: 1 word") - return gl.TranspileExec(ewords, false) + return st.TranspileExec(ewords, false) case !f0exec: // exec must be IDENT logx.PrintlnDebug("go: not ident") - return gl.TranspileGo(toks, code) + return st.TranspileGo(toks, code) case f0exec && en > 1 && (ewords[1][0] == '=' || ewords[1][0] == ':' || ewords[1][0] == '+' || toks[1].Tok == token.COMMA): logx.PrintlnDebug("go: assignment or defn") - return gl.TranspileGo(toks, code) + return st.TranspileGo(toks, code) case f0exec: // now any ident logx.PrintlnDebug("exec: ident..") - return gl.TranspileExec(ewords, false) + return st.TranspileExec(ewords, false) default: logx.PrintlnDebug("go: default") - return gl.TranspileGo(toks, code) + return st.TranspileGo(toks, code) } return toks } // TranspileGo returns transpiled tokens assuming Go code. // Unpacks any encapsulated shell or math expressions. -func (gl *Goal) TranspileGo(toks Tokens, code string) Tokens { +func (st *State) TranspileGo(toks Tokens, code string) Tokens { n := len(toks) if n == 0 { return toks } - if gl.FuncToVar && toks[0].Tok == token.FUNC { // reorder as an assignment + if st.FuncToVar && toks[0].Tok == token.FUNC { // reorder as an assignment if len(toks) > 1 && toks[1].Tok == token.IDENT { toks[0] = toks[1] toks.Insert(1, token.DEFINE) @@ -157,9 +157,9 @@ func (gl *Goal) TranspileGo(toks Tokens, code string) Tokens { et := toks[i:].ModeEnd() if et > 0 { if tok.Str == "#" { - gtoks.AddTokens(gl.TranspileMath(toks[i+1:i+et], code, false)...) + gtoks.AddTokens(st.TranspileMath(toks[i+1:i+et], code, false)...) } else { - gtoks.AddTokens(gl.TranspileExecTokens(toks[i+1:i+et+1], code, true)...) + gtoks.AddTokens(st.TranspileExecTokens(toks[i+1:i+et+1], code, true)...) } i += et continue @@ -175,32 +175,32 @@ func (gl *Goal) TranspileGo(toks Tokens, code string) Tokens { // TranspileExecString returns transpiled tokens assuming Exec code, // from a string, with the given bool indicating whether [Output] is needed. -func (gl *Goal) TranspileExecString(str string, output bool) Tokens { +func (st *State) TranspileExecString(str string, output bool) Tokens { if len(str) <= 1 { return nil } ewords, err := ExecWords(str) if err != nil { - gl.AddError(err) + st.AddError(err) } - return gl.TranspileExec(ewords, output) + return st.TranspileExec(ewords, output) } // TranspileExecTokens returns transpiled tokens assuming Exec code, // from given tokens, with the given bool indicating // whether [Output] is needed. -func (gl *Goal) TranspileExecTokens(toks Tokens, code string, output bool) Tokens { +func (st *State) TranspileExecTokens(toks Tokens, code string, output bool) Tokens { nt := len(toks) if nt == 0 { return nil } str := code[toks[0].Pos-1 : toks[nt-1].Pos-1] - return gl.TranspileExecString(str, output) + return st.TranspileExecString(str, output) } // TranspileExec returns transpiled tokens assuming Exec code, // with the given bools indicating the type of run to execute. -func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { +func (st *State) TranspileExec(ewords []string, output bool) Tokens { n := len(ewords) if n == 0 { return nil @@ -246,10 +246,10 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { // case f == "#": // embedded math TODO case f == "{": // embedded go if n < i+3 { - gl.AddError(fmt.Errorf("goal: no matching right brace } found in exec command line")) + st.AddError(fmt.Errorf("goal: no matching right brace } found in exec command line")) } else { gstr := ewords[i+1] - etoks.AddTokens(gl.TranspileGo(gl.Tokens(gstr), gstr)...) + etoks.AddTokens(st.TranspileGo(TokensFromString(gstr), gstr)...) etoks.Add(token.COMMA) i += 2 } @@ -266,12 +266,12 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { etoks.Add(token.COMMA) endExec() etoks.Add(token.SEMICOLON) - etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)...) + etoks.AddTokens(st.TranspileExec(ewords[i+1:], output)...) return etoks case f == ";": endExec() etoks.Add(token.SEMICOLON) - etoks.AddTokens(gl.TranspileExec(ewords[i+1:], output)...) + etoks.AddTokens(st.TranspileExec(ewords[i+1:], output)...) return etoks default: if f[0] == '"' || f[0] == '`' { @@ -285,3 +285,9 @@ func (gl *Goal) TranspileExec(ewords []string, output bool) Tokens { endExec() return etoks } + +// TranspileMath does math mode transpiling. fullLine indicates code should be +// full statement(s). +func (st *State) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { + return MathParse(toks, code, fullLine) +} diff --git a/goal/transpile_test.go b/goal/transpile/transpile_test.go similarity index 93% rename from goal/transpile_test.go rename to goal/transpile/transpile_test.go index cd54f3038d..c5b7e089d8 100644 --- a/goal/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package goal +package transpile import ( "testing" @@ -74,9 +74,8 @@ func TestPaths(t *testing.T) { {"../an-other/dir/", `../an-other/dir/`}, {"https://google.com/search?q=hello%20world#body", `https://google.com/search?q=hello%20world#body`}, } - gl := NewGoal() for _, test := range tests { - toks := gl.Tokens(test.i) + toks := TokensFromString(test.i) p, _ := toks.Path(false) assert.Equal(t, test.e, p) } @@ -170,9 +169,9 @@ func TestTranspile(t *testing.T) { {"var data map[string]any", "var data map[string]any"}, } - gl := NewGoal() + st := NewState() for _, test := range tests { - o := gl.TranspileLine(test.i) + o := st.TranspileLine(test.i) assert.Equal(t, test.e, o) } } @@ -190,10 +189,10 @@ goal.Run("ls", "-la", "args...") })`}, } - gl := NewGoal() + st := NewState() for _, test := range tests { - gl.TranspileCode(test.i) - o := gl.Code() + st.TranspileCode(test.i) + o := st.Code() assert.Equal(t, test.e, o) } } @@ -201,16 +200,16 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - // {"# a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, + // {"# x := 1", `x := tensor.NewIntScalar(1)`}, + // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + {"# a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, } - gl := NewGoal() + st := NewState() for _, test := range tests { - o := gl.TranspileLine(test.i) + o := st.TranspileLine(test.i) assert.Equal(t, test.e, o) } } From 2de0c6fa40a6f8da847f5dcf6a48553b25c08cbd Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 17:02:38 -0700 Subject: [PATCH 069/311] basic kind inference for array literal --- goal/transpile/math.go | 6 +++- goal/transpile/types.go | 72 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 goal/transpile/types.go diff --git a/goal/transpile/math.go b/goal/transpile/math.go index df30521243..40dc812bd6 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -249,7 +249,11 @@ func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { } func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { - + kind := inferKindExprList(il.Indices) + if kind == token.ILLEGAL { + kind = token.FLOAT // default + } + // todo: make it! } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/types.go b/goal/transpile/types.go new file mode 100644 index 0000000000..471e961044 --- /dev/null +++ b/goal/transpile/types.go @@ -0,0 +1,72 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package transpile + +import ( + "go/ast" + "go/token" +) + +// inferKindExpr infers the basic Kind level type from given expression +func inferKindExpr(ex ast.Expr) token.Token { + if ex == nil { + return token.ILLEGAL + } + switch x := ex.(type) { + case *ast.BadExpr: + return token.ILLEGAL + + case *ast.Ident: + // todo: get type of object is not possible! + + case *ast.BinaryExpr: + ta := inferKindExpr(x.X) + tb := inferKindExpr(x.Y) + if ta == tb { + return ta + } + if ta != token.ILLEGAL { + return ta + } else { + return tb + } + + case *ast.BasicLit: + return x.Kind // key grounding + + case *ast.FuncLit: + + case *ast.ParenExpr: + return inferKindExpr(x.X) + + case *ast.SelectorExpr: + + case *ast.TypeAssertExpr: + + case *ast.IndexListExpr: + if x.X == nil { // array literal + return inferKindExprList(x.Indices) + } else { + return inferKindExpr(x.X) + } + + case *ast.SliceExpr: + + case *ast.CallExpr: + + } + return token.ILLEGAL +} + +func inferKindExprList(ex []ast.Expr) token.Token { + n := len(ex) + for i := range n { + t := inferKindExpr(ex[i]) + if t != token.ILLEGAL { + return t + } + } + return token.ILLEGAL +} From 4a3fe3cf512c366feb46d43578160821cc56f93a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 18 Sep 2024 23:08:11 -0700 Subject: [PATCH 070/311] numpy selectors and full math test that can be run to test actual function --- goal/math_test.go | 33 ++++ goal/transpile/math.go | 87 ++++++++- goal/transpile/state.go | 2 +- goal/transpile/transpile_test.go | 3 +- tensor/base.go | 6 +- tensor/indexed.go | 41 +++- tensor/number.go | 3 + tensor/string.go | 4 +- .../symbols/cogentcore_org-core-tensor.go | 177 +++++++++--------- yaegicore/symbols/make | 4 +- 10 files changed, 260 insertions(+), 100 deletions(-) create mode 100644 goal/math_test.go diff --git a/goal/math_test.go b/goal/math_test.go new file mode 100644 index 0000000000..8aac05cf6c --- /dev/null +++ b/goal/math_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package goal + +import ( + "testing" + + "cogentcore.org/core/goal/goalib" + "github.com/stretchr/testify/assert" +) + +var test = ` +# x := [3, 5, 4] +# nd := x.ndim +# sz := x.size +# sh := x.shape + +fmt.Println(x) +fmt.Println(nd) +fmt.Println(sz) +fmt.Println(sh) +` + +func TestMath(t *testing.T) { + gl := NewGoal() + tfile := "testdata/test.goal" + ofile := "testdata/test.go" + goalib.WriteFile(tfile, test) + err := gl.TranspileFile(tfile, ofile) + assert.NoError(t, err) +} diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 40dc812bd6..9d0addcc92 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -18,9 +18,13 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { // fmt.Println(nt, toks) str := code[toks[0].Pos-1 : toks[nt-1].Pos] + if toks[nt-1].Str != "" { + str += toks[nt-1].Str[1:] + } + // fmt.Println(str) mp := mathParse{toks: toks, code: code} - mods := AllErrors | Trace + mods := AllErrors // | Trace if fullLine { stmts, err := ParseLine(str, mods) @@ -41,11 +45,17 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { return mp.out } +// mathParse has the parsing state type mathParse struct { code string // code string toks Tokens // source tokens we are parsing idx int // current index in source tokens out Tokens // output tokens we generate + + // goLiteral means generate basic literals as standard go literals instead of + // wrapping them in tensor constructors. for inner expressions contstructing go + // objects etc. + goLiteral bool } // addToken adds output token and increments idx @@ -228,6 +238,10 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { } func (mp *mathParse) basicLit(lit *ast.BasicLit) { + if mp.goLiteral { + mp.out.Add(lit.Kind, lit.Value) + return + } switch lit.Kind { case token.INT: mp.out.Add(token.IDENT, "tensor.NewIntScalar("+lit.Value+")") @@ -241,7 +255,55 @@ func (mp *mathParse) basicLit(lit *ast.BasicLit) { } } +type funWrap struct { + fun string + wrap string +} + +// nis: NewIntScalar +var numpyProps = map[string]funWrap{ + "ndim": {"NumDims()", "nis"}, + "len": {"Len()", "nis"}, + "size": {"Len()", "nis"}, + "shape": {"Shape().Sizes", "nifs"}, +} + func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { + fw, ok := numpyProps[ex.Sel.Name] + if !ok { + mp.expr(ex.X) + mp.addToken(token.PERIOD) + mp.out.Add(token.IDENT, ex.Sel.Name) + mp.idx++ + return + } + elip := false + switch fw.wrap { + case "nis": + mp.out.Add(token.IDENT, "tensor.NewIntScalar") + case "nfs": + mp.out.Add(token.IDENT, "tensor.NewFloat64Scalar") + case "nss": + mp.out.Add(token.IDENT, "tensor.NewStringScalar") + case "nifs": + mp.out.Add(token.IDENT, "tensor.NewIntFromSlice") + elip = true + case "nffs": + mp.out.Add(token.IDENT, "tensor.NewFloat64FromSlice") + elip = true + case "nsfs": + mp.out.Add(token.IDENT, "tensor.NewStringFromSlice") + elip = true + } + mp.out.Add(token.LPAREN) + mp.expr(ex.X) + mp.addToken(token.PERIOD) + mp.out.Add(token.IDENT, fw.fun) + mp.idx++ + if elip { + mp.out.Add(token.ELLIPSIS) + } + mp.out.Add(token.RPAREN) } func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { @@ -253,7 +315,28 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { if kind == token.ILLEGAL { kind = token.FLOAT // default } - // todo: make it! + // todo: look for sub-arrays etc. + typ := "float64" + fun := "Float" + switch kind { + case token.FLOAT: + case token.INT: + typ = "int" + fun = "Int" + case token.STRING: + typ = "string" + fun = "String" + } + mp.out.Add(token.IDENT, "tensor.New"+fun+"FromSlice") + mp.out.Add(token.LPAREN) + mp.out.Add(token.IDENT, "[]"+typ) + mp.addToken(token.LBRACE) + mp.goLiteral = true + mp.exprList(il.Indices) + mp.goLiteral = false + mp.addToken(token.RBRACE) + mp.addToken(token.ELLIPSIS) + mp.out.Add(token.RPAREN, ")") } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/state.go b/goal/transpile/state.go index a1e8c24afe..48682a105e 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -114,7 +114,7 @@ func (st *State) TranspileFile(in string, out string) error { ` _ "cogentcore.org/core/tensor/tmath"`, ` _ "cogentcore.org/core/tensor/stats/stats"`, ` _ "cogentcore.org/core/tensor/stats/metric"`, - ")", "func main() {", "goal := goal.NewGoal()") + ")", "func main() {", "goal := goal.NewGoal()", "_ = goal") st.Lines = append(st.Lines, "}") } src := []byte(st.Code()) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index c5b7e089d8..635a9653a7 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,8 @@ func TestMath(t *testing.T) { // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - {"# a = [1,2,3,4]", `a = tensor.NewNumberFromSlice([]int{1,2,3,4})`}, + // {"# a := [1,2,3,4]", `a := tensor.NewIntFromSlice([]int { 1, 2, 3, 4 } ...)`}, + {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, } st := NewState() diff --git a/tensor/base.go b/tensor/base.go index 3ca8eb6b38..cf3d8af854 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -215,11 +215,9 @@ func stringIndexed(tsr Tensor, maxLen int, idxs []int) string { if maxLen == 0 { maxLen = MaxSprintLength } - str := tsr.Label() var b strings.Builder - b.WriteString(str) - b.WriteString("\n") - oddRow := true + b.WriteString(tsr.Shape().String() + " ") + oddRow := false rows, cols, _, _ := Projection2DShape(tsr.Shape(), oddRow) if idxs != nil { rows = min(rows, len(idxs)) diff --git a/tensor/indexed.go b/tensor/indexed.go index af8b1b5cc4..e4a94b7e7b 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -11,6 +11,8 @@ import ( "slices" "sort" "strings" + + "cogentcore.org/core/base/metadata" ) // Indexed is an indexed wrapper around a tensor.Tensor that provides a @@ -74,7 +76,28 @@ func NewIntScalar(val int) *Indexed { // NewStringScalar is a convenience method to quickly get an Indexed // representation of a single string scalar value, for use in math routines etc. func NewStringScalar(val string) *Indexed { - return &Indexed{Tensor: NewStringFromSlice(val)} + return &Indexed{Tensor: NewStringTensorFromSlice(val)} +} + +// NewFloat64FromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewFloat64FromSlice(vals ...float64) *Indexed { + return &Indexed{Tensor: NewNumberFromSlice(vals...)} +} + +// NewIntFromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewIntFromSlice(vals ...int) *Indexed { + return &Indexed{Tensor: NewNumberFromSlice(vals...)} +} + +// NewStringFromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewStringFromSlice(vals ...string) *Indexed { + return &Indexed{Tensor: NewStringTensorFromSlice(vals...)} } // SetTensor sets as indexes into given tensor with sequential initial indexes @@ -114,6 +137,22 @@ func (ix *Indexed) Label() string { return ix.Tensor.Label() } +// note: goal transpiling needs all expressions to work directly on Indexed +// so we need wrappers for everything. + +// Shape returns a pointer to the shape that fully parametrizes the tensor shape. +func (ix *Indexed) Shape() *Shape { return ix.Tensor.Shape() } + +// Metadata returns the metadata for this tensor, which can be used +// to encode plotting options, etc. +func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } + +// NumDims returns the total number of dimensions. +func (ix *Indexed) NumDims() int { return ix.Tensor.NumDims() } + +// DimSize returns size of given dimension. +func (ix *Indexed) DimSize(dim int) int { return ix.Tensor.DimSize(dim) } + // RowCellSize returns the size of the outermost Row shape dimension // (via [Indexed.Rows] method), and the size of all the remaining // inner dimensions (the "cell" size). diff --git a/tensor/number.go b/tensor/number.go index 7a52fecde6..1cc0e79785 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -110,6 +110,9 @@ func NewNumberShape[T num.Number](shape *Shape) *Number[T] { return tsr } +// todo: this should in principle work with yaegi:add but it is crashing +// will come back to it later. + // NewNumberFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. diff --git a/tensor/string.go b/tensor/string.go index 74ac0c1193..59bfead895 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -37,10 +37,10 @@ func NewStringShape(shape *Shape) *String { return tsr } -// NewStringFromSlice returns a new 1-dimensional tensor of given value type +// NewStringTensorFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewStringFromSlice(vals ...string) Tensor { +func NewStringTensorFromSlice(vals ...string) Tensor { n := len(vals) tsr := &String{} tsr.Values = vals diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 52f1041901..4771e5d312 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -12,93 +12,96 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/tensor"] = map[string]reflect.Value{ // function, constant and variable definitions - "AddFunc": reflect.ValueOf(tensor.AddFunc), - "AddShapes": reflect.ValueOf(tensor.AddShapes), - "AsFloat32": reflect.ValueOf(tensor.AsFloat32), - "AsFloat64": reflect.ValueOf(tensor.AsFloat64), - "Ascending": reflect.ValueOf(tensor.Ascending), - "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), - "BoolToInt": reflect.ValueOf(tensor.BoolToInt), - "Calc": reflect.ValueOf(tensor.Calc), - "Call": reflect.ValueOf(tensor.Call), - "CallOut": reflect.ValueOf(tensor.CallOut), - "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), - "CallString": reflect.ValueOf(tensor.CallString), - "ColMajorStrides": reflect.ValueOf(tensor.ColMajorStrides), - "Comma": reflect.ValueOf(tensor.Comma), - "Contains": reflect.ValueOf(tensor.Contains), - "CopyDense": reflect.ValueOf(tensor.CopyDense), - "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), - "DelimsN": reflect.ValueOf(tensor.DelimsN), - "DelimsValues": reflect.ValueOf(tensor.DelimsValues), - "Descending": reflect.ValueOf(tensor.Descending), - "Detect": reflect.ValueOf(tensor.Detect), - "Equals": reflect.ValueOf(tensor.Equals), - "Exclude": reflect.ValueOf(tensor.Exclude), - "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), - "Float64ToString": reflect.ValueOf(tensor.Float64ToString), - "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), - "GetPrecision": reflect.ValueOf(tensor.GetPrecision), - "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), - "Include": reflect.ValueOf(tensor.Include), - "IntToBool": reflect.ValueOf(tensor.IntToBool), - "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), - "NFirstLen": reflect.ValueOf(tensor.NFirstLen), - "NFirstRows": reflect.ValueOf(tensor.NFirstRows), - "NMinLen": reflect.ValueOf(tensor.NMinLen), - "New1DViewOf": reflect.ValueOf(tensor.New1DViewOf), - "NewBits": reflect.ValueOf(tensor.NewBits), - "NewBitsShape": reflect.ValueOf(tensor.NewBitsShape), - "NewByte": reflect.ValueOf(tensor.NewByte), - "NewFloat32": reflect.ValueOf(tensor.NewFloat32), - "NewFloat64": reflect.ValueOf(tensor.NewFloat64), - "NewFloat64Indexed": reflect.ValueOf(tensor.NewFloat64Indexed), - "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), - "NewFunc": reflect.ValueOf(tensor.NewFunc), - "NewIndexed": reflect.ValueOf(tensor.NewIndexed), - "NewInt": reflect.ValueOf(tensor.NewInt), - "NewInt32": reflect.ValueOf(tensor.NewInt32), - "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), - "NewOfType": reflect.ValueOf(tensor.NewOfType), - "NewShape": reflect.ValueOf(tensor.NewShape), - "NewString": reflect.ValueOf(tensor.NewString), - "NewStringFromSlice": reflect.ValueOf(tensor.NewStringFromSlice), - "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), - "NewStringShape": reflect.ValueOf(tensor.NewStringShape), - "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), - "OddColumn": reflect.ValueOf(tensor.OddColumn), - "OddRow": reflect.ValueOf(tensor.OddRow), - "OpenCSV": reflect.ValueOf(tensor.OpenCSV), - "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), - "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), - "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), - "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), - "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), - "Projection2DString": reflect.ValueOf(tensor.Projection2DString), - "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), - "ReadCSV": reflect.ValueOf(tensor.ReadCSV), - "RowCellSplit": reflect.ValueOf(tensor.RowCellSplit), - "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), - "SaveCSV": reflect.ValueOf(tensor.SaveCSV), - "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), - "SetPrecision": reflect.ValueOf(tensor.SetPrecision), - "Slice": reflect.ValueOf(tensor.Slice), - "SliceSet": reflect.ValueOf(tensor.SliceSet), - "SliceSize": reflect.ValueOf(tensor.SliceSize), - "Space": reflect.ValueOf(tensor.Space), - "Sprint": reflect.ValueOf(tensor.Sprint), - "SprintIndexed": reflect.ValueOf(tensor.SprintIndexed), - "Stable": reflect.ValueOf(tensor.Stable), - "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), - "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), - "Tab": reflect.ValueOf(tensor.Tab), - "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), - "Unstable": reflect.ValueOf(tensor.Unstable), - "UseCase": reflect.ValueOf(tensor.UseCase), - "Vectorize": reflect.ValueOf(tensor.Vectorize), - "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), - "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), - "WriteCSV": reflect.ValueOf(tensor.WriteCSV), + "AddFunc": reflect.ValueOf(tensor.AddFunc), + "AddShapes": reflect.ValueOf(tensor.AddShapes), + "AsFloat32": reflect.ValueOf(tensor.AsFloat32), + "AsFloat64": reflect.ValueOf(tensor.AsFloat64), + "Ascending": reflect.ValueOf(tensor.Ascending), + "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), + "BoolToInt": reflect.ValueOf(tensor.BoolToInt), + "Calc": reflect.ValueOf(tensor.Calc), + "Call": reflect.ValueOf(tensor.Call), + "CallOut": reflect.ValueOf(tensor.CallOut), + "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), + "CallString": reflect.ValueOf(tensor.CallString), + "ColMajorStrides": reflect.ValueOf(tensor.ColMajorStrides), + "Comma": reflect.ValueOf(tensor.Comma), + "Contains": reflect.ValueOf(tensor.Contains), + "CopyDense": reflect.ValueOf(tensor.CopyDense), + "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), + "DelimsN": reflect.ValueOf(tensor.DelimsN), + "DelimsValues": reflect.ValueOf(tensor.DelimsValues), + "Descending": reflect.ValueOf(tensor.Descending), + "Detect": reflect.ValueOf(tensor.Detect), + "Equals": reflect.ValueOf(tensor.Equals), + "Exclude": reflect.ValueOf(tensor.Exclude), + "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), + "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), + "GetPrecision": reflect.ValueOf(tensor.GetPrecision), + "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), + "Include": reflect.ValueOf(tensor.Include), + "IntToBool": reflect.ValueOf(tensor.IntToBool), + "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), + "NFirstLen": reflect.ValueOf(tensor.NFirstLen), + "NFirstRows": reflect.ValueOf(tensor.NFirstRows), + "NMinLen": reflect.ValueOf(tensor.NMinLen), + "New1DViewOf": reflect.ValueOf(tensor.New1DViewOf), + "NewBits": reflect.ValueOf(tensor.NewBits), + "NewBitsShape": reflect.ValueOf(tensor.NewBitsShape), + "NewByte": reflect.ValueOf(tensor.NewByte), + "NewFloat32": reflect.ValueOf(tensor.NewFloat32), + "NewFloat64": reflect.ValueOf(tensor.NewFloat64), + "NewFloat64FromSlice": reflect.ValueOf(tensor.NewFloat64FromSlice), + "NewFloat64Indexed": reflect.ValueOf(tensor.NewFloat64Indexed), + "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), + "NewFunc": reflect.ValueOf(tensor.NewFunc), + "NewIndexed": reflect.ValueOf(tensor.NewIndexed), + "NewInt": reflect.ValueOf(tensor.NewInt), + "NewInt32": reflect.ValueOf(tensor.NewInt32), + "NewIntFromSlice": reflect.ValueOf(tensor.NewIntFromSlice), + "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), + "NewOfType": reflect.ValueOf(tensor.NewOfType), + "NewShape": reflect.ValueOf(tensor.NewShape), + "NewString": reflect.ValueOf(tensor.NewString), + "NewStringFromSlice": reflect.ValueOf(tensor.NewStringFromSlice), + "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), + "NewStringShape": reflect.ValueOf(tensor.NewStringShape), + "NewStringTensorFromSlice": reflect.ValueOf(tensor.NewStringTensorFromSlice), + "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), + "OddColumn": reflect.ValueOf(tensor.OddColumn), + "OddRow": reflect.ValueOf(tensor.OddRow), + "OpenCSV": reflect.ValueOf(tensor.OpenCSV), + "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), + "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), + "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), + "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), + "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), + "Projection2DString": reflect.ValueOf(tensor.Projection2DString), + "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), + "ReadCSV": reflect.ValueOf(tensor.ReadCSV), + "RowCellSplit": reflect.ValueOf(tensor.RowCellSplit), + "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), + "SaveCSV": reflect.ValueOf(tensor.SaveCSV), + "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), + "SetPrecision": reflect.ValueOf(tensor.SetPrecision), + "Slice": reflect.ValueOf(tensor.Slice), + "SliceSet": reflect.ValueOf(tensor.SliceSet), + "SliceSize": reflect.ValueOf(tensor.SliceSize), + "Space": reflect.ValueOf(tensor.Space), + "Sprint": reflect.ValueOf(tensor.Sprint), + "SprintIndexed": reflect.ValueOf(tensor.SprintIndexed), + "Stable": reflect.ValueOf(tensor.Stable), + "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), + "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), + "Tab": reflect.ValueOf(tensor.Tab), + "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), + "Unstable": reflect.ValueOf(tensor.Unstable), + "UseCase": reflect.ValueOf(tensor.UseCase), + "Vectorize": reflect.ValueOf(tensor.Vectorize), + "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), + "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), + "WriteCSV": reflect.ValueOf(tensor.WriteCSV), // type definitions "Bits": reflect.ValueOf((*tensor.Bits)(nil)), diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 1a78ad064f..c2fe416f10 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -1,4 +1,4 @@ -#!/usr/bin/env goal +#!/usr/bin/env cosh yaegi extract fmt strconv strings image image/color image/draw time log/slog reflect @@ -8,4 +8,4 @@ command extract { } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo base/num From 9ef24f100b16c4dfa00660598caa1b2358f4b13c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 00:37:21 -0700 Subject: [PATCH 071/311] math table updated --- goal/README.md | 481 +++++++------------------------------------------ 1 file changed, 70 insertions(+), 411 deletions(-) diff --git a/goal/README.md b/goal/README.md index 57a4585c5f..19054f4719 100644 --- a/goal/README.md +++ b/goal/README.md @@ -239,186 +239,78 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | same: | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D array a | | same: | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D array a; unspecified dimensions are equivalent to `:` | | same: | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | -| same: | `a[-5:]` | `a(end-4:end,:)` | first 5 rows of 2D array a | -| same: | `a[0:3, 4:9]` | `a(1:3,5:9)` | last 5 rows of 2D array a | -| same: | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | The first through third rows and fifth through ninth columns of a 2D array, a. | +| same: | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D array a | +| same: | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D array, a. | +| same: | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | +| same: (makes a copy) | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of a, starting with the third and going to the twenty-first | +| same: | `a[::2, :]` | `a(1:2:end,:)` | every other row of a, starting with the first | +| same: | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | a with rows in reverse order | +| same: | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | a with copy of the first row appended to the end | +| same: | `a.transpose() or a.T` | `a.'` | transpose of a | +| same: | `a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of a | +| same: | `a @ b` | `a * b` | matrix multiply | +| same: | `a * b` | `a .* b` | element-wise multiply | +| same: | `a/b` | `a./b` | element-wise divide | +| `a^3` or: | `a**3` | `a.^3` | element-wise exponentiation | +| same: | `(a > 0.5)` | `(a > 0.5)` | matrix whose `i,jth` element is `(a_ij > 0.5)`. The MATLAB result is an array of logical values 0 and 1. The NumPy result is an array of the boolean values False and True. +| same: | `np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | +| same: | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of a where vector v > 0.5 | +| same: | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of a where column vector v > 0.5 | +| same: | `a[a < 0.5]=0` | `a(a<0.5)=0` | a with elements less than 0.5 zeroed out | +| same: | `a * (a > 0.5)` | `a .* (a>0.5)` | a with elements less than 0.5 zeroed out | +| same: | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | +| same: | `y = x.copy()` | `y=x` | NumPy assigns by reference | +| same: | `y = x[1, :].copy()` | `y=x(2,:)` | NumPy slices are by reference | +| same: | `y = x.flatten()` | `y=x(:)` | turn array into vector (note that this forces a copy). To obtain the same data ordering as in MATLAB, use x.flatten('F'). | +| same: | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector (see note RANGES) | +| same: | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector (see note RANGES) | +| same: | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | +| same: | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional array full of 64-bit floating point zeros | +| same: | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional array full of 64-bit floating point zeros | +| same: | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional array full of 64-bit floating point ones | +| same: | `np.eye(3)` | `eye(3)` | 3x3 identity matrix | +| same: | `np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D array, a | +| same: | `np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | +| same: | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | +| same: | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D arrays: one of x values, the other of y values | +| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | the best way to eval functions on a grid | +| same: | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | +| same: | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | +| same: | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | +| same: | `np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | +| same: | `np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | +| same: | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of a (with ndims(a)<=2 for MATLAB, if there are NaN’s, nanmax will ignore these and return largest value) | +| same: | `a.max(0)` | `max(a)` | maximum element of each column of array a | +| same: | `a.max(1)` | `max(a,[],2)` | maximum element of each row of array a | +| same: | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | +| same: | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | +| same: | `logical_and(a,b)` | `a & b` | element-by-element AND operator (NumPy ufunc) See note LOGICOPS | +| same: | `np.logical_or(a,b)` | `a | b` | element-by-element OR operator (NumPy ufunc) | +| same: | `a & b` | `bitand(a,b)` | bitwise AND operator (Python native and NumPy ufunc) | +| same: | `a | b` | `bitor(a,b)` | bitwise OR operator (Python native and NumPy ufunc) | +| same: | `linalg.inv(a)` | `inv(a)` | inverse of square 2D array a | +| same: | `linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D array a | +| same: | `np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D array a | +| same: | `linalg.solve(a, b)` if `a` is square; `linalg.lstsq(a, b)` otherwise | `a\b` | solution of `a x = b` for x | +| same: | Solve `a.T x.T = b.T` instead | `b/a` | solution of x a = b for x | +| same: | `U, S, Vh = linalg.svd(a); V = Vh.T` | `[U,S,V]=svd(a)` | singular value decomposition of a | +| same: | `linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D array | +| same: | `D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | +| same: | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D array, a | +| same: | `Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition +| same: | `P,L,U = linalg.lu(a) where a == P@L@U` | `[L,U,P]=lu(a) where a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | +| same: | `cg` | `conjgrad` | conjugate gradients solver | +| same: | `np.fft.fft(a)` | `fft(a)` | Fourier transform of a | +| same: | `np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of a | +| same: | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D array, a | +| same: | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D array, a | +| same: | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the array a as array b with rows sorted by the first column | +| same: | `x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | +| same: | `signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | +| same: | `np.unique(a)` | `unique(a)` | a vector of unique values in array a | +| same: | `a.squeeze()` | `squeeze(a)` | remove singleton dimensions of array a. Note that MATLAB will always return arrays of 2D or higher while NumPy will return arrays of 0D or higher | -rows 2,4 and 5 and columns 1 and 3. This allows the matrix to be modified, and doesn’t require a regular slice. - -a(3:2:21,:) - -a[2:21:2,:] - -every other row of a, starting with the third and going to the twenty-first - -a(1:2:end,:) - -a[::2, :] - -every other row of a, starting with the first - -a(end:-1:1,:) or flipud(a) - -a[::-1,:] - -a with rows in reverse order - -a([1:end 1],:) - -a[np.r_[:len(a),0]] - -a with copy of the first row appended to the end - -a.' - -a.transpose() or a.T - -transpose of a - -a' - -a.conj().transpose() or a.conj().T - -conjugate transpose of a - -a * b - -a @ b - -matrix multiply - -a .* b - -a * b - -element-wise multiply - -a./b - -a/b - -element-wise divide - -a.^3 - -a**3 - -element-wise exponentiation - -(a > 0.5) - -(a > 0.5) - -matrix whose i,jth element is (a_ij > 0.5). The MATLAB result is an array of logical values 0 and 1. The NumPy result is an array of the boolean values False and True. - -find(a > 0.5) - -np.nonzero(a > 0.5) - -find the indices where (a > 0.5) - -a(:,find(v > 0.5)) - -a[:,np.nonzero(v > 0.5)[0]] - -extract the columns of a where vector v > 0.5 - -a(:,find(v>0.5)) - -a[:, v.T > 0.5] - -extract the columns of a where column vector v > 0.5 - -a(a<0.5)=0 - -a[a < 0.5]=0 - -a with elements less than 0.5 zeroed out - -a .* (a>0.5) - -a * (a > 0.5) - -a with elements less than 0.5 zeroed out - -a(:) = 3 - -a[:] = 3 - -set all values to the same scalar value - -y=x - -y = x.copy() - -NumPy assigns by reference - -y=x(2,:) - -y = x[1, :].copy() - -NumPy slices are by reference - -y=x(:) - -y = x.flatten() - -turn array into vector (note that this forces a copy). To obtain the same data ordering as in MATLAB, use x.flatten('F'). - -1:10 - -np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j] - -create an increasing vector (see note RANGES) - -0:9 - -np.arange(10.) or np.r_[:10.] or np.r_[:9:10j] - -create an increasing vector (see note RANGES) - -[1:10]' - -np.arange(1.,11.)[:, np.newaxis] - -create a column vector - -zeros(3,4) - -np.zeros((3, 4)) - -3x4 two-dimensional array full of 64-bit floating point zeros - -zeros(3,4,5) - -np.zeros((3, 4, 5)) - -3x4x5 three-dimensional array full of 64-bit floating point zeros - -ones(3,4) - -np.ones((3, 4)) - -3x4 two-dimensional array full of 64-bit floating point ones - -eye(3) - -np.eye(3) - -3x3 identity matrix - -diag(a) - -np.diag(a) - -returns a vector of the diagonal elements of 2D array, a - -diag(v,0) - -np.diag(v, 0) - -returns a square diagonal matrix whose nonzero values are the elements of vector, v rng(42,'twister') rand(3,4) @@ -429,236 +321,3 @@ or older version: random.rand((3, 4)) generate a random 3x4 array with default random number generator and seed = 42 -linspace(1,3,4) - -np.linspace(1,3,4) - -4 equally spaced samples between 1 and 3, inclusive - -[x,y]=meshgrid(0:8,0:5) - -np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.]) - -two 2D arrays: one of x values, the other of y values - -ogrid[0:9.,0:6.] or np.ix_(np.r_[0:9.],np.r_[0:6.] - -the best way to eval functions on a grid - -[x,y]=meshgrid([1,2,4],[2,4,5]) - -np.meshgrid([1,2,4],[2,4,5]) - -np.ix_([1,2,4],[2,4,5]) - -the best way to eval functions on a grid - -repmat(a, m, n) - -np.tile(a, (m, n)) - -create m by n copies of a - -[a b] - -np.concatenate((a,b),1) or np.hstack((a,b)) or np.column_stack((a,b)) or np.c_[a,b] - -concatenate columns of a and b - -[a; b] - -np.concatenate((a,b)) or np.vstack((a,b)) or np.r_[a,b] - -concatenate rows of a and b - -max(max(a)) - -a.max() or np.nanmax(a) - -maximum element of a (with ndims(a)<=2 for MATLAB, if there are NaN’s, nanmax will ignore these and return largest value) - -max(a) - -a.max(0) - -maximum element of each column of array a - -max(a,[],2) - -a.max(1) - -maximum element of each row of array a - -max(a,b) - -np.maximum(a, b) - -compares a and b element-wise, and returns the maximum value from each pair - -norm(v) - -np.sqrt(v @ v) or np.linalg.norm(v) - -L2 norm of vector v - -a & b - -logical_and(a,b) - -element-by-element AND operator (NumPy ufunc) See note LOGICOPS - -a | b - -np.logical_or(a,b) - -element-by-element OR operator (NumPy ufunc) See note LOGICOPS - -bitand(a,b) - -a & b - -bitwise AND operator (Python native and NumPy ufunc) - -bitor(a,b) - -a | b - -bitwise OR operator (Python native and NumPy ufunc) - -inv(a) - -linalg.inv(a) - -inverse of square 2D array a - -pinv(a) - -linalg.pinv(a) - -pseudo-inverse of 2D array a - -rank(a) - -np.linalg.matrix_rank(a) - -matrix rank of a 2D array a - -a\b - -linalg.solve(a, b) if a is square; linalg.lstsq(a, b) otherwise - -solution of a x = b for x - -b/a - -Solve a.T x.T = b.T instead - -solution of x a = b for x - -[U,S,V]=svd(a) - -U, S, Vh = linalg.svd(a); V = Vh.T - -singular value decomposition of a - -chol(a) - -linalg.cholesky(a) - -Cholesky factorization of a 2D array - -[V,D]=eig(a) - -D,V = linalg.eig(a) - -eigenvalues - and eigenvectors - of a, where - -[V,D]=eig(a,b) - -D,V = linalg.eig(a, b) - -eigenvalues - and eigenvectors - of a, b where - -[V,D]=eigs(a,3) - -D,V = eigs(a, k=3) - -find the k=3 largest eigenvalues and eigenvectors of 2D array, a - -[Q,R]=qr(a,0) - -Q,R = linalg.qr(a) - -QR decomposition - -[L,U,P]=lu(a) where a==P'*L*U - -P,L,U = linalg.lu(a) where a == P@L@U - -LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) - -conjgrad - -cg - -conjugate gradients solver - -fft(a) - -np.fft.fft(a) - -Fourier transform of a - -ifft(a) - -np.fft.ifft(a) - -inverse Fourier transform of a - -sort(a) - -np.sort(a) or a.sort(axis=0) - -sort each column of a 2D array, a - -sort(a, 2) - -np.sort(a, axis=1) or a.sort(axis=1) - -sort the each row of 2D array, a - -[b,I]=sortrows(a,1) - -I = np.argsort(a[:, 0]); b = a[I,:] - -save the array a as array b with rows sorted by the first column - -x = Z\y - -x = linalg.lstsq(Z, y) - -perform a linear regression of the form - -decimate(x, q) - -signal.resample(x, np.ceil(len(x)/q)) - -downsample with low-pass filtering - -unique(a) - -np.unique(a) - -a vector of unique values in array a - -squeeze(a) - -a.squeeze() - -remove singleton dimensions of array a. Note that MATLAB will always return arrays of 2D or higher while NumPy will return arrays of 0D or higher - - From 19354cc0b6fe058967c91d89f6617db0a93fcb45 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 01:12:16 -0700 Subject: [PATCH 072/311] function calls --- goal/README.md | 2 +- goal/math_test.go | 3 ++- goal/transpile/math.go | 25 +++++++++++++++++++++++-- goal/transpile/transpile_test.go | 3 ++- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/goal/README.md b/goal/README.md index 19054f4719..588d3e3b8a 100644 --- a/goal/README.md +++ b/goal/README.md @@ -273,7 +273,7 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | same: | `np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | | same: | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | | same: | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D arrays: one of x values, the other of y values | -| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | the best way to eval functions on a grid | +| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | | same: | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | | same: | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | | same: | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | diff --git a/goal/math_test.go b/goal/math_test.go index 8aac05cf6c..51d4ee7b50 100644 --- a/goal/math_test.go +++ b/goal/math_test.go @@ -12,7 +12,8 @@ import ( ) var test = ` -# x := [3, 5, 4] +// # x := [3, 5, 4] +# x := zeros(3, 4) # nd := x.ndim # sz := x.size # sh := x.shape diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 9d0addcc92..03018be454 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -187,6 +187,7 @@ func (mp *mathParse) expr(ex ast.Expr) { // todo: probably this is subsumed in indexListExpr case *ast.CallExpr: + mp.callExpr(x) case *ast.ArrayType: // basically at this point we have a bad expression and @@ -260,7 +261,7 @@ type funWrap struct { wrap string } -// nis: NewIntScalar +// nis: NewIntScalar, nifs: NewIntFromSlice var numpyProps = map[string]funWrap{ "ndim": {"NumDims()", "nis"}, "len": {"Len()", "nis"}, @@ -336,10 +337,30 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { mp.goLiteral = false mp.addToken(token.RBRACE) mp.addToken(token.ELLIPSIS) - mp.out.Add(token.RPAREN, ")") + mp.out.Add(token.RPAREN) +} + +var numpyFuncs = map[string]funWrap{ + "zeros": {"tensor.NewFloat64Indexed", ""}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { + if fnm, ok := ex.Fun.(*ast.Ident); ok { + if fw, ok := numpyFuncs[fnm.Name]; ok { + // todo: wrap + mp.out.Add(token.IDENT, fw.fun) + } else { + mp.out.Add(token.IDENT, fnm.Name) + } + } else { + mp.expr(ex.Fun) + } + mp.addToken(token.LPAREN) + mp.goLiteral = true + mp.exprList(ex.Args) + mp.goLiteral = false + // todo: ellipsis + mp.addToken(token.RPAREN) } func (mp *mathParse) ident(id *ast.Ident) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 635a9653a7..33b059513b 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -205,7 +205,8 @@ func TestMath(t *testing.T) { // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, // {"# a := [1,2,3,4]", `a := tensor.NewIntFromSlice([]int { 1, 2, 3, 4 } ...)`}, - {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64Indexed(3, 4)`}, } st := NewState() From cf76580f2bea179775c74b03f34f3efeaba0b76b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 08:39:32 -0700 Subject: [PATCH 073/311] table experiments --- goal/README.md | 161 +++++++++++++++++++++++++------------------------ 1 file changed, 82 insertions(+), 79 deletions(-) diff --git a/goal/README.md b/goal/README.md index 588d3e3b8a..1abe2aed44 100644 --- a/goal/README.md +++ b/goal/README.md @@ -221,7 +221,7 @@ TODO: update aboven # Math mode -In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. The `np.` prefix on numpy global functions is optional, and corresponding field-like properties of tensors turn into the appropriate methods during the transpiling process. +In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as numpy, without the `np.` prefix, so existing code can easily be converted by just removing that. Corresponding field-like properties of tensors are converted into into appropriate method calls. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. @@ -229,86 +229,89 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | Goal | Python | MATLAB | Notes | | ----- | ------ | ------ | ------ | -| same: | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of array `a` | -| `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of array `a` | -| same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` array | -| same: | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of array a | +| `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of array `a` | +| `len(a)` or `a.len` or `size(a)` or `a.size` | `np.size(a)` or `a.size` | `numel(a)` | number of elements of array `a` | +| `shape(a)` or `a.shape` | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` array | +| `a.shape[n-1]` | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of array a | +| | _Construction_ | | | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D array | -| same: | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks a, b, c, and d | -| same: | `a[-1]` | `a(end)` | access last element | -| same: | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D array a | -| same: | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D array a; unspecified dimensions are equivalent to `:` | -| same: | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | -| same: | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D array a | -| same: | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D array, a. | -| same: | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | -| same: (makes a copy) | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of a, starting with the third and going to the twenty-first | -| same: | `a[::2, :]` | `a(1:2:end,:)` | every other row of a, starting with the first | -| same: | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | a with rows in reverse order | -| same: | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | a with copy of the first row appended to the end | -| same: | `a.transpose() or a.T` | `a.'` | transpose of a | -| same: | `a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of a | -| same: | `a @ b` | `a * b` | matrix multiply | -| same: | `a * b` | `a .* b` | element-wise multiply | -| same: | `a/b` | `a./b` | element-wise divide | +| `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks a, b, c, and d | +| | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional array full of 64-bit floating point zeros | +| | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional array full of 64-bit floating point zeros | +| | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional array full of 64-bit floating point ones | +| | | | +| | `a[-1]` | `a(end)` | access last element | +| | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D array a | +| | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D array a; unspecified dimensions are equivalent to `:` | +| | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | +| | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D array a | +| | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D array, a. | +| | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | +| (makes a copy) | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of a, starting with the third and going to the twenty-first | +| | `a[::2, :]` | `a(1:2:end,:)` | every other row of a, starting with the first | +| | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | a with rows in reverse order | +| | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | a with copy of the first row appended to the end | +| | | | +| | `a.transpose() or a.T` | `a.'` | transpose of a | +| | `a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of a | +| | `a @ b` | `a * b` | matrix multiply | +| | `a * b` | `a .* b` | element-wise multiply | +| | `a/b` | `a./b` | element-wise divide | | `a^3` or: | `a**3` | `a.^3` | element-wise exponentiation | -| same: | `(a > 0.5)` | `(a > 0.5)` | matrix whose `i,jth` element is `(a_ij > 0.5)`. The MATLAB result is an array of logical values 0 and 1. The NumPy result is an array of the boolean values False and True. -| same: | `np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | -| same: | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of a where vector v > 0.5 | -| same: | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of a where column vector v > 0.5 | -| same: | `a[a < 0.5]=0` | `a(a<0.5)=0` | a with elements less than 0.5 zeroed out | -| same: | `a * (a > 0.5)` | `a .* (a>0.5)` | a with elements less than 0.5 zeroed out | -| same: | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | -| same: | `y = x.copy()` | `y=x` | NumPy assigns by reference | -| same: | `y = x[1, :].copy()` | `y=x(2,:)` | NumPy slices are by reference | -| same: | `y = x.flatten()` | `y=x(:)` | turn array into vector (note that this forces a copy). To obtain the same data ordering as in MATLAB, use x.flatten('F'). | -| same: | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector (see note RANGES) | -| same: | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector (see note RANGES) | -| same: | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | -| same: | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional array full of 64-bit floating point zeros | -| same: | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional array full of 64-bit floating point zeros | -| same: | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional array full of 64-bit floating point ones | -| same: | `np.eye(3)` | `eye(3)` | 3x3 identity matrix | -| same: | `np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D array, a | -| same: | `np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | -| same: | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | -| same: | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D arrays: one of x values, the other of y values | -| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | -| same: | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | -| same: | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | -| same: | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | -| same: | `np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | -| same: | `np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | -| same: | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of a (with ndims(a)<=2 for MATLAB, if there are NaN’s, nanmax will ignore these and return largest value) | -| same: | `a.max(0)` | `max(a)` | maximum element of each column of array a | -| same: | `a.max(1)` | `max(a,[],2)` | maximum element of each row of array a | -| same: | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | -| same: | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | -| same: | `logical_and(a,b)` | `a & b` | element-by-element AND operator (NumPy ufunc) See note LOGICOPS | -| same: | `np.logical_or(a,b)` | `a | b` | element-by-element OR operator (NumPy ufunc) | -| same: | `a & b` | `bitand(a,b)` | bitwise AND operator (Python native and NumPy ufunc) | -| same: | `a | b` | `bitor(a,b)` | bitwise OR operator (Python native and NumPy ufunc) | -| same: | `linalg.inv(a)` | `inv(a)` | inverse of square 2D array a | -| same: | `linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D array a | -| same: | `np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D array a | -| same: | `linalg.solve(a, b)` if `a` is square; `linalg.lstsq(a, b)` otherwise | `a\b` | solution of `a x = b` for x | -| same: | Solve `a.T x.T = b.T` instead | `b/a` | solution of x a = b for x | -| same: | `U, S, Vh = linalg.svd(a); V = Vh.T` | `[U,S,V]=svd(a)` | singular value decomposition of a | -| same: | `linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D array | -| same: | `D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | -| same: | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D array, a | -| same: | `Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition -| same: | `P,L,U = linalg.lu(a) where a == P@L@U` | `[L,U,P]=lu(a) where a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | -| same: | `cg` | `conjgrad` | conjugate gradients solver | -| same: | `np.fft.fft(a)` | `fft(a)` | Fourier transform of a | -| same: | `np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of a | -| same: | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D array, a | -| same: | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D array, a | -| same: | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the array a as array b with rows sorted by the first column | -| same: | `x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | -| same: | `signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | -| same: | `np.unique(a)` | `unique(a)` | a vector of unique values in array a | -| same: | `a.squeeze()` | `squeeze(a)` | remove singleton dimensions of array a. Note that MATLAB will always return arrays of 2D or higher while NumPy will return arrays of 0D or higher | +| | `(a > 0.5)` | `(a > 0.5)` | matrix whose `i,jth` element is `(a_ij > 0.5)`. The MATLAB result is an array of logical values 0 and 1. The NumPy result is an array of the boolean values False and True. +| | `np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | +| | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of a where vector v > 0.5 | +| | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of a where column vector v > 0.5 | +| | `a[a < 0.5]=0` | `a(a<0.5)=0` | a with elements less than 0.5 zeroed out | +| | `a * (a > 0.5)` | `a .* (a>0.5)` | a with elements less than 0.5 zeroed out | +| | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | +| | `y = x.copy()` | `y=x` | NumPy assigns by reference | +| | `y = x[1, :].copy()` | `y=x(2,:)` | NumPy slices are by reference | +| | `y = x.flatten()` | `y=x(:)` | turn array into vector (note that this forces a copy). To obtain the same data ordering as in MATLAB, use x.flatten('F'). | +| | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector (see note RANGES) | +| | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector (see note RANGES) | +| | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | +| | `np.eye(3)` | `eye(3)` | 3x3 identity matrix | +| | `np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D array, a | +| | `np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | +| | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | +| | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D arrays: one of x values, the other of y values | +| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | +| | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | +| | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | +| | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | +| | `np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | +| | `np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | +| | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of a (with ndims(a)<=2 for MATLAB, if there are NaN’s, nanmax will ignore these and return largest value) | +| | `a.max(0)` | `max(a)` | maximum element of each column of array a | +| | `a.max(1)` | `max(a,[],2)` | maximum element of each row of array a | +| | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | +| | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | +| | `logical_and(a,b)` | `a & b` | element-by-element AND operator (NumPy ufunc) See note LOGICOPS | +| | `np.logical_or(a,b)` | `a \| b` | element-by-element OR operator (NumPy ufunc) | +| | `a & b` | `bitand(a,b)` | bitwise AND operator (Python native and NumPy ufunc) | +| | `a | b` | `bitor(a,b)` | bitwise OR operator (Python native and NumPy ufunc) | +| | `linalg.inv(a)` | `inv(a)` | inverse of square 2D array a | +| | `linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D array a | +| | `np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D array a | +| | `linalg.solve(a, b)` if `a` is square; `linalg.lstsq(a, b)` otherwise | `a\b` | solution of `a x = b` for x | +| | Solve `a.T x.T = b.T` instead | `b/a` | solution of x a = b for x | +| | `U, S, Vh = linalg.svd(a); V = Vh.T` | `[U,S,V]=svd(a)` | singular value decomposition of a | +| | `linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D array | +| | `D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | +| | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D array, a | +| | `Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition +| | `P,L,U = linalg.lu(a) where a == P@L@U` | `[L,U,P]=lu(a) where a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | +| | `cg` | `conjgrad` | conjugate gradients solver | +| | `np.fft.fft(a)` | `fft(a)` | Fourier transform of a | +| | `np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of a | +| | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D array, a | +| | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D array, a | +| | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the array a as array b with rows sorted by the first column | +| | `x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | +| | `signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | +| | `np.unique(a)` | `unique(a)` | a vector of unique values in array a | +| | `a.squeeze()` | `squeeze(a)` | remove singleton dimensions of array a. Note that MATLAB will always return arrays of 2D or higher while NumPy will return arrays of 0D or higher | From 96ce9eb949c38a530d54303d4ec6b6c9df0dc0bc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 10:39:23 -0700 Subject: [PATCH 074/311] numpy table update and organization --- goal/README.md | 170 ++++++++++++++++++++------------------ gpu/gosl/slrand/README.md | 11 +-- 2 files changed, 91 insertions(+), 90 deletions(-) diff --git a/goal/README.md b/goal/README.md index 1abe2aed44..801944688e 100644 --- a/goal/README.md +++ b/goal/README.md @@ -37,7 +37,7 @@ for _, x := range #[1,2,3]# { } ``` -In general, the math mode syntax in _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. +In general, the math mode syntax in _Goal_ is designed to be as compatible with Python NumPy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in NumPy terms. The rationale and mnemonics for using `$` and `#` are as follows: @@ -221,106 +221,116 @@ TODO: update aboven # Math mode -In general, _Goal_ is designed to be as compatible with Python numpy / scipy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as numpy, without the `np.` prefix, so existing code can easily be converted by just removing that. Corresponding field-like properties of tensors are converted into into appropriate method calls. +In general, _Goal_ is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. -All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in numpy terms. +All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor, and directly supports an indexed view of the outermost row dimension, for optimized sorting and filtering operations. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information (_Goal_ does not support `matrix`). Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) -| Goal | Python | MATLAB | Notes | -| ----- | ------ | ------ | ------ | -| `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of array `a` | -| `len(a)` or `a.len` or `size(a)` or `a.size` | `np.size(a)` or `a.size` | `numel(a)` | number of elements of array `a` | -| `shape(a)` or `a.shape` | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` array | -| `a.shape[n-1]` | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of array a | -| | _Construction_ | | -| `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D array | -| `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks a, b, c, and d | -| | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional array full of 64-bit floating point zeros | -| | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional array full of 64-bit floating point zeros | -| | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional array full of 64-bit floating point ones | +| Goal | Python | MATLAB | Notes | +| -------- | ------ | ------ | ------ | +| `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of tensor `a` | +| `len(a)` or `a.len` or `size(a)` or `a.size` | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | +| `shape(a)` or `a.shape` | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | +| `a.shape[n-1]` | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of tensor `a` | +| `a.reshape([10, 2])` or `a.reshape(10, 2)` or `reshape(a, [10, 2])` | `a.reshape(10, 2)` or `np.reshape(a, 10, 2` | `reshape(a,10,2)` | set the shape of `a`, preserving existing values (and adding zeros if larger, in _Goal_) | +| | `y = x.flatten()` | `y=x(:)` | turn tensor into vector (forces a copy in NumPy). | | | | | +| **Construction** | | | +| `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | +| `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | +| `zeros([3,4]` or `zeros(3, 4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional tensor of float64 zeros | +| `zeros([3, 4, 5])` or `zeros(3, 4, 5)` | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional tensor of float64 zeros | +| `ones([3, 4])` or `ones(3, 4)` | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional tensor of 64-bit floating point ones | +| `rand([3, 4])` or `slrand([3,4], c, fi)` | `rng.random(3, 4)` | `rand(3,4)` | 3x4 2D float64 tensor with uniform random 0..1 elements; `rand` uses current Go `rand` source, while `slrand` uses [gosl](../gpu/gosl/slrand) GPU-safe call with counter `c` and function index `fi` and key = index of element | +| | `np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | +| | `np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | +| | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | +| | `a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | +| | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | +| | | | +| **Ranges and Grids** [numpy](https://numpy.org/doc/stable/user/how-to-partition.html) | | | +| | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector (see note RANGES) | +| | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector (see note RANGES) | +| | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | +| | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | +| | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | +| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | +| | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | +| | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | +| | | | +| **Access** | | | +| (returns a _copy_, use `copy` to assign back) | (returns a _reference_, changes modify original) | (returns a _copy_) | | +| | `y = x.copy()` | `y=x` | NumPy assigns by reference | +| | `y = x[1, :].copy()` | `y=x(2,:)` | NumPy slices are by reference | | | `a[-1]` | `a(end)` | access last element | -| | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D array a | -| | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D array a; unspecified dimensions are equivalent to `:` | +| | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | +| | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | | | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | -| | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D array a | -| | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D array, a. | +| | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | +| | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | | | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | -| (makes a copy) | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of a, starting with the third and going to the twenty-first | -| | `a[::2, :]` | `a(1:2:end,:)` | every other row of a, starting with the first | -| | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | a with rows in reverse order | -| | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | a with copy of the first row appended to the end | +| | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | +| | `a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | +| | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | | | | | -| | `a.transpose() or a.T` | `a.'` | transpose of a | -| | `a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of a | -| | `a @ b` | `a * b` | matrix multiply | -| | `a * b` | `a .* b` | element-wise multiply | -| | `a/b` | `a./b` | element-wise divide | -| `a^3` or: | `a**3` | `a.^3` | element-wise exponentiation | -| | `(a > 0.5)` | `(a > 0.5)` | matrix whose `i,jth` element is `(a_ij > 0.5)`. The MATLAB result is an array of logical values 0 and 1. The NumPy result is an array of the boolean values False and True. +| **Boolean Tensors** | | | +| `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | +| `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | +| `a || b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | +| `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | +| `a | b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | +| | `a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | +| | | | +| **Indexed Sorting and Filtering** | | | +| (indexes only on outer row dim) | index on all elements | | | | | `np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | +| | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of `a` where column vector `v` > 0.5 | | | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of a where vector v > 0.5 | -| | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of a where column vector v > 0.5 | -| | `a[a < 0.5]=0` | `a(a<0.5)=0` | a with elements less than 0.5 zeroed out | -| | `a * (a > 0.5)` | `a .* (a>0.5)` | a with elements less than 0.5 zeroed out | +| | `a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | | | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | -| | `y = x.copy()` | `y=x` | NumPy assigns by reference | -| | `y = x[1, :].copy()` | `y=x(2,:)` | NumPy slices are by reference | -| | `y = x.flatten()` | `y=x(:)` | turn array into vector (note that this forces a copy). To obtain the same data ordering as in MATLAB, use x.flatten('F'). | -| | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector (see note RANGES) | -| | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector (see note RANGES) | -| | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | +| | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D tensor, a | +| | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, a | +| | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the tensor `a` as tensor `b` with rows sorted by the first column | +| | `np.unique(a)` | `unique(a)` | a vector of unique values in tensor `a` | +| | | | +| **Math** | | | +| | `a * b` | `a .* b` | element-wise multiply | +| | `a/b` | `a./b` | element-wise divide | +| `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | +| `cos(a)` | `cos(a)` | `cos(a)` | element-wise function application | +| | | | +| **2D Matrix Linear Algebra** | | | +| (n-dimensional tensors resliced as lists of 2D matricies) | | | +| | `a @ b` | `a * b` | matrix multiply | +| | `a.transpose() or a.T` | `a.'` | transpose of a | +| | `a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | | | `np.eye(3)` | `eye(3)` | 3x3 identity matrix | -| | `np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D array, a | +| | `np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D tensor, `a` | | | `np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | -| | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | -| | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D arrays: one of x values, the other of y values | -| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | -| | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | -| | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | -| | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | -| | `np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | -| | `np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | -| | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of a (with ndims(a)<=2 for MATLAB, if there are NaN’s, nanmax will ignore these and return largest value) | -| | `a.max(0)` | `max(a)` | maximum element of each column of array a | -| | `a.max(1)` | `max(a,[],2)` | maximum element of each row of array a | -| | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | -| | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | -| | `logical_and(a,b)` | `a & b` | element-by-element AND operator (NumPy ufunc) See note LOGICOPS | -| | `np.logical_or(a,b)` | `a \| b` | element-by-element OR operator (NumPy ufunc) | -| | `a & b` | `bitand(a,b)` | bitwise AND operator (Python native and NumPy ufunc) | -| | `a | b` | `bitor(a,b)` | bitwise OR operator (Python native and NumPy ufunc) | -| | `linalg.inv(a)` | `inv(a)` | inverse of square 2D array a | -| | `linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D array a | -| | `np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D array a | +| | `linalg.inv(a)` | `inv(a)` | inverse of square 2D tensor a | +| | `linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D tensor a | +| | `np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D tensor a | | | `linalg.solve(a, b)` if `a` is square; `linalg.lstsq(a, b)` otherwise | `a\b` | solution of `a x = b` for x | | | Solve `a.T x.T = b.T` instead | `b/a` | solution of x a = b for x | | | `U, S, Vh = linalg.svd(a); V = Vh.T` | `[U,S,V]=svd(a)` | singular value decomposition of a | -| | `linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D array | +| | `linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D tensor | | | `D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | -| | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D array, a | +| | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D tensor, a | | | `Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition | | `P,L,U = linalg.lu(a) where a == P@L@U` | `[L,U,P]=lu(a) where a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | +| | | | +| **Statistics** | | | +| `a.max()` or `max(a)` or `stats.Max(a)` | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of `a`, _Goal_ always ignores `NaN` as missing data | +| `stats.Max(a[) | `a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | +| | `a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | +| | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | +| `stats.L2Norm(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | | | `cg` | `conjgrad` | conjugate gradients solver | -| | `np.fft.fft(a)` | `fft(a)` | Fourier transform of a | -| | `np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of a | -| | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D array, a | -| | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D array, a | -| | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the array a as array b with rows sorted by the first column | +| | | | +| **Misc Functions** | | | +| | `np.fft.fft(a)` | `fft(a)` | Fourier transform of `a` | +| | `np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of `a` | | | `x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | | | `signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | -| | `np.unique(a)` | `unique(a)` | a vector of unique values in array a | -| | `a.squeeze()` | `squeeze(a)` | remove singleton dimensions of array a. Note that MATLAB will always return arrays of 2D or higher while NumPy will return arrays of 0D or higher | - - - -rng(42,'twister') -rand(3,4) -from numpy.random import default_rng -rng = default_rng(42) -rng.random(3, 4) -or older version: random.rand((3, 4)) - -generate a random 3x4 array with default random number generator and seed = 42 diff --git a/gpu/gosl/slrand/README.md b/gpu/gosl/slrand/README.md index 15e6ca02ff..0268ef99d9 100644 --- a/gpu/gosl/slrand/README.md +++ b/gpu/gosl/slrand/README.md @@ -16,9 +16,7 @@ The key advantage of this algorithm is its *stateless* nature, where the result ``` where the WGSL `vec2` type is 2 `uint32` 32-bit unsigned integers. For GPU usage, the `key` is always set to the unique element being processed (e.g., the index of the data structure being updated), ensuring that different numbers are generated for each such element, and the `counter` should be configured as a shared global value that is incremented after each iteration of computation. - - - For example, if 4 RNG calls happen within a given set of GPU code, each thread starts with the same starting `counter` value, which is passed around as a local `vec2` variable and incremented locally for each RNG. Then, after all threads have been performed, the shared starting `counter` is incremented using `CounterAdd` by 4. +For example, if 4 RNG calls happen within a given set of GPU code, each thread starts with the same starting `counter` value, which is passed around as a local `vec2` variable and incremented locally for each RNG. Then, after all threads have been performed, the shared starting `counter` is incremented using `CounterAdd` by 4. The `Float` and `Uint32` etc wrapper functions around Philox2x32 will automatically increment the counter var passed to it, using the `CounterIncr()` method that manages the two 32 bit numbers as if they are a full 64 bit uint. @@ -30,11 +28,4 @@ See the [axon](https://github.com/emer/gosl/v2/tree/main/examples/axon) and [ran Critically, these examples show that the CPU and GPU code produce identical random number sequences, which is otherwise quite difficult to achieve without this specific form of RNG. -# Implementational details - -Unfortunately, vulkan `glslang` does not support 64 bit integers, even though the shader language model has somehow been updated to support them: https://github.com/KhronosGroup/glslang/issues/2965 -- https://github.com/microsoft/DirectXShaderCompiler/issues/2067. This would also greatly speed up the impl: https://github.com/microsoft/DirectXShaderCompiler/issues/2821. - -The result is that we have to use the slower version of the MulHiLo algorithm using only 32 bit uints. - - From dd209b1063d9e2e6793e9fc875c7ce902459d6e3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 10:41:00 -0700 Subject: [PATCH 075/311] table fixes --- goal/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/goal/README.md b/goal/README.md index 801944688e..91ecc03948 100644 --- a/goal/README.md +++ b/goal/README.md @@ -277,9 +277,9 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | **Boolean Tensors** | | | | `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | -| `a || b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | +| `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | | `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | -| `a | b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | +| `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | | | `a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | | | | | | **Indexed Sorting and Filtering** | | | From 5bfe896795564d655faa1e6c363c7e741437a46a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 10:47:08 -0700 Subject: [PATCH 076/311] table fixes --- goal/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/goal/README.md b/goal/README.md index 91ecc03948..d946d2a704 100644 --- a/goal/README.md +++ b/goal/README.md @@ -227,8 +227,8 @@ All elements of a _Goal_ math expression are [tensors](../tensor), specifically Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) -| Goal | Python | MATLAB | Notes | -| -------- | ------ | ------ | ------ | +| Goal | Python | MATLAB | Notes | +| -------------------------- | ------ | ------ | ------ | | `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of tensor `a` | | `len(a)` or `a.len` or `size(a)` or `a.size` | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | | `shape(a)` or `a.shape` | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | @@ -318,11 +318,11 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | | | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D tensor, a | | | `Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition -| | `P,L,U = linalg.lu(a) where a == P@L@U` | `[L,U,P]=lu(a) where a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | +| | `P,L,U = linalg.lu(a)` where `a == P@L@U` | `[L,U,P]=lu(a)` where `a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | | | | | | **Statistics** | | | | `a.max()` or `max(a)` or `stats.Max(a)` | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of `a`, _Goal_ always ignores `NaN` as missing data | -| `stats.Max(a[) | `a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | +| | `a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | | | `a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | | | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | | `stats.L2Norm(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | From 6262c00945ff1e6a7fb5077aa547cb42986428da Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 10:52:33 -0700 Subject: [PATCH 077/311] table fixes --- goal/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/goal/README.md b/goal/README.md index d946d2a704..7fcd3fd5c4 100644 --- a/goal/README.md +++ b/goal/README.md @@ -250,8 +250,8 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | | | | | | **Ranges and Grids** [numpy](https://numpy.org/doc/stable/user/how-to-partition.html) | | | -| | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector (see note RANGES) | -| | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector (see note RANGES) | +| | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector | +| | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector | | | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | | | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | | | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | @@ -275,19 +275,20 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | | | | | | **Boolean Tensors** | | | +| (bool tensor of same shape can filter access to other tensor) | | | | `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | | `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | | `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | | `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | | | `a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | +| | `a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | | | | | -| **Indexed Sorting and Filtering** | | | +| **Indexed Filtering and Sorting** | | | | (indexes only on outer row dim) | index on all elements | | | | | `np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | | | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of `a` where column vector `v` > 0.5 | -| | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of a where vector v > 0.5 | -| | `a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | +| | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of `a` where vector `v` > 0.5 | | | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | | | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D tensor, a | | | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, a | From 5b9c2a5f32fef13176fdeac7e848bb26d69a77ba Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 11:43:51 -0700 Subject: [PATCH 078/311] fixes from kai review --- base/keylist/keylist.go | 83 ++++++++++++++++--------------- base/keylist/keylist_test.go | 6 +-- base/metadata/metadata.go | 8 +-- base/ordmap/ordmap.go | 1 + core/textfield.go | 2 +- filetree/node.go | 3 +- math32/math.go | 25 +--------- tensor/base.go | 2 +- tensor/datafs/copy.go | 2 +- tensor/datafs/dir.go | 18 +++---- tensor/datafs/fs.go | 4 +- tensor/examples/datafs-sim/sim.go | 6 +-- tensor/stats/metric/funcs.go | 2 +- tensor/stats/stats/describe.go | 2 +- tensor/stats/stats/group.go | 6 +-- tensor/table/columns.go | 7 +-- tensor/table/table.go | 6 +-- tensor/tmath/norm.go | 2 +- 18 files changed, 83 insertions(+), 102 deletions(-) diff --git a/base/keylist/keylist.go b/base/keylist/keylist.go index 4f0b0b1a0b..e9703bb234 100644 --- a/base/keylist/keylist.go +++ b/base/keylist/keylist.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -package keylist implements an ordered list (slice) of items, +Package keylist implements an ordered list (slice) of items, with a map from a key (e.g., names) to indexes, to support fast lookup by name. This is a different implementation of the [ordmap] package, @@ -18,7 +18,7 @@ import ( "slices" ) -// TODO: probably want to consolidate ordmap and keylist +// TODO: probably want to consolidate ordmap and keylist: https://github.com/cogentcore/core/issues/1224 // List implements an ordered list (slice) of Values, // with a map from a key (e.g., names) to indexes, @@ -27,27 +27,28 @@ type List[K comparable, V any] struct { //types:add // List is the ordered slice of items. Values []V + // Keys is the ordered list of keys, in same order as [List.Values] Keys []K // indexes is the key-to-index mapping. indexes map[K]int } -// New returns a new Key List. Zero value +// New returns a new [List]. The zero value // is usable without initialization, so this is // just a simple standard convenience method. func New[K comparable, V any]() *List[K, V] { return &List[K, V]{} } -func (kl *List[K, V]) newIndexes() { +func (kl *List[K, V]) makeIndexes() { kl.indexes = make(map[K]int) } // initIndexes ensures that the index map exists. func (kl *List[K, V]) initIndexes() { if kl.indexes == nil { - kl.newIndexes() + kl.makeIndexes() } } @@ -55,59 +56,61 @@ func (kl *List[K, V]) initIndexes() { func (kl *List[K, V]) Reset() { kl.Values = nil kl.Keys = nil - kl.newIndexes() + kl.makeIndexes() } -// Add adds an item to the list with given key. -// An error is returned if the key is already on the list. -// See [AddReplace] for a method that automatically replaces. -func (kl *List[K, V]) Add(key K, val V) error { +// Set sets given key to given value, adding to the end of the list +// if not already present, and otherwise replacing with this new value. +// This is the same semantics as a Go map. +// See [List.Add] for version that only adds and does not replace. +func (kl *List[K, V]) Set(key K, val V) { kl.initIndexes() - if _, ok := kl.indexes[key]; ok { - return fmt.Errorf("keylist.Add: key %v is already on the list", key) + if idx, ok := kl.indexes[key]; ok { + kl.Values[idx] = val + kl.Keys[idx] = key + return } kl.indexes[key] = len(kl.Values) kl.Values = append(kl.Values, val) kl.Keys = append(kl.Keys, key) - return nil } -// AddReplace adds an item to the list with given key, -// replacing any existing item with the same key. -func (kl *List[K, V]) AddReplace(key K, val V) { +// Add adds an item to the list with given key, +// An error is returned if the key is already on the list. +// See [List.Set] for a method that automatically replaces. +func (kl *List[K, V]) Add(key K, val V) error { kl.initIndexes() - if idx, ok := kl.indexes[key]; ok { - kl.Values[idx] = val - kl.Keys[idx] = key - return + if _, ok := kl.indexes[key]; ok { + return fmt.Errorf("keylist.Add: key %v is already on the list", key) } kl.indexes[key] = len(kl.Values) kl.Values = append(kl.Values, val) kl.Keys = append(kl.Keys, key) + return nil } // Insert inserts the given value with the given key at the given index. // This is relatively slow because it needs regenerate the keys list. -// It returns an error if the key already exists because -// the behavior is undefined in that situation. -func (kl *List[K, V]) Insert(idx int, key K, val V) error { +// It panics if the key already exists because the behavior is undefined +// in that situation. +func (kl *List[K, V]) Insert(idx int, key K, val V) { if _, has := kl.indexes[key]; has { - return fmt.Errorf("keylist.Add: key %v is already on the list", key) + panic("keylist.Add: key is already on the list") } kl.Keys = slices.Insert(kl.Keys, idx, key) kl.Values = slices.Insert(kl.Values, idx, val) - kl.newIndexes() + kl.makeIndexes() for i, k := range kl.Keys { kl.indexes[k] = i } - return nil } -// ValueByKey returns the value corresponding to the given key, -// with a zero value returned for a missing key. See [List.ValueByKeyTry] +// At returns the value corresponding to the given key, +// with a zero value returned for a missing key. See [List.AtTry] // for one that returns a bool for missing keys. -func (kl *List[K, V]) ValueByKey(key K) V { +// For index-based access, use [List.Values] or [List.Keys] slices directly. +func (kl *List[K, V]) At(key K) V { idx, ok := kl.indexes[key] if ok { return kl.Values[idx] @@ -116,10 +119,10 @@ func (kl *List[K, V]) ValueByKey(key K) V { return zv } -// ValueByKeyTry returns the value corresponding to the given key, +// AtTry returns the value corresponding to the given key, // with false returned for a missing key, in case the zero value // is not diagnostic. -func (kl *List[K, V]) ValueByKeyTry(key K) (V, bool) { +func (kl *List[K, V]) AtTry(key K) (V, bool) { idx, ok := kl.indexes[key] if ok { return kl.Values[idx], true @@ -153,43 +156,43 @@ func (kl *List[K, V]) Len() int { return len(kl.Values) } -// DeleteIndex deletes item(s) within the index range [i:j]. +// DeleteByIndex deletes item(s) within the index range [i:j]. // This is relatively slow because it needs to regenerate the // index map. -func (kl *List[K, V]) DeleteIndex(i, j int) { +func (kl *List[K, V]) DeleteByIndex(i, j int) { ndel := j - i if ndel <= 0 { panic("index range is <= 0") } kl.Keys = slices.Delete(kl.Keys, i, j) kl.Values = slices.Delete(kl.Values, i, j) - kl.newIndexes() + kl.makeIndexes() for i, k := range kl.Keys { kl.indexes[k] = i } } -// DeleteKey deletes the item with the given key, +// DeleteByKey deletes the item with the given key, // returning false if it does not find it. // This is relatively slow because it needs to regenerate the // index map. -func (kl *List[K, V]) DeleteKey(key K) bool { +func (kl *List[K, V]) DeleteByKey(key K) bool { idx, ok := kl.indexes[key] if !ok { return false } - kl.DeleteIndex(idx, idx+1) + kl.DeleteByIndex(idx, idx+1) return true } -// Copy copies all of the entries from the given keyed list +// Copy copies all of the entries from the given key list // into this list. It keeps existing entries in this // list unless they also exist in the given list, in which case -// they are overwritten. Use [Reset] first to get an exact copy. +// they are overwritten. Use [List.Reset] first to get an exact copy. func (kl *List[K, V]) Copy(from *List[K, V]) { for i, v := range from.Values { - kl.AddReplace(kl.Keys[i], v) + kl.Set(kl.Keys[i], v) } } diff --git a/base/keylist/keylist_test.go b/base/keylist/keylist_test.go index e880a899e7..b88876ecf0 100644 --- a/base/keylist/keylist_test.go +++ b/base/keylist/keylist_test.go @@ -16,14 +16,14 @@ func TestKeyList(t *testing.T) { kl.Add("key1", 1) kl.Add("key2", 2) - assert.Equal(t, 1, kl.ValueByKey("key1")) + assert.Equal(t, 1, kl.At("key1")) assert.Equal(t, 2, kl.IndexByKey("key2")) assert.Equal(t, 1, kl.Values[1]) assert.Equal(t, 3, kl.Len()) - kl.DeleteIndex(1, 2) + kl.DeleteByIndex(1, 2) assert.Equal(t, 2, kl.Values[1]) assert.Equal(t, 1, kl.IndexByKey("key2")) @@ -35,5 +35,5 @@ func TestKeyList(t *testing.T) { // nm := Make([]KeyValue[string, int]{{"one", 1}, {"two", 2}, {"three", 3}}) // assert.Equal(t, 3, nm.Values[2]) // assert.Equal(t, 2, nm.Values[1]) - // assert.Equal(t, 3, nm.ValueByKey("three")) + // assert.Equal(t, 3, nm.At("three")) } diff --git a/base/metadata/metadata.go b/base/metadata/metadata.go index 85c3bb7f03..4517452659 100644 --- a/base/metadata/metadata.go +++ b/base/metadata/metadata.go @@ -70,8 +70,8 @@ func (md *Data) SetName(name string) { md.Set("Name", name) } -// GetName returns the "Name" standard key value (empty if not set). -func (md *Data) GetName() string { +// Name returns the "Name" standard key value (empty if not set). +func (md *Data) Name() string { return errors.Ignore1(Get[string](*md, "Name")) } @@ -80,7 +80,7 @@ func (md *Data) SetDoc(doc string) { md.Set("Doc", doc) } -// GetDoc returns the "Doc" standard key value (empty if not set). -func (md *Data) GetDoc() string { +// Doc returns the "Doc" standard key value (empty if not set). +func (md *Data) Doc() string { return errors.Ignore1(Get[string](*md, "Doc")) } diff --git a/base/ordmap/ordmap.go b/base/ordmap/ordmap.go index f459ad88af..9c66951ed4 100644 --- a/base/ordmap/ordmap.go +++ b/base/ordmap/ordmap.go @@ -35,6 +35,7 @@ type KeyValue[K comparable, V any] struct { // and the fast key lookup of a map. A map stores an index // into a slice that has the value and key associated with the value. type Map[K comparable, V any] struct { + // Order is an ordered list of values and associated keys, in the order added. Order []KeyValue[K, V] diff --git a/core/textfield.go b/core/textfield.go index 3bc4ae54d8..deb2a406c2 100644 --- a/core/textfield.go +++ b/core/textfield.go @@ -1456,7 +1456,7 @@ func (tf *TextField) autoScroll() { availSz := sz.Actual.Content.Sub(icsz) tf.configTextSize(availSz) n := len(tf.editText) - tf.cursorPos = math32.ClampInt(tf.cursorPos, 0, n) + tf.cursorPos = math32.Clamp(tf.cursorPos, 0, n) if tf.hasWordWrap() { // does not scroll tf.startPos = 0 diff --git a/filetree/node.go b/filetree/node.go index 0d8d2a4a53..4cd44609ae 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -335,9 +335,10 @@ func (fn *Node) dirFileList() []fs.FileInfo { return files } +// sortByModTime sorts by _reverse_ mod time (newest first) func sortByModTime(files []fs.FileInfo) { slices.SortFunc(files, func(a, b fs.FileInfo) int { - return -a.ModTime().Compare(b.ModTime()) + return b.ModTime().Compare(a.ModTime()) // reverse order }) } diff --git a/math32/math.go b/math32/math.go index efce54652a..b67e02150e 100644 --- a/math32/math.go +++ b/math32/math.go @@ -15,6 +15,7 @@ package math32 //go:generate core generate import ( + "cmp" "math" "strconv" @@ -783,29 +784,7 @@ func Yn(n int, x float32) float32 { // Special additions to math. functions // Clamp clamps x to the provided closed interval [a, b] -func Clamp(x, a, b float32) float32 { - if x < a { - return a - } - if x > b { - return b - } - return x -} - -// ClampInt clamps x to the provided closed interval [a, b] -func ClampInt(x, a, b int) int { - if x < a { - return a - } - if x > b { - return b - } - return x -} - -// Clamp64 clamps x to the provided closed interval [a, b] -func Clamp64(x, a, b float64) float64 { +func Clamp[T cmp.Ordered](x, a, b T) T { if x < a { return a } diff --git a/tensor/base.go b/tensor/base.go index 3ca8eb6b38..7b7e69c36c 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -146,7 +146,7 @@ func (tsr *Base[T]) StringRowCell(row, cell int) string { // Label satisfies the core.Labeler interface for a summary description of the tensor. func (tsr *Base[T]) Label() string { - nm := tsr.Meta.GetName() + nm := tsr.Meta.Name() if nm != "" { nm += " " + tsr.shape.String() } else { diff --git a/tensor/datafs/copy.go b/tensor/datafs/copy.go index b043916ef2..9da7e0404c 100644 --- a/tensor/datafs/copy.go +++ b/tensor/datafs/copy.go @@ -73,7 +73,7 @@ func (d *Data) Copy(overwrite bool, to string, from ...string) error { continue } if targf == "" { - if trg, ok := targd.Dir.ValueByKeyTry(frd.name); ok { // target exists + if trg, ok := targd.Dir.AtTry(frd.name); ok { // target exists switch { case trg.IsDir() && frd.IsDir(): // todo: copy all items from frd into trg diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 0642a23d88..a16705572e 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -45,7 +45,7 @@ func NewDir(name string, parent ...*Data) (*Data, error) { // This is for fast access and direct usage of known // items, and it will panic if this data is not a directory. func (d *Data) Item(name string) *Data { - return d.Dir.ValueByKey(name) + return d.Dir.At(name) } // Value returns the [tensor.Indexed] Value for given item @@ -53,7 +53,7 @@ func (d *Data) Item(name string) *Data { // found, and will return nil if it is not a Value // (i.e., it is a directory). func (d *Data) Value(name string) *tensor.Indexed { - return d.Dir.ValueByKey(name).Data + return d.Dir.At(name).Data } // Items returns data items in given directory by name. @@ -65,7 +65,7 @@ func (d *Data) Items(names ...string) ([]*Data, error) { var errs []error var its []*Data for _, nm := range names { - dt := d.Dir.ValueByKey(nm) + dt := d.Dir.At(nm) if dt != nil { its = append(its, dt) } else { @@ -85,7 +85,7 @@ func (d *Data) Values(names ...string) ([]*tensor.Indexed, error) { var errs []error var its []*tensor.Indexed for _, nm := range names { - it := d.Dir.ValueByKey(nm) + it := d.Dir.At(nm) if it != nil && it.Data != nil { its = append(its, it.Data) } else { @@ -147,7 +147,7 @@ func (d *Data) ItemsAlphaFunc(fun func(item *Data) bool) []*Data { names := d.DirNamesAlpha() var its []*Data for _, nm := range names { - it := d.Dir.ValueByKey(nm) + it := d.Dir.At(nm) if fun != nil && !fun(it) { continue } @@ -218,7 +218,7 @@ func (d *Data) FlatValuesAlphaFunc(fun func(item *Data) bool) []*tensor.Indexed names := d.DirNamesAlpha() var its []*tensor.Indexed for _, nm := range names { - it := d.Dir.ValueByKey(nm) + it := d.Dir.At(nm) if fun != nil && !fun(it) { continue } @@ -260,7 +260,7 @@ func (d *Data) ItemAtPath(name string) (*Data, error) { if err != nil { return nil, err } - itm, ok := sd.Dir.ValueByKeyTry(file) + itm, ok := sd.Dir.AtTry(file) if !ok { if dir == "" && (file == d.name || file == ".") { return d, nil @@ -301,7 +301,7 @@ func (d *Data) DirNamesAlpha() []string { func (d *Data) DirNamesByTime() []string { names := slices.Clone(d.Dir.Keys) slices.SortFunc(names, func(a, b string) int { - return d.Dir.ValueByKey(a).ModTime().Compare(d.Dir.ValueByKey(b).ModTime()) + return d.Dir.At(a).ModTime().Compare(d.Dir.At(b).ModTime()) }) return names } @@ -349,7 +349,7 @@ func (d *Data) RecycleDir(name string) (*Data, error) { if err := d.mustDir("RecycleDir", name); err != nil { return nil, err } - if cd := d.Dir.ValueByKey(name); cd != nil { + if cd := d.Dir.At(name); cd != nil { return cd, nil } return NewDir(name, d) diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index b4b1b75567..5e0d937e4f 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -60,7 +60,7 @@ func (d *Data) Sub(dir string) (fs.FS, error) { return cur, nil } cd = rest - sd, ok := cur.Dir.ValueByKeyTry(root) + sd, ok := cur.Dir.AtTry(root) if !ok { return nil, &fs.PathError{Op: "Sub", Path: dir, Err: errors.New("directory not found")} } @@ -81,7 +81,7 @@ func (d *Data) ReadDir(dir string) ([]fs.DirEntry, error) { names := sd.DirNamesAlpha() ents := make([]fs.DirEntry, len(names)) for i, nm := range names { - ents[i] = sd.Dir.ValueByKey(nm) + ents[i] = sd.Dir.At(nm) } return ents, nil } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index a0064118e5..7a6e74f60b 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -65,7 +65,7 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { ntrial := ss.Config.Item("NTrial").AsInt() sitems := ss.Stats.ValuesFunc(nil) for _, st := range sitems { - nm := st.Tensor.Metadata().GetName() + nm := st.Tensor.Metadata().Name() lt := logd.NewOfType(nm, st.Tensor.DataType(), ntrial) lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source tensor.SetCalcFunc(lt.Tensor, func() error { @@ -80,7 +80,7 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { } alllogd, _ := dir.Mkdir("AllTrials") for _, st := range sitems { - nm := st.Tensor.Metadata().GetName() + nm := st.Tensor.Metadata().Name() // allocate full size lt := alllogd.NewOfType(nm, st.Tensor.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) lt.Tensor.SetShape(0) // then truncate to 0 @@ -109,7 +109,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a if st.Tensor.IsString() { continue } - nm := st.Tensor.Metadata().GetName() + nm := st.Tensor.Metadata().Name() src := from.Value(nm) if st.Tensor.DataType() >= reflect.Float32 { // todo: pct correct etc diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 4043c46865..00d651d39e 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -176,7 +176,7 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { func CrossEntropyFunc(a, b, out *tensor.Indexed) { stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { - b = math32.Clamp64(b, 0.000001, 0.999999) + b = math32.Clamp(b, 0.000001, 0.999999) if a >= 1.0 { agg += -math.Log(b) } else if a <= 0.0 { diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go index 13a5bfdae0..3776d5c823 100644 --- a/tensor/stats/stats/describe.go +++ b/tensor/stats/stats/describe.go @@ -33,7 +33,7 @@ func Describe(dir *datafs.Data, tsrs ...*tensor.Indexed) { if nr == 0 { continue } - nm := tsr.Tensor.Metadata().GetName() + nm := tsr.Tensor.Metadata().Name() if nm == "" { nm = strconv.Itoa(i) } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 50c9299ca0..066b09efed 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -49,7 +49,7 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { if nr == 0 { continue } - nm := tsr.Tensor.Metadata().GetName() + nm := tsr.Tensor.Metadata().Name() if nm == "" { nm = strconv.Itoa(i) } @@ -157,11 +157,11 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { if gv == nil { gtsr := datafs.NewValue[string](sgd, gpnm, nv) for i, v := range vals { - gtsr.SetStringRow(v.Tensor.Metadata().GetName(), i) + gtsr.SetStringRow(v.Tensor.Metadata().Name(), i) } } for _, tsr := range tsrs { - vd, _ := sgd.RecycleDir(tsr.Tensor.Metadata().GetName()) + vd, _ := sgd.RecycleDir(tsr.Tensor.Metadata().Name()) sv := datafs.NewValue[float64](vd, stnm, nv) for i, v := range vals { idx := v.Tensor.(*tensor.Int).Values diff --git a/tensor/table/columns.go b/tensor/table/columns.go index 4881831051..6ea345c60a 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -53,10 +53,7 @@ func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows. func (cl *Columns) InsertColumn(idx int, name string, tsr tensor.Tensor) error { - err := cl.Insert(idx, name, tsr) - if err != nil { - return err - } + cl.Insert(idx, name, tsr) tsr.SetNumRows(cl.Rows) return nil } @@ -74,7 +71,7 @@ func (cl *Columns) Clone() *Columns { // AppendRows appends shared columns in both tables with input table rows. func (cl *Columns) AppendRows(cl2 *Columns) { for i, nm := range cl.Keys { - c2 := cl2.ValueByKey(nm) + c2 := cl2.At(nm) if c2 == nil { continue } diff --git a/tensor/table/table.go b/tensor/table/table.go index e5adfa611d..1f03351120 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -82,7 +82,7 @@ func (dt *Table) NumColumns() int { return dt.Columns.Len() } // provide the shared table-wide Indexes. // Returns nil if not found. func (dt *Table) Column(name string) *tensor.Indexed { - cl := dt.Columns.ValueByKey(name) + cl := dt.Columns.At(name) if cl == nil { return nil } @@ -228,12 +228,12 @@ func (dt *Table) AddIntColumn(name string, cellSizes ...int) *tensor.Int { // DeleteColumnName deletes column of given name. // returns false if not found. func (dt *Table) DeleteColumnName(name string) bool { - return dt.Columns.DeleteKey(name) + return dt.Columns.DeleteByKey(name) } // DeleteColumnIndex deletes column within the index range [i:j]. func (dt *Table) DeleteColumnByIndex(i, j int) { - dt.Columns.DeleteIndex(i, j) + dt.Columns.DeleteByIndex(i, j) } // DeleteAll deletes all columns, does full reset. diff --git a/tensor/tmath/norm.go b/tensor/tmath/norm.go index 87c4acd98a..a2e65afd9c 100644 --- a/tensor/tmath/norm.go +++ b/tensor/tmath/norm.go @@ -38,7 +38,7 @@ func Clamp(in, minv, maxv, out *tensor.Indexed) { mx := maxv.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math32.Clamp64(tsr[0].Float1D(i), mn, mx), i) + tsr[1].SetFloat1D(math32.Clamp(tsr[0].Float1D(i), mn, mx), i) }, in, out) } From a2df8aaac81053e9e308fd18b465313bc7235d6d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 11:51:45 -0700 Subject: [PATCH 079/311] update yaegicore --- yaegicore/symbols/cogentcore_org-core-math32.go | 3 --- yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go | 1 + yaegicore/symbols/cogentcore_org-core-tensor.go | 3 +++ 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/yaegicore/symbols/cogentcore_org-core-math32.go b/yaegicore/symbols/cogentcore_org-core-math32.go index 3d9fabd2c2..757eaa5d39 100644 --- a/yaegicore/symbols/cogentcore_org-core-math32.go +++ b/yaegicore/symbols/cogentcore_org-core-math32.go @@ -29,9 +29,6 @@ func init() { "BarycoordFromPoint": reflect.ValueOf(math32.BarycoordFromPoint), "Cbrt": reflect.ValueOf(math32.Cbrt), "Ceil": reflect.ValueOf(math32.Ceil), - "Clamp": reflect.ValueOf(math32.Clamp), - "Clamp64": reflect.ValueOf(math32.Clamp64), - "ClampInt": reflect.ValueOf(math32.ClampInt), "ContainsPoint": reflect.ValueOf(math32.ContainsPoint), "CopyFloat32s": reflect.ValueOf(math32.CopyFloat32s), "CopyFloat64s": reflect.ValueOf(math32.CopyFloat64s), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index a010efe98c..8df9b6bfd9 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -17,6 +17,7 @@ func init() { "DescribeTable": reflect.ValueOf(stats.DescribeTable), "DescribeTableAll": reflect.ValueOf(stats.DescribeTableAll), "DescriptiveStats": reflect.ValueOf(&stats.DescriptiveStats).Elem(), + "GroupAll": reflect.ValueOf(stats.GroupAll), "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), "GroupStats": reflect.ValueOf(stats.GroupStats), "Groups": reflect.ValueOf(stats.Groups), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index fcb1d8e00d..5dee5308ac 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -41,6 +41,7 @@ func init() { "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), "Include": reflect.ValueOf(tensor.Include), "IntToBool": reflect.ValueOf(tensor.IntToBool), + "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), "NFirstLen": reflect.ValueOf(tensor.NFirstLen), "NFirstRows": reflect.ValueOf(tensor.NFirstRows), "NMinLen": reflect.ValueOf(tensor.NMinLen), @@ -82,6 +83,8 @@ func init() { "SliceSet": reflect.ValueOf(tensor.SliceSet), "SliceSize": reflect.ValueOf(tensor.SliceSize), "Space": reflect.ValueOf(tensor.Space), + "Sprint": reflect.ValueOf(tensor.Sprint), + "SprintIndexed": reflect.ValueOf(tensor.SprintIndexed), "Stable": reflect.ValueOf(tensor.Stable), "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), From 39ab3401259492580e466a69d2e482134a1602db Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 19 Sep 2024 17:21:41 -0700 Subject: [PATCH 080/311] tensor Indexed implements tensor api with updated interface to support views as tensors. key step to allow other views all implementing tensor api. --- tensor/README.md | 24 +- tensor/base.go | 62 ++- tensor/bits.go | 70 ++- tensor/convert.go | 127 +++++ tensor/funcs.go | 60 +-- tensor/funcs_test.go | 4 +- tensor/indexed.go | 504 +++++++++++------- tensor/matrix/README.md | 5 + tensor/matrix/matrix.go | 25 + tensor/number.go | 81 ++- tensor/projection2d.go | 12 +- tensor/shape.go | 96 +--- tensor/slice.go | 24 +- tensor/stats/metric/vec.go | 16 +- tensor/stats/stats/vec.go | 18 +- tensor/string.go | 55 +- tensor/table/io.go | 6 +- tensor/table/io_test.go | 30 +- tensor/table/table_test.go | 14 +- tensor/tensor.go | 119 +++-- tensor/tensor_test.go | 42 +- tensor/tensorcore/table.go | 10 +- .../symbols/cogentcore_org-core-tensor.go | 3 +- 23 files changed, 871 insertions(+), 536 deletions(-) create mode 100644 tensor/convert.go create mode 100644 tensor/matrix/README.md create mode 100644 tensor/matrix/matrix.go diff --git a/tensor/README.md b/tensor/README.md index dddb2f0a92..ce14836908 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -2,17 +2,25 @@ Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used `numpy` and `pandas` libraries in python, and the commercial MATLAB framework. -The `tensor.Indexed` type provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix, and beyond, because it can efficiently represent any kind of element with sufficient flexibility to enable a huge range of computations to be elegantly expressed. The indexes provide a specific view onto the underlying [Tensor] data, applying to the outermost _row_ dimension (with default row-major indexing). For example, sorting and filtering a tensor only requires updating the indexes while doing nothing to the Tensor itself. +The [Goal](../goal) augmented version of the _Go_ language directly supports numpy-like operations on tensors. A `Tensor` is comparable to the numpy `array` type, and it provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix and beyond. All functions take and return `Tensor` arguments. -The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. +The `Tensor` interface is implemented at the basic level with n-dimensional indexing into flat Go slices of any numeric data type (by `Number`), along with `String`, and `Bool` (which uses [bitslice](bitslice) for maximum efficiency). The `Shape` type provides all the n-dimensional indexing with arbitrary strides to allow any ordering, although _row major_ is the default and other orders have to be manually imposed. + +In addition, there are three important "view" implementations of `Tensor` that wrap another Tensor to provide more flexible and efficient access to the data, consistent with the numpy functionality: -The [goal](../goal) _Cogent [Scripting, Science, Statistics, Shell...] Language_ uses `tensor.Indexed` data types exclusively to allow simple intuitive math expressions to be transpiled into corresponding Go code, providing an efficient, elegant, yet type-safe and computationally powerful framework for data processing of all sorts. All of the standard math, statistics, etc functionality is available using the [tmath](tmath), [stats](stats), and other associated packages described below. +* `Indexed` provides an index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension. Organizing data systematically along the row dimension provides a natural and efficient constraint that is leveraged throughout the `tensor` package: it eliminates many sources of ambiguity about how to process higher-dimensional data, and any given n-dimensional structure can be reshaped to fit in this row-based format. -Use the `[Set]FloatRow[Cell]` methods wherever possible, for the most efficient and natural indirection through the `tensor.Indexed` indexes. The 1D methods on underlying tensor data do not indirect through the indexes. +* `Sliced` is a more general version of `Indexed` that provides a sub-sliced view into the wrapped `Tensor`, using an indexed list for access along each dimension. + +* `Bitmask` provides a `Bool` masked view onto each element in the wrapped `Tensor` (the two maintain the same shape), such that any cell with a `false` state returns a `NaN` (missing data) and `Set` functions are no-ops. + +The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. + +The `[Set]FloatRow[Cell]` methods are used wherever possible, for the most efficient and natural indirection for row-major organized data. See [Standard shapes](#standard-shapes) for more info. The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. It is completely up to the compute function how to interpret the index. There is a Threaded version of this for parallelizable functions, and a GPU version. -All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensors for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. +All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensor for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. * [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. @@ -36,11 +44,11 @@ All tensor package functions are registered using a global name-to-function map - [metric](metric) computes similarity / distance metrics for comparing two tensors, and associated distance / similarity matrix functions, including PCA and SVD analysis functions that operate on a covariance matrix. - [stats](stats) provides a set of standard summary statistics on a range of different data types, including basic slices of floats, to tensor and table data. It also includes the ability to extract Groups of values and generate statistics for each group, as in a "pivot table" in a spreadsheet. -# Standard shapes and dimensional terminology +# Standard shapes In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: -* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one row-wise dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data (indexes are only available for the outermost row dimension). +* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data. Any `Tensor` can be accessed via a flat 1D index, which goes directly into the underlying Go slice for the basic types, and is appropriately (though somewhat expensively in some cases) indirected through the effective geometry in `Sliced` and `Indexed` types. * **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Indexed` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In most packages including [tmath](tmath) and [stats](stats), 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. @@ -48,6 +56,8 @@ In general, **1D** refers to a flat, 1-dimensional list. There are various stan * **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations. +* **Matrix 3D**: For functions that specifically process 2D matricies, a 3D shape can be used as well, which iterates over the outer row-wise dimension to process the inner 2D matricies. + ## Dynamic row sizing (e.g., for logs) The `SetNumRows` method can be used to progressively increase the number of rows to fit more data, as is typically the case when logging data (often using a [table](table)). You can set the row dimension to 0 to start -- that is (now) safe. However, for greatest efficiency, it is best to set the number of rows to the largest expected size first, and _then_ set it back to 0. The underlying slice of data retains its capacity when sized back down. During incremental increasing of the slice size, if it runs out of capacity, all the elements need to be copied, so it is more efficient to establish the capacity up front instead of having multiple incremental re-allocations. diff --git a/tensor/base.go b/tensor/base.go index 3abd0f0648..6a256e21a4 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -23,13 +23,35 @@ type Base[T any] struct { Meta metadata.Data } -// Shape returns a pointer to the shape that fully parametrizes the tensor shape. -func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } - // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. func (tsr *Base[T]) Metadata() *metadata.Data { return &tsr.Meta } +func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } + +// ShapeSizes returns the sizes of each dimension as an int tensor. +func (tsr *Base[T]) ShapeSizes() Tensor { return tsr.shape.AsTensor() } + +// ShapeInts returns the sizes of each dimension as a slice of ints. +// This is the preferred access for Go code. +func (tsr *Base[T]) ShapeInts() []int { return tsr.shape.Sizes } + +// SetShape sets the dimension sizes as 1D int values from given tensor. +// The backing storage is resized appropriately, retaining all existing data that fits. +func (tsr *Base[T]) SetShape(sizes Tensor) { + tsr.shape.SetShape(sizes) + nln := tsr.Len() + tsr.Values = slicesx.SetLength(tsr.Values, nln) +} + +// SetShapeInts sets the dimension sizes of the tensor, and resizes +// backing storage appropriately, retaining all existing data that fits. +func (tsr *Base[T]) SetShapeInts(sizes ...int) { + tsr.shape.SetShapeInts(sizes...) + nln := tsr.Len() + tsr.Values = slicesx.SetLength(tsr.Values, nln) +} + // Len returns the number of elements in the tensor (product of shape dimensions). func (tsr *Base[T]) Len() int { return tsr.shape.Len() } @@ -63,18 +85,18 @@ func (tsr *Base[T]) Bytes() []byte { } func (tsr *Base[T]) Value(i ...int) T { - return tsr.Values[tsr.shape.Offset(i...)] + return tsr.Values[tsr.shape.IndexTo1D(i...)] } func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[i] } func (tsr *Base[T]) Set(val T, i ...int) { - tsr.Values[tsr.shape.Offset(i...)] = val + tsr.Values[tsr.shape.IndexTo1D(i...)] = val } func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[i] = val } -// view is implementation of View -- needs final casting +// view is implementation of View -- needs final casting to tensor type. func (tsr *Base[T]) view() *Base[T] { nw := &Base[T]{} nw.shape.CopyShape(&tsr.shape) @@ -83,18 +105,6 @@ func (tsr *Base[T]) view() *Base[T] { return nw } -// SetShape sets the shape params, resizing backing storage appropriately. -func (tsr *Base[T]) SetShape(sizes ...int) { - tsr.shape.SetShape(sizes...) - nln := tsr.Len() - tsr.Values = slicesx.SetLength(tsr.Values, nln) -} - -// SetNames sets the dimension names of the tensor shape. -func (tsr *Base[T]) SetNames(names ...string) { - tsr.shape.SetNames(names...) -} - // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) // it is best to first set the anticipated full size, which allocates the @@ -119,13 +129,10 @@ func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { return nil } stsr := &Base[T]{} - stsr.SetShape(tsr.shape.Sizes[od:]...) - if tsr.shape.Names != nil { - stsr.shape.SetNames(tsr.shape.Names...) - } + stsr.SetShapeInts(tsr.shape.Sizes[od:]...) sti := make([]int, nd) copy(sti, offs) - stoff := tsr.shape.Offset(sti...) + stoff := tsr.shape.IndexTo1D(sti...) sln := stsr.Len() stsr.Values = tsr.Values[stoff : stoff+sln] return stsr @@ -134,7 +141,7 @@ func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { ///////////////////// Strings func (tsr *Base[T]) StringValue(i ...int) string { - return reflectx.ToString(tsr.Values[tsr.shape.Offset(i...)]) + return reflectx.ToString(tsr.Values[tsr.shape.IndexTo1D(i...)]) } func (tsr *Base[T]) String1D(off int) string { return reflectx.ToString(tsr.Values[off]) } @@ -216,15 +223,16 @@ func stringIndexed(tsr Tensor, maxLen int, idxs []int) string { maxLen = MaxSprintLength } var b strings.Builder - b.WriteString(tsr.Shape().String() + " ") + sh := tsr.Shape() + b.WriteString(sh.String() + " ") oddRow := false - rows, cols, _, _ := Projection2DShape(tsr.Shape(), oddRow) + rows, cols, _, _ := Projection2DShape(sh, oddRow) if idxs != nil { rows = min(rows, len(idxs)) } ctr := 0 for r := range rows { - rc, _ := Projection2DCoords(tsr.Shape(), oddRow, r, 0) + rc, _ := Projection2DCoords(sh, oddRow, r, 0) b.WriteString(fmt.Sprintf("%v: ", rc)) ri := r if idxs != nil { diff --git a/tensor/bits.go b/tensor/bits.go index 325815aee8..63831957df 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -30,7 +30,7 @@ type Bits struct { // with the given sizes per dimension (shape), and optional dimension names. func NewBits(sizes ...int) *Bits { tsr := &Bits{} - tsr.SetShape(sizes...) + tsr.SetShapeInts(sizes...) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr } @@ -87,9 +87,15 @@ func (tsr *Bits) Bytes() []byte { return slicesx.ToBytes(tsr.Values) } -// Shape returns a pointer to the shape that fully parametrizes the tensor shape func (tsr *Bits) Shape() *Shape { return &tsr.shape } +// ShapeSizes returns the sizes of each dimension as an int tensor. +func (tsr *Bits) ShapeSizes() Tensor { return tsr.shape.AsTensor() } + +// ShapeInts returns the sizes of each dimension as a slice of ints. +// This is the preferred access for Go code. +func (tsr *Bits) ShapeInts() []int { return tsr.shape.Sizes } + // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. func (tsr *Bits) Metadata() *metadata.Data { return &tsr.Meta } @@ -112,28 +118,28 @@ func (tsr *Bits) RowCellSize() (rows, cells int) { // Value returns value at given tensor index func (tsr *Bits) Value(i ...int) bool { - return tsr.Values.Index(tsr.shape.Offset(i...)) + return tsr.Values.Index(tsr.shape.IndexTo1D(i...)) } // Value1D returns value at given tensor 1D (flat) index func (tsr *Bits) Value1D(i int) bool { return tsr.Values.Index(i) } func (tsr *Bits) Set(val bool, i ...int) { - tsr.Values.Set(val, tsr.shape.Offset(i...)) + tsr.Values.Set(val, tsr.shape.IndexTo1D(i...)) } func (tsr *Bits) Set1D(val bool, i int) { tsr.Values.Set(val, i) } -// SetShape sets the shape params, resizing backing storage appropriately -func (tsr *Bits) SetShape(sizes ...int) { - tsr.shape.SetShape(sizes...) +func (tsr *Bits) SetShape(sizes Tensor) { + tsr.shape.SetShape(sizes) nln := tsr.Len() tsr.Values.SetLen(nln) } -// SetNames sets the dimension names of the tensor shape. -func (tsr *Bits) SetNames(names ...string) { - tsr.shape.SetNames(names...) +func (tsr *Bits) SetShapeInts(sizes ...int) { + tsr.shape.SetShapeInts(sizes...) + nln := tsr.Len() + tsr.Values.SetLen(nln) } // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. @@ -175,12 +181,12 @@ func (tsr *Bits) SetString1D(val string, off int) { } func (tsr *Bits) StringValue(i ...int) string { - return reflectx.ToString(tsr.Values.Index(tsr.shape.Offset(i...))) + return reflectx.ToString(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) } func (tsr *Bits) SetString(val string, i ...int) { if bv, err := reflectx.ToBool(val); err == nil { - tsr.Values.Set(bv, tsr.shape.Offset(i...)) + tsr.Values.Set(bv, tsr.shape.IndexTo1D(i...)) } } @@ -196,14 +202,22 @@ func (tsr *Bits) SetStringRowCell(val string, row, cell int) { } } +func (tsr *Bits) StringRow(row int) string { + return tsr.StringRowCell(row, 0) +} + +func (tsr *Bits) SetStringRow(val string, row int) { + tsr.SetStringRowCell(val, row, 0) +} + ///////////////////// Floats func (tsr *Bits) Float(i ...int) float64 { - return BoolToFloat64(tsr.Values.Index(tsr.shape.Offset(i...))) + return BoolToFloat64(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) } func (tsr *Bits) SetFloat(val float64, i ...int) { - tsr.Values.Set(Float64ToBool(val), tsr.shape.Offset(i...)) + tsr.Values.Set(Float64ToBool(val), tsr.shape.IndexTo1D(i...)) } func (tsr *Bits) Float1D(off int) float64 { @@ -224,14 +238,22 @@ func (tsr *Bits) SetFloatRowCell(val float64, row, cell int) { tsr.Values.Set(Float64ToBool(val), row*sz+cell) } +func (tsr *Bits) FloatRow(row int) float64 { + return tsr.FloatRowCell(row, 0) +} + +func (tsr *Bits) SetFloatRow(val float64, row int) { + tsr.SetFloatRowCell(val, row, 0) +} + ///////////////////// Ints func (tsr *Bits) Int(i ...int) int { - return BoolToInt(tsr.Values.Index(tsr.shape.Offset(i...))) + return BoolToInt(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) } func (tsr *Bits) SetInt(val int, i ...int) { - tsr.Values.Set(IntToBool(val), tsr.shape.Offset(i...)) + tsr.Values.Set(IntToBool(val), tsr.shape.IndexTo1D(i...)) } func (tsr *Bits) Int1D(off int) int { @@ -252,6 +274,14 @@ func (tsr *Bits) SetIntRowCell(val int, row, cell int) { tsr.Values.Set(IntToBool(val), row*sz+cell) } +func (tsr *Bits) IntRow(row int) int { + return tsr.IntRowCell(row, 0) +} + +func (tsr *Bits) SetIntRow(val int, row int) { + tsr.SetIntRowCell(val, row, 0) +} + // Label satisfies the core.Labeler interface for a summary description of the tensor func (tsr *Bits) Label() string { return fmt.Sprintf("tensor.Bits: %s", tsr.shape.String()) @@ -327,14 +357,6 @@ func (tsr *Bits) AppendFrom(frm Tensor) error { return nil } -// SetShapeFrom copies just the shape from given source tensor -// calling SetShape with the shape params from source (see for more docs). -func (tsr *Bits) SetShapeFrom(frm Tensor) { - sh := frm.Shape() - tsr.SetShape(sh.Sizes...) - tsr.SetNames(sh.Names...) -} - // CopyCellsFrom copies given range of values from other tensor into this tensor, // using flat 1D indexes: to = starting index in this Tensor to start copying into, // start = starting index on from Tensor to start copying from, and n = number of diff --git a/tensor/convert.go b/tensor/convert.go new file mode 100644 index 0000000000..9ff7de3267 --- /dev/null +++ b/tensor/convert.go @@ -0,0 +1,127 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +// New1DViewOf returns a 1D view into the given tensor, using the same +// underlying values, and just changing the shape to a 1D view. +// This can be useful e.g., for stats and metric functions that report +// on the 1D list of values. +func New1DViewOf(tsr Tensor) Tensor { + vw := tsr.View() + vw.SetShapeInts(tsr.Len()) + return vw +} + +// NewFloat64Scalar is a convenience method for a Tensor +// representation of a single float64 scalar value. +func NewFloat64Scalar(val float64) Tensor { + return NewNumberFromSlice(val) +} + +// NewIntScalar is a convenience method for a Tensor +// representation of a single int scalar value. +func NewIntScalar(val int) Tensor { + return NewNumberFromSlice(val) +} + +// NewStringScalar is a convenience method for a Tensor +// representation of a single string scalar value. +func NewStringScalar(val string) Tensor { + return NewStringFromSlice(val) +} + +// NewFloat64FromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewFloat64FromSlice(vals ...float64) Tensor { + return NewNumberFromSlice(vals...) +} + +// NewIntFromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewIntFromSlice(vals ...int) Tensor { + return NewNumberFromSlice(vals...) +} + +// NewStringFromSlice returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewStringFromSlice(vals ...string) Tensor { + n := len(vals) + tsr := &String{} + tsr.Values = vals + tsr.SetShapeInts(n) + return tsr +} + +// AsFloat64 returns the first value of tensor as an float64. Returns 0 if no values. +func AsFloat64(tsr Tensor) float64 { + if tsr.Len() == 0 { + return 0 + } + return tsr.Float1D(0) +} + +// AsInt returns the first value of tensor as an int. Returns 0 if no values. +func AsInt(tsr Tensor) int { + if tsr.Len() == 0 { + return 0 + } + return tsr.Int1D(0) +} + +// AsString returns the first value of tensor as an string. Returns "" if no values. +func AsString(tsr Tensor) string { + if tsr.Len() == 0 { + return "" + } + return tsr.String1D(0) +} + +// AsFloat64s returns all the tensor values as a slice of float64s. +// This allocates a new slice for the return values, and is not +// a good option for performance-critical code. +func AsFloat64s(tsr Tensor) []float64 { + if tsr.Len() == 0 { + return nil + } + sz := tsr.Len() + slc := make([]float64, sz) + for i := range sz { + slc[i] = tsr.Float1D(i) + } + return slc +} + +// AsInts returns all the tensor values as a slice of ints. +// This allocates a new slice for the return values, and is not +// a good option for performance-critical code. +func AsInts(tsr Tensor) []int { + if tsr.Len() == 0 { + return nil + } + sz := tsr.Len() + slc := make([]int, sz) + for i := range sz { + slc[i] = tsr.Int1D(i) + } + return slc +} + +// AsStrings returns all the tensor values as a slice of strings. +// This allocates a new slice for the return values, and is not +// a good option for performance-critical code. +func AsStrings(tsr Tensor) []string { + if tsr.Len() == 0 { + return nil + } + sz := tsr.Len() + slc := make([]string, sz) + for i := range sz { + slc[i] = tsr.String1D(i) + } + return slc +} diff --git a/tensor/funcs.go b/tensor/funcs.go index bd0e7f32bc..f6100f78b7 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -95,7 +95,7 @@ func AddFunc(name string, fun any, out int, stringFirst ...bool) error { // An error is returned if the function name has not been registered // in the Funcs global function registry, or the argument count // does not match. -func Call(name string, tsr ...*Indexed) error { +func Call(name string, tsr ...Tensor) error { nm := strings.ToLower(name) fn, ok := Funcs[nm] if !ok { @@ -110,7 +110,7 @@ func Call(name string, tsr ...*Indexed) error { // in the Funcs global function registry, or the argument count // does not match. This version of [Call] is for functions that // have an initial string argument -func CallString(name, first string, tsr ...*Indexed) error { +func CallString(name, first string, tsr ...Tensor) error { nm := strings.ToLower(name) fn, ok := Funcs[nm] if !ok { @@ -125,7 +125,7 @@ func CallString(name, first string, tsr ...*Indexed) error { // An error is logged if the function name has not been registered // in the Funcs global function registry, or the argument count // does not match. -func CallOut(name string, tsr ...*Indexed) *Indexed { +func CallOut(name string, tsr ...Tensor) Tensor { nm := strings.ToLower(name) fn, ok := Funcs[nm] if !ok { @@ -141,7 +141,7 @@ func CallOut(name string, tsr ...*Indexed) *Indexed { // An error is logged if the function name has not been registered // in the Funcs global function registry, or the argument count // does not match. -func CallOutMulti(name string, tsr ...*Indexed) []*Indexed { +func CallOutMulti(name string, tsr ...Tensor) []Tensor { nm := strings.ToLower(name) fn, ok := Funcs[nm] if !ok { @@ -156,26 +156,26 @@ func CallOutMulti(name string, tsr ...*Indexed) []*Indexed { func (fn *Func) ArgCount() int { nargs := -1 switch fn.Fun.(type) { - case func(a *Indexed): + case func(a Tensor): nargs = 1 - case func(a, b *Indexed): + case func(a, b Tensor): nargs = 2 - case func(a, b, c *Indexed): + case func(a, b, c Tensor): nargs = 3 - case func(a, b, c, d *Indexed): + case func(a, b, c, d Tensor): nargs = 4 - case func(a, b, c, d, e *Indexed): + case func(a, b, c, d, e Tensor): nargs = 5 // string cases: - case func(s string, a *Indexed): + case func(s string, a Tensor): nargs = 1 - case func(s string, a, b *Indexed): + case func(s string, a, b Tensor): nargs = 2 - case func(s string, a, b, c *Indexed): + case func(s string, a, b, c Tensor): nargs = 3 - case func(s string, a, b, c, d *Indexed): + case func(s string, a, b, c, d Tensor): nargs = 4 - case func(s string, a, b, c, d, e *Indexed): + case func(s string, a, b, c, d, e Tensor): nargs = 5 } return nargs @@ -183,7 +183,7 @@ func (fn *Func) ArgCount() int { // ArgCheck returns an error if the number of args in list does not // match the number required as specified. -func (fn *Func) ArgCheck(n int, tsr ...*Indexed) error { +func (fn *Func) ArgCheck(n int, tsr ...Tensor) error { if len(tsr) != n { return fmt.Errorf("tensor.Call: args passed to %q: %d does not match required: %d", fn.Name, len(tsr), n) } @@ -192,32 +192,32 @@ func (fn *Func) ArgCheck(n int, tsr ...*Indexed) error { // Call calls function with given set of input & output arguments // appropriate for the given function (error if not). -func (fn *Func) Call(tsr ...*Indexed) error { +func (fn *Func) Call(tsr ...Tensor) error { if fn.StringFirst { return fmt.Errorf("tensor.Call: function %q: requires a first string argument", fn.Name) } switch f := fn.Fun.(type) { - case func(a *Indexed): + case func(a Tensor): if err := fn.ArgCheck(1, tsr...); err != nil { return err } f(tsr[0]) - case func(a, b *Indexed): + case func(a, b Tensor): if err := fn.ArgCheck(2, tsr...); err != nil { return err } f(tsr[0], tsr[1]) - case func(a, b, c *Indexed): + case func(a, b, c Tensor): if err := fn.ArgCheck(3, tsr...); err != nil { return err } f(tsr[0], tsr[1], tsr[2]) - case func(a, b, c, d *Indexed): + case func(a, b, c, d Tensor): if err := fn.ArgCheck(4, tsr...); err != nil { return err } f(tsr[0], tsr[1], tsr[2], tsr[3]) - case func(a, b, c, d, e *Indexed): + case func(a, b, c, d, e Tensor): if err := fn.ArgCheck(5, tsr...); err != nil { return err } @@ -229,32 +229,32 @@ func (fn *Func) Call(tsr ...*Indexed) error { // CallString calls function with given set of input & output arguments // appropriate for the given function (error if not), // with an initial string argument. -func (fn *Func) CallString(s string, tsr ...*Indexed) error { +func (fn *Func) CallString(s string, tsr ...Tensor) error { if !fn.StringFirst { return fmt.Errorf("tensor.CallString: function %q: does not take a first string argument", fn.Name) } switch f := fn.Fun.(type) { - case func(s string, a *Indexed): + case func(s string, a Tensor): if err := fn.ArgCheck(1, tsr...); err != nil { return err } f(s, tsr[0]) - case func(s string, a, b *Indexed): + case func(s string, a, b Tensor): if err := fn.ArgCheck(2, tsr...); err != nil { return err } f(s, tsr[0], tsr[1]) - case func(s string, a, b, c *Indexed): + case func(s string, a, b, c Tensor): if err := fn.ArgCheck(3, tsr...); err != nil { return err } f(s, tsr[0], tsr[1], tsr[2]) - case func(s string, a, b, c, d *Indexed): + case func(s string, a, b, c, d Tensor): if err := fn.ArgCheck(4, tsr...); err != nil { return err } f(s, tsr[0], tsr[1], tsr[2], tsr[3]) - case func(s string, a, b, c, d, e *Indexed): + case func(s string, a, b, c, d, e Tensor): if err := fn.ArgCheck(5, tsr...); err != nil { return err } @@ -266,16 +266,16 @@ func (fn *Func) CallString(s string, tsr ...*Indexed) error { // CallOut calls function with given set of _input_ arguments // appropriate for the given function (error if not). // Newly-created output values are returned. -func (fn *Func) CallOut(tsr ...*Indexed) ([]*Indexed, error) { +func (fn *Func) CallOut(tsr ...Tensor) ([]Tensor, error) { if fn.Out == 0 { err := fn.Call(tsr...) return nil, err } typ := reflect.Float64 if fn.In > 0 { - typ = tsr[0].Tensor.DataType() + typ = tsr[0].DataType() } - outs := make([]*Indexed, fn.Out) + outs := make([]Tensor, fn.Out) for i := range outs { outs[i] = NewIndexed(NewOfType(typ)) } diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 53f8c1ef82..774230df2c 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -12,7 +12,7 @@ import ( ) func abs(in, out *Indexed) { - out.SetShapeFrom(in) + SetShapeFrom(out, in) VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...*Indexed) { i, _, _ := tsr[0].RowCellIndex(idx) tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(i)), i) @@ -44,5 +44,5 @@ func TestFuncs(t *testing.T) { out := CallOut("Abs", oned) // assert.NoError(t, err) - assert.Equal(t, oneout.Tensor.(*Float64).Values, out.Tensor.(*Float64).Values) + assert.Equal(t, AsFloat64(oneout), AsFloat64(out)) } diff --git a/tensor/indexed.go b/tensor/indexed.go index e4a94b7e7b..45bcfcf4ab 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -6,31 +6,29 @@ package tensor import ( "cmp" + "log" "math" "math/rand" + "reflect" "slices" "sort" "strings" "cogentcore.org/core/base/metadata" + "gonum.org/v1/gonum/mat" ) // Indexed is an indexed wrapper around a tensor.Tensor that provides a // specific view onto the Tensor defined by the set of indexes, which // apply to the outermost row dimension (with default row-major indexing). -// This is the universal representation of a homogenous data type in the -// [tensor] package framework, from scalar to vector, matrix, and beyond, -// because it can efficiently represent any kind of element with sufficient -// flexibility to enable a full range of computations to be elegantly expressed. -// For example, sorting and filtering a tensor only requires -// updating the indexes while doing nothing to the Tensor itself. +// Sorting and filtering a tensor only requires updating the indexes while +// leaving the underlying Tensor alone. // To produce a new [Tensor] that has its raw data actually organized according -// to the indexed order, call the [NewTensor] method. -// Use the [Set]FloatRowCell methods wherever possible, for the most efficient -// and natural indirection through the indexes. The 1D methods on underlying -// tensor data do not indirect through the indexes and must be called directly -// on the [Tensor]. +// to the indexed order (i.e., the copy function of numpy), call the [NewTensor] method. +// Use the [Set]FloatRow[Cell] methods wherever possible, for the most efficient +// and natural indirection through the indexes. type Indexed struct { //types:add + // Tensor that we are an indexed view onto. Tensor Tensor @@ -40,96 +38,41 @@ type Indexed struct { //types:add Indexes []int } -// NewIndexed returns a new Indexed based on given tensor. -// If a list of indexes is passed, then our indexes are initialized -// as a copy of those. This is used e.g. from a Indexed Table column. -// Otherwise it is initialized with default sequential indexes. -func NewIndexed(tsr Tensor, idxs ...[]int) *Indexed { - ix := &Indexed{} - if len(idxs) == 1 { // indexes were passed - ix.Tensor = tsr - ix.Indexes = slices.Clone(idxs[0]) - } else { - ix.SetTensor(tsr) - } +// NewIndexed returns a new Indexed based on given tensor, +// with optional list of indexes (none / nil = sequential). +func NewIndexed(tsr Tensor, idxs ...int) *Indexed { + ix := &Indexed{Tensor: tsr, Indexes: idxs} return ix } -// NewFloat64Indexed is a convenience method to quickly get an Indexed -// representation of [Float64] tensor of given shape, for use in math routines etc. -func NewFloat64Indexed(sizes ...int) *Indexed { - return &Indexed{Tensor: NewFloat64(sizes...)} -} - -// NewFloat64Scalar is a convenience method to quickly get an Indexed -// representation of a single float64 scalar value, for use in math routines etc. -func NewFloat64Scalar(val float64) *Indexed { - return &Indexed{Tensor: NewNumberFromSlice(val)} -} - -// NewIntScalar is a convenience method to quickly get an Indexed -// representation of a single int scalar value, for use in math routines etc. -func NewIntScalar(val int) *Indexed { - return &Indexed{Tensor: NewNumberFromSlice(val)} -} - -// NewStringScalar is a convenience method to quickly get an Indexed -// representation of a single string scalar value, for use in math routines etc. -func NewStringScalar(val string) *Indexed { - return &Indexed{Tensor: NewStringTensorFromSlice(val)} -} - -// NewFloat64FromSlice returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewFloat64FromSlice(vals ...float64) *Indexed { - return &Indexed{Tensor: NewNumberFromSlice(vals...)} -} - -// NewIntFromSlice returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewIntFromSlice(vals ...int) *Indexed { - return &Indexed{Tensor: NewNumberFromSlice(vals...)} -} - -// NewStringFromSlice returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewStringFromSlice(vals ...string) *Indexed { - return &Indexed{Tensor: NewStringTensorFromSlice(vals...)} -} - -// SetTensor sets as indexes into given tensor with sequential initial indexes +// SetTensor sets as indexes into given tensor with sequential initial indexes. func (ix *Indexed) SetTensor(tsr Tensor) { ix.Tensor = tsr ix.Sequential() } -// SetShapeFrom sets our shape from given source, calling -// [Tensor.SetShape] with the shape params from source, -// and copying the indexes if present. -func (ix *Indexed) SetShapeFrom(src *Indexed) { - ix.Tensor.SetShapeFrom(src.Tensor) - if src.Indexes == nil { - ix.Indexes = nil - } else { - ix.Indexes = slices.Clone(src.Indexes) - } -} - -// Index returns the actual index into underlying tensor row based on given +// RowIndex returns the actual index into underlying tensor row based on given // index value. If Indexes == nil, index is passed through. -func (ix *Indexed) Index(idx int) int { +func (ix *Indexed) RowIndex(idx int) int { if ix.Indexes == nil { return idx } return ix.Indexes[idx] } +// NumRows returns the effective number of rows in this Indexed view, +// which is the length of the index list or number of outer +// rows dimension of tensor if no indexes (full sequential view). +func (ix *Indexed) NumRows() int { + if ix.Indexes == nil { + return ix.Tensor.DimSize(0) + } + return len(ix.Indexes) +} + // String satisfies the fmt.Stringer interface for string of tensor data. func (ix *Indexed) String() string { - return stringIndexed(ix.Tensor, 0, ix.Indexes) + return stringIndexed(ix.Tensor, 0, ix.Indexes) // todo: no need } // Label satisfies the core.Labeler interface for a summary description of the tensor. @@ -137,21 +80,93 @@ func (ix *Indexed) Label() string { return ix.Tensor.Label() } -// note: goal transpiling needs all expressions to work directly on Indexed -// so we need wrappers for everything. - -// Shape returns a pointer to the shape that fully parametrizes the tensor shape. -func (ix *Indexed) Shape() *Shape { return ix.Tensor.Shape() } - // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } +// If we have Indexes, this is the effective shape sizes using +// the current number of indexes as the outermost row dimension size. +func (ix *Indexed) ShapeInts() []int { + if ix.Indexes == nil || ix.Tensor.NumDims() == 0 { + return ix.Tensor.ShapeInts() + } + sh := slices.Clone(ix.Tensor.ShapeInts()) + sh[0] = len(ix.Indexes) + return sh +} + +func (ix *Indexed) ShapeSizes() Tensor { + if ix.Indexes == nil { + return ix.Tensor.ShapeSizes() + } + return NewIntFromSlice(ix.ShapeInts()...) +} + +// Shape() returns a [Shape] representation of the tensor shape +// (dimension sizes). If we have Indexes, this is the effective +// shape using the current number of indexes as the outermost row dimension size. +func (ix *Indexed) Shape() *Shape { + if ix.Indexes == nil { + return ix.Tensor.Shape() + } + return NewShape(ix.ShapeInts()...) +} + +// SetShapeInts sets our shape to given sizes. +// If we do not have indexes, or the row-wise shape dimension +// in the new shape is the same as current, then we set the shape +// of the wrapped Tensor accordingly. +// This allows reshaping of inner dimensions while preserving indexes, +// e.g., for computational routines that use a 1D cell view. +// Otherwise, we reset the indexes and then set the wrapped shape, +// because our current indexes are now invalidated. +func (ix *Indexed) SetShapeInts(sizes ...int) { + if ix.Indexes == nil || ix.Tensor.NumDims() == 0 { + ix.Tensor.SetShapeInts(sizes...) + return + } + sh := ix.Tensor.ShapeInts() + if sizes[0] == sh[0] { // keep our indexes + ix.Tensor.SetShapeInts(sizes...) + return + } + ix.Indexes = nil // now invalid + ix.Tensor.SetShapeInts(sizes...) +} + +// SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. +// This invalidates the indexes. +func (ix *Indexed) SetNumRows(rows int) { + ix.Sequential() + ix.Tensor.SetNumRows(rows) +} + +// SetShape sets our shape to given sizes. +// See [Indexed.SetShapeInts] for details. +func (ix *Indexed) SetShape(sizes Tensor) { + ix.SetShapeInts(AsInts(sizes)...) +} + +// Len returns the total number of elements in the tensor, +// taking into account the Indexes via [Rows], +// as NumRows() * cell size. +func (ix *Indexed) Len() int { + rows := ix.NumRows() + _, cells := ix.Tensor.RowCellSize() + return cells * rows +} + // NumDims returns the total number of dimensions. func (ix *Indexed) NumDims() int { return ix.Tensor.NumDims() } -// DimSize returns size of given dimension. -func (ix *Indexed) DimSize(dim int) int { return ix.Tensor.DimSize(dim) } +// DimSize returns size of given dimension, returning NumRows() +// for first dimension. +func (ix *Indexed) DimSize(dim int) int { + if dim == 0 { + return ix.NumRows() + } + return ix.Tensor.DimSize(dim) +} // RowCellSize returns the size of the outermost Row shape dimension // (via [Indexed.Rows] method), and the size of all the remaining @@ -168,29 +183,10 @@ func (ix *Indexed) RowCellIndex(idx int) (i1d, ri, ci int) { _, cells := ix.Tensor.RowCellSize() ri = idx / cells ci = idx % cells - i1d = ix.Index(ri)*cells + ci + i1d = ix.RowIndex(ri)*cells + ci return } -// NumRows returns the effective number of rows in this Indexed view, -// which is the length of the index list or number of outer -// rows dimension of tensor if no indexes (full sequential view). -func (ix *Indexed) NumRows() int { - if ix.Indexes == nil { - return ix.Tensor.DimSize(0) - } - return len(ix.Indexes) -} - -// Len returns the total number of elements in the tensor, -// taking into account the Indexes via [Rows], -// as Rows() * cell size. -func (ix *Indexed) Len() int { - rows := ix.NumRows() - _, cells := ix.Tensor.RowCellSize() - return cells * rows -} - // DeleteInvalid deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. func (ix *Indexed) DeleteInvalid() { @@ -435,13 +431,20 @@ func (ix *Indexed) NewTensor() Tensor { // Clone returns a copy of the current Indexed view with a cloned copy of // the underlying Tensor and copy of the indexes. -func (ix *Indexed) Clone() *Indexed { +func (ix *Indexed) Clone() Tensor { nix := &Indexed{} nix.Tensor = ix.Tensor.Clone() nix.CopyIndexes(ix) return nix } +func (ix *Indexed) View() Tensor { + nix := &Indexed{} + nix.Tensor = ix.Tensor.View() + nix.CopyIndexes(ix) + return nix +} + // CloneIndexes returns a copy of the current Indexed view with new indexes, // with a pointer to the same underlying Tensor as the source. func (ix *Indexed) CloneIndexes() *Indexed { @@ -526,30 +529,12 @@ func (ix *Indexed) SetFloat(val float64, i ...int) { ix.Tensor.SetFloat(val, ic...) } -// FloatRow returns the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for FloatRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (ix *Indexed) FloatRow(row int) float64 { - return ix.Tensor.FloatRowCell(ix.Index(row), 0) -} - -// SetFloatRow sets the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (ix *Indexed) SetFloatRow(val float64, row int) { - ix.Tensor.SetFloatRowCell(val, ix.Index(row), 0) -} - // FloatRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) FloatRowCell(row, cell int) float64 { - return ix.Tensor.FloatRowCell(ix.Index(row), cell) + return ix.Tensor.FloatRowCell(ix.RowIndex(row), cell) } // SetFloatRowCell sets the value at given row and cell, @@ -557,21 +542,33 @@ func (ix *Indexed) FloatRowCell(row, cell int) float64 { // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { - ix.Tensor.SetFloatRowCell(val, ix.Index(row), cell) + ix.Tensor.SetFloatRowCell(val, ix.RowIndex(row), cell) } -// Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. -// This is just a convenience pass-through to the Tensor, and does _not_ use -// the [Indexed.Indexes]. +// Float1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. func (ix *Indexed) Float1D(i int) float64 { - return ix.Tensor.Float1D(i) + if ix.Indexes == nil { + return ix.Tensor.Float1D(i) + } + return ix.Float(ix.Tensor.Shape().IndexFrom1D(i)...) } -// SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. -// This is just a convenience pass-through to the Tensor, and does _not_ use -// the [Indexed.Indexes]. +// SetFloat1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. func (ix *Indexed) SetFloat1D(val float64, i int) { - ix.Tensor.SetFloat1D(val, i) + if ix.Indexes == nil { + ix.Tensor.SetFloat1D(val, i) + } + ix.SetFloat(val, ix.Tensor.Shape().IndexFrom1D(i)...) +} + +func (ix *Indexed) FloatRow(row int) float64 { + return ix.FloatRowCell(row, 0) +} + +func (ix *Indexed) SetFloatRow(val float64, row int) { + ix.SetFloatRowCell(val, row, 0) } ///////////////////// Strings @@ -598,30 +595,12 @@ func (ix *Indexed) SetString(val string, i ...int) { ix.Tensor.SetString(val, ic...) } -// StringRow returns the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for StringRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (ix *Indexed) StringRow(row int) string { - return ix.Tensor.StringRowCell(ix.Index(row), 0) -} - -// SetStringRow sets the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for SetStringRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (ix *Indexed) SetStringRow(val string, row int) { - ix.Tensor.SetStringRowCell(val, ix.Index(row), 0) -} - // StringRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) StringRowCell(row, cell int) string { - return ix.Tensor.StringRowCell(ix.Index(row), cell) + return ix.Tensor.StringRowCell(ix.RowIndex(row), cell) } // SetStringRowCell sets the value at given row and cell, @@ -629,21 +608,33 @@ func (ix *Indexed) StringRowCell(row, cell int) string { // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetStringRowCell(val string, row, cell int) { - ix.Tensor.SetStringRowCell(val, ix.Index(row), cell) + ix.Tensor.SetStringRowCell(val, ix.RowIndex(row), cell) } -// String1D returns the value of given 1-dimensional index (0-Len()-1) as a string. -// This is just a convenience pass-through to the Tensor, and does _not_ use -// the [Indexed.Indexes]. +// String1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. func (ix *Indexed) String1D(i int) string { - return ix.Tensor.String1D(i) + if ix.Indexes == nil { + return ix.Tensor.String1D(i) + } + return ix.StringValue(ix.Tensor.Shape().IndexFrom1D(i)...) } -// SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. -// This is just a convenience pass-through to the Tensor, and does _not_ use -// the [Indexed.Indexes]. +// SetString1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. func (ix *Indexed) SetString1D(val string, i int) { - ix.Tensor.SetString1D(val, i) + if ix.Indexes == nil { + ix.Tensor.SetString1D(val, i) + } + ix.SetString(val, ix.Tensor.Shape().IndexFrom1D(i)...) +} + +func (ix *Indexed) StringRow(row int) string { + return ix.StringRowCell(row, 0) +} + +func (ix *Indexed) SetStringRow(val string, row int) { + ix.SetStringRowCell(val, row, 0) } ///////////////////// Ints @@ -671,30 +662,12 @@ func (ix *Indexed) SetInt(val int, i ...int) { ix.Tensor.SetInt(val, ic...) } -// IntRow returns the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for IntRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (ix *Indexed) IntRow(row int) int { - return ix.Tensor.IntRowCell(ix.Index(row), 0) -} - -// SetIntRow sets the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for SetIntRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (ix *Indexed) SetIntRow(val int, row int) { - ix.Tensor.SetIntRowCell(val, ix.Index(row), 0) -} - // IntRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) IntRowCell(row, cell int) int { - return ix.Tensor.IntRowCell(ix.Index(row), cell) + return ix.Tensor.IntRowCell(ix.RowIndex(row), cell) } // SetIntRowCell sets the value at given row and cell, @@ -702,21 +675,33 @@ func (ix *Indexed) IntRowCell(row, cell int) int { // Row is indirected through the [Indexed.Indexes]. // This is the preferred interface for all Indexed operations. func (ix *Indexed) SetIntRowCell(val int, row, cell int) { - ix.Tensor.SetIntRowCell(val, ix.Index(row), cell) + ix.Tensor.SetIntRowCell(val, ix.RowIndex(row), cell) } -// Int1D returns the value of given 1-dimensional index (0-Len()-1) as a int. -// This is just a convenience pass-through to the Tensor, and does _not_ use -// the [Indexed.Indexes]. +// Int1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. func (ix *Indexed) Int1D(i int) int { - return ix.Tensor.Int1D(i) + if ix.Indexes == nil { + return ix.Tensor.Int1D(i) + } + return ix.Int(ix.Tensor.Shape().IndexFrom1D(i)...) } -// SetInt1D sets the value of given 1-dimensional index (0-Len()-1) as a int. -// This is just a convenience pass-through to the Tensor, and does _not_ use -// the [Indexed.Indexes]. +// SetInt1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. func (ix *Indexed) SetInt1D(val int, i int) { - ix.Tensor.SetInt1D(val, i) + if ix.Indexes == nil { + ix.Tensor.SetInt1D(val, i) + } + ix.SetInt(val, ix.Tensor.Shape().IndexFrom1D(i)...) +} + +func (ix *Indexed) IntRow(row int) int { + return ix.IntRowCell(row, 0) +} + +func (ix *Indexed) SetIntRow(val int, row int) { + ix.SetIntRowCell(val, row, 0) } ///////////////////// SubSpaces @@ -733,7 +718,7 @@ func (ix *Indexed) SubSpace(offs ...int) Tensor { if len(offs) == 0 { return nil } - offs[0] = ix.Index(offs[0]) + offs[0] = ix.RowIndex(offs[0]) return ix.Tensor.SubSpace(offs...) } @@ -747,11 +732,130 @@ func (ix *Indexed) Cells1D(row int) *Indexed { // RowTensor is a convenience version of [Indexed.SubSpace] to return the // SubSpace for the outermost row dimension, indirected through the indexes. func (ix *Indexed) RowTensor(row int) Tensor { - return ix.Tensor.RowTensor(ix.Index(row)) + return ix.Tensor.RowTensor(ix.RowIndex(row)) } // SetRowTensor sets the values of the SubSpace at given row to given values, // with row indirected through the indexes. func (ix *Indexed) SetRowTensor(val Tensor, row int) { - ix.Tensor.SetRowTensor(val, ix.Index(row)) + ix.Tensor.SetRowTensor(val, ix.RowIndex(row)) +} + +// CopyFrom copies all values from other tensor into this tensor. +// Checks if source is an Indexed and copies indexes too, +// otherwise underlying tensor copies from and indexes are reset. +func (ix *Indexed) CopyFrom(from Tensor) { + if fix, ok := from.(*Indexed); ok { + ix.Tensor.CopyFrom(fix.Tensor) + ix.CopyIndexes(fix) + return + } + ix.Sequential() + ix.Tensor.CopyFrom(from) } + +// AppendFrom appends all values from other tensor into this tensor. +// This invalidates the indexes which are reset. +func (ix *Indexed) AppendFrom(from Tensor) error { + ix.Sequential() + return ix.Tensor.AppendFrom(from) +} + +// CopyCellsFrom copies given range of values from other tensor into this tensor, +// This invalidates the indexes which are reset. +func (ix *Indexed) CopyCellsFrom(from Tensor, to, start, n int) { + ix.Sequential() + ix.Tensor.CopyCellsFrom(from, to, start, n) +} + +func (ix *Indexed) Sizeof() int64 { + return ix.Tensor.Sizeof() // todo: could be out of sync with shape! +} + +func (ix *Indexed) Bytes() []byte { + return ix.Tensor.Bytes() // todo: could be out of sync with shape! +} + +func (ix *Indexed) IsString() bool { + return ix.Tensor.IsString() +} + +func (ix *Indexed) DataType() reflect.Kind { + return ix.Tensor.DataType() +} + +func (ix *Indexed) Range() (min, max float64, minIndex, maxIndex int) { + return ix.Tensor.Range() +} + +func (ix *Indexed) SetZeros() { + ix.Tensor.SetZeros() +} + +////////////////////////// gonum matrix api + +// Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the +// 2D Matrix. Assumes Row-major ordering and logs an error if NumDims < 2. +func (ix *Indexed) Dims() (r, c int) { + nd := ix.NumDims() + if nd < 2 { + log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") + return 0, 0 + } + return ix.DimSize(nd - 2), ix.DimSize(nd - 1) +} + +// Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric +// 2D Matrix. +func (ix *Indexed) Symmetric() (r int) { + nd := ix.NumDims() + if nd < 2 { + log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") + return 0 + } + if ix.DimSize(nd-2) != ix.DimSize(nd-1) { + log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") + return 0 + } + return ix.DimSize(nd - 1) +} + +// SymmetricDim returns the number of rows/columns in the matrix. +func (ix *Indexed) SymmetricDim() int { + nd := ix.NumDims() + if nd < 2 { + log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") + return 0 + } + if ix.DimSize(nd-2) != ix.DimSize(nd-1) { + log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") + return 0 + } + return ix.DimSize(nd - 1) +} + +// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given +// row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. +func (ix *Indexed) At(i, j int) float64 { + nd := ix.NumDims() + if nd < 2 { + log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") + return 0 + } else if nd == 2 { + return ix.Float(i, j) + } else { + nix := make([]int, nd) + nix[nd-2] = i + nix[nd-1] = j + return ix.Float(nix...) + } +} + +// T is the gonum/mat.Matrix transpose method. +// It performs an implicit transpose by returning the receiver inside a Transpose. +func (ix *Indexed) T() mat.Matrix { + return mat.Transpose{ix} +} + +// check for interface impl +var _ Tensor = (*Indexed)(nil) diff --git a/tensor/matrix/README.md b/tensor/matrix/README.md new file mode 100644 index 0000000000..df190c2731 --- /dev/null +++ b/tensor/matrix/README.md @@ -0,0 +1,5 @@ +# matrix: linear algebra with tensors + +This package provides interfaces for `Tensor` types to the [gonum](https://github.com/gonum/gonum) functions for linear algebra, defined on the 2D `mat.Matrix` interface. + + diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go new file mode 100644 index 0000000000..ab088407ec --- /dev/null +++ b/tensor/matrix/matrix.go @@ -0,0 +1,25 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "cogentcore.org/core/tensor" + "gonum.org/v1/gonum/mat" +) + +// CopyDense copies a gonum mat.Dense matrix into given Tensor +// using standard Float64 interface +func CopyDense(to tensor.Tensor, dm *mat.Dense) { + nr, nc := dm.Dims() + to.SetShape(nr, nc) + idx := 0 + for ri := 0; ri < nr; ri++ { + for ci := 0; ci < nc; ci++ { + v := dm.At(ri, ci) + to.SetFloat1D(v, idx) + idx++ + } + } +} diff --git a/tensor/number.go b/tensor/number.go index 1cc0e79785..796f185bfa 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -47,8 +47,7 @@ func AsFloat32(tsr Tensor) *Float32 { if f, ok := tsr.(*Float32); ok { return f } - f := NewFloat32(tsr.Shape().Sizes...) - f.SetNames(tsr.Shape().Names...) + f := NewFloat32(AsInts(tsr.ShapeSizes())...) f.CopyFrom(tsr) return f } @@ -59,17 +58,16 @@ func NewFloat64(sizes ...int) *Float64 { return New[float64](sizes...).(*Float64) } -// AsFloat64 returns the tensor as a [Float64] tensor. +// AsFloat64Tensor returns the tensor as a [Float64] tensor. // If already is a Float64, it is returned as such. // Otherwise, a new Float64 tensor is created and values are copied. // Use this function for interfacing with gonum or other apis that // only operate on float64 types. -func AsFloat64(tsr Tensor) *Float64 { +func AsFloat64Tensor(tsr Tensor) *Float64 { if f, ok := tsr.(*Float64); ok { return f } - f := NewFloat64(tsr.Shape().Sizes...) - f.SetNames(tsr.Shape().Names...) + f := NewFloat64(AsInts(tsr.ShapeSizes())...) f.CopyFrom(tsr) return f } @@ -96,7 +94,7 @@ func NewByte(sizes ...int) *Byte { // with the given sizes per dimension (shape), and optional dimension names. func NewNumber[T num.Number](sizes ...int) *Number[T] { tsr := &Number[T]{} - tsr.SetShape(sizes...) + tsr.SetShapeInts(sizes...) tsr.Values = make([]T, tsr.Len()) return tsr } @@ -120,7 +118,7 @@ func NewNumberFromSlice[T num.Number](vals ...T) Tensor { n := len(vals) tsr := &Number[T]{} tsr.Values = vals - tsr.SetShape(n) + tsr.SetShapeInts(n) return tsr } @@ -137,7 +135,7 @@ func (tsr *Number[T]) IsString() bool { func (tsr *Number[T]) SetString(val string, i ...int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { - tsr.Values[tsr.shape.Offset(i...)] = T(fv) + tsr.Values[tsr.shape.IndexTo1D(i...)] = T(fv) } } @@ -154,14 +152,30 @@ func (tsr *Number[T]) SetStringRowCell(val string, row, cell int) { } } +// StringRow returns the value at given row (outermost dimension). +// It is a convenience wrapper for StringRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (tsr *Number[T]) StringRow(row int) string { + return tsr.StringRowCell(row, 0) +} + +// SetStringRow sets the value at given row (outermost dimension). +// It is a convenience wrapper for SetStringRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (tsr *Number[T]) SetStringRow(val string, row int) { + tsr.SetStringRowCell(val, row, 0) +} + ///////////////////// Floats func (tsr *Number[T]) Float(i ...int) float64 { - return float64(tsr.Values[tsr.shape.Offset(i...)]) + return float64(tsr.Values[tsr.shape.IndexTo1D(i...)]) } func (tsr *Number[T]) SetFloat(val float64, i ...int) { - tsr.Values[tsr.shape.Offset(i...)] = T(val) + tsr.Values[tsr.shape.IndexTo1D(i...)] = T(val) } func (tsr *Number[T]) Float1D(i int) float64 { @@ -183,14 +197,31 @@ func (tsr *Number[T]) SetFloatRowCell(val float64, row, cell int) { tsr.Values[row*sz+cell] = T(val) } +// FloatRow returns the value at given row (outermost dimension). +// It is a convenience wrapper for FloatRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (tsr *Number[T]) FloatRow(row int) float64 { + return tsr.FloatRowCell(row, 0) +} + +// SetFloatRow sets the value at given row (outermost dimension). +// Row is indirected through the [Indexed.Indexes]. +// It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (tsr *Number[T]) SetFloatRow(val float64, row int) { + tsr.SetFloatRowCell(val, row, 0) +} + ///////////////////// Ints func (tsr *Number[T]) Int(i ...int) int { - return int(tsr.Values[tsr.shape.Offset(i...)]) + return int(tsr.Values[tsr.shape.IndexTo1D(i...)]) } func (tsr *Number[T]) SetInt(val int, i ...int) { - tsr.Values[tsr.shape.Offset(i...)] = T(val) + tsr.Values[tsr.shape.IndexTo1D(i...)] = T(val) } func (tsr *Number[T]) Int1D(i int) int { @@ -212,6 +243,22 @@ func (tsr *Number[T]) SetIntRowCell(val int, row, cell int) { tsr.Values[row*sz+cell] = T(val) } +// IntRow returns the value at given row (outermost dimension). +// It is a convenience wrapper for IntRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (tsr *Number[T]) IntRow(row int) int { + return tsr.IntRowCell(row, 0) +} + +// SetIntRow sets the value at given row (outermost dimension). +// It is a convenience wrapper for SetIntRowCell(row, 0), providing robust +// operations on 1D and higher-dimensional data (which nevertheless should +// generally be processed separately in ways that treat it properly). +func (tsr *Number[T]) SetIntRow(val int, row int) { + tsr.SetIntRowCell(val, row, 0) +} + // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. func (tsr *Number[T]) At(i, j int) float64 { @@ -316,14 +363,6 @@ func (tsr *Number[T]) AppendFrom(frm Tensor) error { return nil } -// SetShapeFrom copies just the shape from given source tensor -// calling SetShape with the shape params from source (see for more docs). -func (tsr *Number[T]) SetShapeFrom(frm Tensor) { - sh := frm.Shape() - tsr.SetShape(sh.Sizes...) - tsr.SetNames(sh.Names...) -} - // CopyCellsFrom copies given range of values from other tensor into this tensor, // using flat 1D indexes: to = starting index in this Tensor to start copying into, // start = starting index on from Tensor to start copying from, and n = number of diff --git a/tensor/projection2d.go b/tensor/projection2d.go index d266e1ee34..521805a996 100644 --- a/tensor/projection2d.go +++ b/tensor/projection2d.go @@ -68,18 +68,18 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { return col } case 2: - return shp.Offset(row, col) + return shp.IndexTo1D(row, col) case 3: if oddRow { ny := shp.DimSize(1) yy := row / ny y := row % ny - return shp.Offset(yy, y, col) + return shp.IndexTo1D(yy, y, col) } else { nx := shp.DimSize(2) xx := col / nx x := col % nx - return shp.Offset(xx, row, x) + return shp.IndexTo1D(xx, row, x) } case 4: ny := shp.DimSize(2) @@ -88,7 +88,7 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { nx := shp.DimSize(3) xx := col / nx x := col % nx - return shp.Offset(yy, xx, y, x) + return shp.IndexTo1D(yy, xx, y, x) case 5: // todo: oddRows version! nyy := shp.DimSize(1) @@ -100,7 +100,7 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { nx := shp.DimSize(4) xx := col / nx x := col % nx - return shp.Offset(yyy, yy, xx, y, x) + return shp.IndexTo1D(yyy, yy, xx, y, x) } return 0 } @@ -110,7 +110,7 @@ func Projection2DIndex(shp *Shape, oddRow bool, row, col int) int { // collapsing higher dimensions down to 2D (and 1D up to 2D). func Projection2DCoords(shp *Shape, oddRow bool, row, col int) (rowCoords, colCoords []int) { idx := Projection2DIndex(shp, oddRow, row, col) - dims := shp.Index(idx) + dims := shp.IndexFrom1D(idx) nd := shp.NumDims() switch nd { case 1: diff --git a/tensor/shape.go b/tensor/shape.go index e1f094107c..40c044d35f 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -7,49 +7,46 @@ package tensor import ( "fmt" "slices" - - "cogentcore.org/core/base/slicesx" ) // Shape manages a tensor's shape information, including strides and dimension names // and can compute the flat index into an underlying 1D data storage array based on an // n-dimensional index (and vice-versa). -// Per C / Go / Python conventions, indexes are Row-Major, ordered from +// Per Go / C / Python conventions, indexes are Row-Major, ordered from // outer to inner left-to-right, so the inner-most is right-most. type Shape struct { + // size per dimension. Sizes []int // offsets for each dimension. Strides []int `display:"-"` - - // optional names of each dimension. - Names []string `display:"-"` } // NewShape returns a new shape with given sizes. // RowMajor ordering is used by default. func NewShape(sizes ...int) *Shape { sh := &Shape{} - sh.SetShape(sizes...) + sh.SetShapeInts(sizes...) return sh } -// SetShape sets the shape size and optional names +// SetShapeInts sets the shape sizes from list of ints. // RowMajor ordering is used by default. -func (sh *Shape) SetShape(sizes ...int) { - sh.Sizes = slices.Clone(sizes) +func (sh *Shape) SetShapeInts(sizes ...int) { + sh.Sizes = sizes sh.Strides = RowMajorStrides(sizes...) } -// SetNames sets the shape dimension names. -func (sh *Shape) SetNames(names ...string) { - if len(names) == 0 { - sh.Names = nil - } else { - sh.Names = slicesx.SetLength(names, len(sh.Sizes)) - copy(sh.Names, names) - } +// SetShape sets the shape sizes from int values from Tensor. +// RowMajor ordering is used by default. +func (sh *Shape) SetShape(shp Tensor) { + sh.SetShapeInts(AsInts(shp)...) +} + +// AsTensor returns shape sizes as an Int Tensor. +func (sh *Shape) AsTensor() Tensor { + return NewIntFromSlice(sh.Sizes...) } // CopyShape copies the shape parameters from another Shape struct. @@ -57,15 +54,10 @@ func (sh *Shape) SetNames(names ...string) { func (sh *Shape) CopyShape(cp *Shape) { sh.Sizes = slices.Clone(cp.Sizes) sh.Strides = slices.Clone(cp.Strides) - if cp.Names == nil { - sh.Names = nil - } else { - sh.Names = slices.Clone(cp.Names) - } } // Len returns the total length of elements in the tensor -// (i.e., the product of the shape sizes) +// (i.e., the product of the shape sizes). func (sh *Shape) Len() int { if len(sh.Sizes) == 0 { return 0 @@ -88,31 +80,6 @@ func (sh *Shape) DimSize(i int) int { return sh.Sizes[i] } -// DimName returns the name of given dimension. -func (sh *Shape) DimName(i int) string { - if sh.Names == nil { - return "" - } - return sh.Names[i] -} - -// DimByName returns the index of the given dimension name. -// returns -1 if not found. -func (sh *Shape) DimByName(name string) int { - for i, nm := range sh.Names { - if nm == name { - return i - } - } - return -1 -} - -// DimSizeByName returns the size of given dimension, specified by name. -// will crash if name not found. -func (sh *Shape) DimSizeByName(name string) int { - return sh.DimSize(sh.DimByName(name)) -} - // IndexIsValid() returns true if given index is valid (within ranges for all dimensions) func (sh *Shape) IndexIsValid(idx ...int) bool { if len(idx) != sh.NumDims() { @@ -156,21 +123,22 @@ func (sh *Shape) RowCellSize() (rows, cells int) { return } -// Offset returns the "flat" 1D array index into an element at the given n-dimensional index. -// No checking is done on the length or size of the index values relative to the shape of the tensor. -func (sh *Shape) Offset(index ...int) int { - var offset int +// IndexTo1D returns the flat 1D index from given n-dimensional indicies. +// No checking is done on the length or size of the index values relative +// to the shape of the tensor. +func (sh *Shape) IndexTo1D(index ...int) int { + var oned int for i, v := range index { - offset += v * sh.Strides[i] + oned += v * sh.Strides[i] } - return offset + return oned } -// Index returns the n-dimensional index from a "flat" 1D array index. -func (sh *Shape) Index(offset int) []int { +// IndexFrom1D returns the n-dimensional index from a "flat" 1D array index. +func (sh *Shape) IndexFrom1D(oned int) []int { nd := len(sh.Sizes) index := make([]int, nd) - rem := offset + rem := oned for i := nd - 1; i >= 0; i-- { s := sh.Sizes[i] if s == 0 { @@ -187,12 +155,6 @@ func (sh *Shape) Index(offset int) []int { func (sh *Shape) String() string { str := "[" for i := range sh.Sizes { - if sh.Names != nil { - nm := sh.Names[i] - if nm != "" { - str += nm + ": " - } - } str += fmt.Sprintf("%d", sh.Sizes[i]) if i < len(sh.Sizes)-1 { str += ", " @@ -260,11 +222,5 @@ func AddShapes(shape1, shape2 *Shape) *Shape { copy(nsh, sh1) copy(nsh[len(sh1):], sh2) sh := NewShape(nsh...) - if shape1.Names != nil || shape2.Names != nil { - nms := make([]string, len(sh1)+len(sh2)) - copy(nms, shape1.Names) - copy(nms[len(sh1):], shape2.Names) - sh.SetNames(nms...) - } return sh } diff --git a/tensor/slice.go b/tensor/slice.go index 7d5a7b87d4..9fba593f46 100644 --- a/tensor/slice.go +++ b/tensor/slice.go @@ -89,20 +89,20 @@ func SliceSize(sizes []int, ranges ...Range) ([]int, error) { // which is necessary to allow discontinuous ranges to be extracted. // Use the [SliceSet] function to copy sliced values back to the original. func Slice(tsr, out *Indexed, ranges ...Range) error { - sizes := slices.Clone(tsr.Tensor.Shape().Sizes) + sizes := slices.Clone(tsr.Tensor.ShapeInts()) sizes[0] = tsr.NumRows() // takes into account indexes nsz, err := SliceSize(sizes, ranges...) if err != nil { return err } ndim := len(nsz) - out.Tensor.SetShape(nsz...) + out.Tensor.SetShapeInts(nsz...) out.Sequential() nl := out.Len() oc := make([]int, ndim) // orig coords nr := len(ranges) for ni := range nl { - nc := out.Tensor.Shape().Index(ni) + nc := out.Tensor.Shape().IndexFrom1D(ni) for i := range ndim { c := nc[i] if i < nr { @@ -112,8 +112,8 @@ func Slice(tsr, out *Indexed, ranges ...Range) error { oc[i] = c } } - oc[0] = tsr.Index(oc[0]) - oi := tsr.Tensor.Shape().Offset(oc...) + oc[0] = tsr.RowIndex(oc[0]) + oi := tsr.Tensor.Shape().IndexTo1D(oc...) if out.Tensor.IsString() { out.Tensor.SetString1D(tsr.Tensor.String1D(oi), ni) } else { @@ -128,13 +128,13 @@ func Slice(tsr, out *Indexed, ranges ...Range) error { // function using the same Range sizes (Start offsets // can be different). func SliceSet(tsr, slc *Indexed, ranges ...Range) error { - sizes := slices.Clone(tsr.Tensor.Shape().Sizes) + sizes := slices.Clone(tsr.Tensor.ShapeInts()) sizes[0] = tsr.NumRows() // takes into account indexes nsz, err := SliceSize(sizes, ranges...) if err != nil { return err } - if slices.Compare(nsz, slc.Tensor.Shape().Sizes) != 0 { + if slices.Compare(nsz, slc.Tensor.ShapeInts()) != 0 { return fmt.Errorf("tensor.SliceSet size from ranges is not the same as the slice tensor") } ndim := len(nsz) @@ -142,7 +142,7 @@ func SliceSet(tsr, slc *Indexed, ranges ...Range) error { oc := make([]int, ndim) // orig coords nr := len(ranges) for ni := range nl { - nc := slc.Tensor.Shape().Index(ni) + nc := slc.Tensor.Shape().IndexFrom1D(ni) for i := range ndim { c := nc[i] if i < nr { @@ -152,8 +152,8 @@ func SliceSet(tsr, slc *Indexed, ranges ...Range) error { oc[i] = c } } - oc[0] = tsr.Index(oc[0]) - oi := tsr.Tensor.Shape().Offset(oc...) + oc[0] = tsr.RowIndex(oc[0]) + oi := tsr.Tensor.Shape().IndexTo1D(oc...) if slc.Tensor.IsString() { tsr.Tensor.SetString1D(slc.Tensor.String1D(ni), oi) } else { @@ -169,7 +169,7 @@ func SliceSet(tsr, slc *Indexed, ranges ...Range) error { // form the cells dimension. The resulting tensor is a re-shaped view // of the original tensor, sharing the same underlying data. func RowCellSplit(tsr Tensor, split int) Tensor { - sizes := tsr.Shape().Sizes + sizes := tsr.ShapeInts() rows := sizes[:split] cells := sizes[split:] nr := 1 @@ -181,6 +181,6 @@ func RowCellSplit(tsr Tensor, split int) Tensor { nc *= c } vw := tsr.View() - vw.SetShape(nr, nc) + vw.SetShapeInts(nr, nc) return vw } diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 51e54bd960..b0746fff67 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -21,7 +21,7 @@ func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } nt := len(tsr) out := tsr[nt-1] - osz := out.Tensor.Shape().Sizes + osz := out.Tensor.ShapeInts() out1 = tensor.NewFloat64Indexed(osz...) out2 = tensor.NewFloat64Indexed(osz...) out3 = tensor.NewFloat64Indexed(osz...) @@ -34,12 +34,10 @@ func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } // OutShape returns the output shape based on given input -// tensor, with outer row dim = 1. -func OutShape(ish *tensor.Shape) *tensor.Shape { - osh := &tensor.Shape{} - osh.CopyShape(ish) - osh.Sizes[0] = 1 - return osh +// shape ints, with outer row dim = 1. +func OutShape(ish ...int) []int { + ish[0] = 1 + return ish } // NFunc is the nfun for metrics functions, returning the min number of rows across the @@ -52,9 +50,7 @@ func NFunc(tsr ...*tensor.Indexed) int { } a, b, out := tsr[0], tsr[1], tsr[nt-1] na, nb := a.NumRows(), b.NumRows() - osh := OutShape(a.Tensor.Shape()) - out.Tensor.SetShape(osh.Sizes...) - out.Tensor.SetNames(osh.Names...) + out.Tensor.SetShape(OutShape(a.Tensor.ShapeInts()...)) out.Indexes = nil return min(na, nb) } diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index ead0d3535b..3b4cb6bee2 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -26,7 +26,7 @@ func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, ts } nt := len(tsr) out := tsr[nt-1] - osz := out.Tensor.Shape().Sizes + osz := out.Tensor.ShapeInts() o64 := tensor.NewIndexed(tensor.NewFloat64(osz...)) etsr := slices.Clone(tsr) etsr[nt-1] = o64 @@ -50,7 +50,7 @@ func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } nt := len(tsr) out := tsr[nt-1] - osz := out.Tensor.Shape().Sizes + osz := out.Tensor.ShapeInts() out1 = tensor.NewIndexed(tensor.NewFloat64(osz...)) out2 = tensor.NewIndexed(tensor.NewFloat64(osz...)) tsrs := slices.Clone(tsr[:nt-1]) @@ -62,12 +62,10 @@ func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t } // OutShape returns the output shape based on given input -// tensor, with outer row dim = 1. -func OutShape(ish *tensor.Shape) *tensor.Shape { - osh := &tensor.Shape{} - osh.CopyShape(ish) - osh.Sizes[0] = 1 - return osh +// shape ints, with outer row dim = 1. +func OutShape(ish ...int) []int { + ish[0] = 1 + return ish } // NFunc is the nfun for stats functions, returning number of rows of the @@ -80,9 +78,7 @@ func NFunc(tsr ...*tensor.Indexed) int { } in, out := tsr[0], tsr[nt-1] n := in.NumRows() - osh := OutShape(in.Tensor.Shape()) - out.Tensor.SetShape(osh.Sizes...) - out.Tensor.SetNames(osh.Names...) + out.Tensor.SetShape(OutShape(in.Tensor.ShapeInts()...)...) out.Indexes = nil return n } diff --git a/tensor/string.go b/tensor/string.go index 59bfead895..9b3b129324 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -23,7 +23,7 @@ type String struct { // with the given sizes per dimension (shape). func NewString(sizes ...int) *String { tsr := &String{} - tsr.SetShape(sizes...) + tsr.SetShapeInts(sizes...) tsr.Values = make([]string, tsr.Len()) return tsr } @@ -37,17 +37,6 @@ func NewStringShape(shape *Shape) *String { return tsr } -// NewStringTensorFromSlice returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewStringTensorFromSlice(vals ...string) Tensor { - n := len(vals) - tsr := &String{} - tsr.Values = vals - tsr.SetShape(n) - return tsr -} - // StringToFloat64 converts string value to float64 using strconv, // returning 0 if any error func StringToFloat64(str string) float64 { @@ -74,7 +63,7 @@ func (tsr *String) IsString() bool { ///////////////////// Strings func (tsr *String) SetString(val string, i ...int) { - j := tsr.shape.Offset(i...) + j := tsr.shape.IndexTo1D(i...) tsr.Values[j] = val } @@ -87,14 +76,22 @@ func (tsr *String) SetStringRowCell(val string, row, cell int) { tsr.Values[row*sz+cell] = val } +func (tsr *String) StringRow(row int) string { + return tsr.StringRowCell(row, 0) +} + +func (tsr *String) SetStringRow(val string, row int) { + tsr.SetStringRowCell(val, row, 0) +} + ///////////////////// Floats func (tsr *String) Float(i ...int) float64 { - return StringToFloat64(tsr.Values[tsr.shape.Offset(i...)]) + return StringToFloat64(tsr.Values[tsr.shape.IndexTo1D(i...)]) } func (tsr *String) SetFloat(val float64, i ...int) { - tsr.Values[tsr.shape.Offset(i...)] = Float64ToString(val) + tsr.Values[tsr.shape.IndexTo1D(i...)] = Float64ToString(val) } func (tsr *String) Float1D(off int) float64 { @@ -115,14 +112,22 @@ func (tsr *String) SetFloatRowCell(val float64, row, cell int) { tsr.Values[row*sz+cell] = Float64ToString(val) } +func (tsr *String) FloatRow(row int) float64 { + return tsr.FloatRowCell(row, 0) +} + +func (tsr *String) SetFloatRow(val float64, row int) { + tsr.SetFloatRowCell(val, row, 0) +} + ///////////////////// Ints func (tsr *String) Int(i ...int) int { - return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.Offset(i...)])) + return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.IndexTo1D(i...)])) } func (tsr *String) SetInt(val int, i ...int) { - tsr.Values[tsr.shape.Offset(i...)] = strconv.Itoa(val) + tsr.Values[tsr.shape.IndexTo1D(i...)] = strconv.Itoa(val) } func (tsr *String) Int1D(off int) int { @@ -143,6 +148,14 @@ func (tsr *String) SetIntRowCell(val int, row, cell int) { tsr.Values[row*sz+cell] = strconv.Itoa(val) } +func (tsr *String) IntRow(row int) int { + return tsr.IntRowCell(row, 0) +} + +func (tsr *String) SetIntRow(val int, row int) { + tsr.SetIntRowCell(val, row, 0) +} + // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. func (tsr *String) At(i, j int) float64 { @@ -248,14 +261,6 @@ func (tsr *String) AppendFrom(frm Tensor) error { return nil } -// SetShapeFrom copies just the shape from given source tensor -// calling SetShape with the shape params from source (see for more docs). -func (tsr *String) SetShapeFrom(frm Tensor) { - sh := frm.Shape() - tsr.SetShape(sh.Sizes...) - tsr.SetNames(sh.Names...) -} - // CopyCellsFrom copies given range of values from other tensor into this tensor, // using flat 1D indexes: to = starting index in this Tensor to start copying into, // start = starting index on from Tensor to start copying from, and n = number of diff --git a/tensor/table/io.go b/tensor/table/io.go index 8271ecb47b..da1c6d6641 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -335,7 +335,7 @@ func (dt *Table) WriteCSV(w io.Writer, delim tensor.Delims, headers bool) error cw.Comma = delim.Rune() nrow := dt.NumRows() for ri := range nrow { - ix := dt.Index(ri) + ix := dt.RowIndex(ri) err = dt.WriteCSVRowWriter(cw, ix, ncol) if err != nil { log.Println(err) @@ -401,7 +401,7 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { } rc++ } else { - csh := tensor.NewShape(tsr.Shape().Sizes[1:]...) // cell shape + csh := tensor.NewShape(tsr.ShapeInts[1:]...) // cell shape tc := csh.Len() for ti := 0; ti < tc; ti++ { vl := "" @@ -433,7 +433,7 @@ func (dt *Table) TableHeaders() []string { if tsr.NumDims() == 1 { hdrs = append(hdrs, nm) } else { - csh := tensor.NewShape(tsr.Shape().Sizes[1:]...) // cell shape + csh := tensor.NewShape(tsr.ShapeInts[1:]...) // cell shape tc := csh.Len() nd := csh.NumDims() fnm := nm + fmt.Sprintf("[%v:", nd) diff --git a/tensor/table/io_test.go b/tensor/table/io_test.go index 4cfea9fc66..12c46097a3 100644 --- a/tensor/table/io_test.go +++ b/tensor/table/io_test.go @@ -34,17 +34,19 @@ func TestTableHeaders(t *testing.T) { if cols[2].DataType() != reflect.Float32 { t.Errorf("TableHeaders: cols[2] != FLOAT32\n") } - if cols[1].Shape().Sizes[1] != 5 { - t.Errorf("TableHeaders: cols[1].Shape().Sizes[1] != 5\n") + shsz := AsInts(cols[1].ShapeSizes()) + if shsz[1] != 5 { + t.Errorf("TableHeaders: cols[1].ShapeSizes[1] != 5\n") } - if cols[1].Shape().Sizes[2] != 5 { - t.Errorf("TableHeaders: cols[1].Shape().Sizes[2] != 5\n") + if shsz[2] != 5 { + t.Errorf("TableHeaders: cols[1].ShapeSizes[2] != 5\n") } - if cols[2].Shape().Sizes[1] != 5 { - t.Errorf("TableHeaders: cols[2].Shape().Sizes[1] != 5\n") + shsz = AsInts(cols[2].ShapeSizes()) + if shsz[1] != 5 { + t.Errorf("TableHeaders: cols[2].ShapeSizes[1] != 5\n") } - if cols[2].Shape().Sizes[2] != 5 { - t.Errorf("TableHeaders: cols[2].Shape().Sizes[2] != 5\n") + if shsz[2] != 5 { + t.Errorf("TableHeaders: cols[2].ShapeSizes[2] != 5\n") } outh := dt.TableHeaders() // fmt.Printf("headers out:\n%v\n", outh) @@ -85,16 +87,16 @@ func TestReadTableDat(t *testing.T) { if sc[2].DataType() != reflect.Float32 { t.Errorf("TableHeaders: sc[2] != FLOAT32\n") } - if sc[1].Shape().DimSize(0) != 6 { - t.Errorf("TableHeaders: sc[1].Dim[0] != 6 = %v\n", sc[1].Shape().DimSize(0)) + if sc[1].DimSize(0) != 6 { + t.Errorf("TableHeaders: sc[1].Dim[0] != 6 = %v\n", sc[1].DimSize(0)) } - if sc[1].Shape().DimSize(1) != 5 { + if sc[1].DimSize(1) != 5 { t.Errorf("TableHeaders: sc[1].Dim[1] != 5\n") } - if sc[2].Shape().DimSize(0) != 6 { - t.Errorf("TableHeaders: sc[2].Dim[0] != 6 = %v\n", sc[2].Shape().DimSize(0)) + if sc[2].DimSize(0) != 6 { + t.Errorf("TableHeaders: sc[2].Dim[0] != 6 = %v\n", sc[2].DimSize(0)) } - if sc[2].Shape().DimSize(1) != 5 { + if sc[2].DimSize(1) != 5 { t.Errorf("TableHeaders: sc[2].Dim[1] != 5\n") } fo, err := os.Create("testdata/emer_simple_lines_5x5_rec.dat") diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index befd574acf..945755d0d8 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -21,19 +21,19 @@ func TestAdd3DCol(t *testing.T) { t.Errorf("Add4DCol: # of dims != 4\n") } - if col.Shape().DimSize(0) != 0 { - t.Errorf("Add4DCol: dim 0 len != 0, was: %v\n", col.Shape().DimSize(0)) + if col.DimSize(0) != 0 { + t.Errorf("Add4DCol: dim 0 len != 0, was: %v\n", col.DimSize(0)) } - if col.Shape().DimSize(1) != 11 { - t.Errorf("Add4DCol: dim 0 len != 11, was: %v\n", col.Shape().DimSize(1)) + if col.DimSize(1) != 11 { + t.Errorf("Add4DCol: dim 0 len != 11, was: %v\n", col.DimSize(1)) } - if col.Shape().DimSize(2) != 1 { - t.Errorf("Add4DCol: dim 0 len != 1, was: %v\n", col.Shape().DimSize(2)) + if col.DimSize(2) != 1 { + t.Errorf("Add4DCol: dim 0 len != 1, was: %v\n", col.DimSize(2)) } - if col.Shape().DimSize(3) != 16 { + if col.DimSize(3) != 16 { t.Errorf("Add4DCol: dim 0 len != 16, was: %v\n", col.Shape().DimSize(3)) } } diff --git a/tensor/tensor.go b/tensor/tensor.go index a5ab3bb999..81325c25d2 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -10,6 +10,7 @@ import ( "fmt" "reflect" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/keylist" "cogentcore.org/core/base/metadata" "gonum.org/v1/gonum/mat" @@ -45,19 +46,36 @@ type Tensor interface { // Label satisfies the core.Labeler interface for a summary description of the tensor. Label() string - // Shape returns a pointer to the Shape that fully parametrizes - // the tensor shape. + // Metadata returns the metadata for this tensor, which can be used + // to encode name, docs, shape dimension names, plotting options, etc. + Metadata() *metadata.Data + + // Shape() returns a [Shape] representation of the tensor shape + // (dimension sizes). For tensors that present a view onto another + // tensor, this typically must be constructed on the fly. + // In general, it is much better to access [Tensor.ShapeSizes], + // [Tensor.ShapeInts], [Tensor.DimSize] etc information as neeed. Shape() *Shape - // SetShape sets the sizes parameters of the tensor, and resizes - // backing storage appropriately, retaining all existing data that fits. - SetShape(sizes ...int) + // ShapeSizes returns the sizes of each dimension as an int tensor. + ShapeSizes() Tensor + + // ShapeInts returns the sizes of each dimension as a slice of ints. + // This is the preferred access for Go code. + ShapeInts() []int - // SetNames sets the dimension names of the tensor shape. - SetNames(names ...string) + // SetShape sets the dimension sizes as 1D int values from given tensor. + // The backing storage is resized appropriately, retaining all existing data that fits. + SetShape(sizes Tensor) - // Len returns the number of elements in the tensor, - // which is the product of all shape dimensions. + // SetShapeInts sets the dimension sizes of the tensor, and resizes + // backing storage appropriately, retaining all existing data that fits. + SetShapeInts(sizes ...int) + + // Len returns the total number of elements in the tensor, + // i.e., the product of all shape dimensions. + // Len must always be such that the 1D() accessors return + // values using indexes from 0..Len()-1. Len() int // NumDims returns the total number of dimensions. @@ -97,9 +115,14 @@ type Tensor interface { SetFloat(val float64, i ...int) // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. + // This can be somewhat expensive in wrapper views ([Indexed], [Sliced]), which + // convert the flat index back into a full n-dimensional index and use that api. + // [Tensor.FloatRowCell] is preferred. Float1D(i int) float64 // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. + // This can be somewhat expensive in the commonly-used [Indexed] view; + // [Tensor.SetFloatRowCell] is preferred. SetFloat1D(val float64, i int) // FloatRowCell returns the value at given row and cell, where row is the outermost @@ -114,6 +137,18 @@ type Tensor interface { // This is useful for lists of patterns, and the [table.Table] container. SetFloatRowCell(val float64, row, cell int) + // FloatRow returns the value at given row (outermost dimension). + // It is a convenience wrapper for FloatRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + FloatRow(row int) float64 + + // SetFloatRow sets the value at given row (outermost dimension). + // It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + SetFloatRow(val float64, row int) + ///////////////////// Strings // StringValue returns the value of given n-dimensional index (matching Shape) as a string. @@ -141,6 +176,18 @@ type Tensor interface { // This is useful for lists of patterns, and the [table.Table] container. SetStringRowCell(val string, row, cell int) + // StringRow returns the value at given row (outermost dimension). + // It is a convenience wrapper for StringRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + StringRow(row int) string + + // SetStringRow sets the value at given row (outermost dimension). + // It is a convenience wrapper for SetStringRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + SetStringRow(val string, row int) + ///////////////////// Ints // Int returns the value of given n-dimensional index (matching Shape) as a int. @@ -167,6 +214,18 @@ type Tensor interface { // This is useful for lists of patterns, and the [table.Table] container. SetIntRowCell(val int, row, cell int) + // IntRow returns the value at given row (outermost dimension). + // It is a convenience wrapper for IntRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + IntRow(row int) int + + // SetIntRow sets the value at given row (outermost dimension). + // It is a convenience wrapper for SetIntRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + SetIntRow(val int, row int) + ///////////////////// SubSpaces // SubSpace returns a new tensor with innermost subspace at given @@ -216,10 +275,6 @@ type Tensor interface { // otherwise it goes through the appropriate standard type (Float, Int, String). AppendFrom(from Tensor) error - // SetShapeFrom sets our shape from given source tensor, calling - // [Tensor.SetShape] with the shape params from source. - SetShapeFrom(from Tensor) - // CopyCellsFrom copies given range of values from other tensor into this tensor, // using flat 1D indexes: to = starting index in this Tensor to start copying into, // start = starting index on from Tensor to start copying from, and n = number of @@ -232,10 +287,6 @@ type Tensor interface { // it is best to first set the anticipated full size, which allocates the // full amount of memory, and then set to 0 and grow incrementally. SetNumRows(rows int) - - // Metadata returns the metadata for this tensor, which can be used - // to encode plotting options, etc. - Metadata() *metadata.Data } // New returns a new n-dimensional tensor of given value type @@ -286,29 +337,21 @@ func NewOfType(typ reflect.Kind, sizes ...int) Tensor { } } -// New1DViewOf returns a 1D view into the given tensor, using the same -// underlying values, and just changing the shape to a 1D view. -// This can be useful e.g., for stats and metric functions that report -// on the 1D list of values. -func New1DViewOf(tsr Tensor) Tensor { - vw := tsr.View() - vw.SetShape(tsr.Len()) - return vw +// SetShapeFrom sets shape of given tensor from a source tensor. +func SetShapeFrom(tsr, from Tensor) { + tsr.SetShapeInts(from.ShapeInts()...) } -// CopyDense copies a gonum mat.Dense matrix into given Tensor -// using standard Float64 interface -func CopyDense(to Tensor, dm *mat.Dense) { - nr, nc := dm.Dims() - to.SetShape(nr, nc) - idx := 0 - for ri := 0; ri < nr; ri++ { - for ci := 0; ci < nc; ci++ { - v := dm.At(ri, ci) - to.SetFloat1D(v, idx) - idx++ - } - } +// metadata helpers + +// SetShapeNames sets the tensor shape dimension names into given metadata. +func SetShapeNames(md *metadata.Data, names ...string) { + md.Set("ShapeNames", names) +} + +// ShapeNames gets the tensor shape dimension names from given metadata. +func ShapeNames(md *metadata.Data) []string { + return errors.Log1(metadata.Get[[]string](*md, "ShapeNames")) } // List is an ordered list of Tensors with name lookup. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index f82226bfc0..30826ddf66 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -14,8 +14,8 @@ import ( func TestTensorString(t *testing.T) { tsr := New[string](4, 2) - tsr.SetNames("Row", "Vals") - assert.Equal(t, []string{"Row", "Vals"}, tsr.Shape().Names) + // tsr.SetNames("Row", "Vals") + // assert.Equal(t, []string{"Row", "Vals"}, tsr.Shape().Names) assert.Equal(t, 8, tsr.Len()) assert.Equal(t, true, tsr.IsString()) assert.Equal(t, reflect.String, tsr.DataType()) @@ -41,13 +41,13 @@ func TestTensorString(t *testing.T) { assert.Equal(t, "", cln.StringValue(2, 1)) assert.Equal(t, "testing", tsr.StringValue(2, 1)) - tsr.SetShape(2, 4) - tsr.SetNames("Vals", "Row") + tsr.SetShapeInts(2, 4) + // tsr.SetNames("Vals", "Row") assert.Equal(t, "test", tsr.StringValue(1, 0)) assert.Equal(t, "testing", tsr.StringValue(1, 1)) cln.SetString1D("ctesting", 5) - cln.SetShapeFrom(tsr) + SetShapeFrom(cln, tsr) assert.Equal(t, "ctesting", cln.StringValue(1, 1)) cln.CopyCellsFrom(tsr, 5, 4, 2) @@ -57,25 +57,24 @@ func TestTensorString(t *testing.T) { tsr.SetNumRows(5) assert.Equal(t, 20, tsr.Len()) - tsr.Metadata().Set("name", "test") - nm, err := metadata.Get[string](*tsr.Metadata(), "name") + tsr.Metadata().SetName("test") + nm := tsr.Metadata().Name() assert.Equal(t, "test", nm) - assert.NoError(t, err) - _, err = metadata.Get[string](*tsr.Metadata(), "type") + _, err := metadata.Get[string](*tsr.Metadata(), "type") assert.Error(t, err) cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) - af := AsFloat64(cln) - assert.Equal(t, 3.14, af.Float1D(0)) - assert.Equal(t, 0.0, af.Float1D(1)) + af := AsFloat64s(cln) + assert.Equal(t, 3.14, af[0]) + assert.Equal(t, 0.0, af[1]) } func TestTensorFloat64(t *testing.T) { tsr := New[float64](4, 2) - tsr.SetNames("Row") - assert.Equal(t, []string{"Row", ""}, tsr.Shape().Names) + // tsr.SetNames("Row") + // assert.Equal(t, []string{"Row", ""}, tsr.Shape().Names) assert.Equal(t, 8, tsr.Len()) assert.Equal(t, false, tsr.IsString()) assert.Equal(t, reflect.Float64, tsr.DataType()) @@ -101,13 +100,12 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 0.0, cln.Float(2, 1)) assert.Equal(t, 2.17, tsr.Float(2, 1)) - tsr.SetShape(2, 4) - tsr.SetNames("Vals", "Row") + tsr.SetShapeInts(2, 4) assert.Equal(t, 3.14, tsr.Float(1, 0)) assert.Equal(t, 2.17, tsr.Float(1, 1)) cln.SetFloat1D(9.9, 5) - cln.SetShapeFrom(tsr) + SetShapeFrom(cln, tsr) assert.Equal(t, 9.9, cln.Float(1, 1)) cln.CopyCellsFrom(tsr, 5, 4, 2) @@ -120,13 +118,13 @@ func TestTensorFloat64(t *testing.T) { cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) - af := AsFloat64(cln) - assert.Equal(t, 3.14, af.Float1D(0)) - assert.Equal(t, 0.0, af.Float1D(1)) + af := AsFloat64s(cln) + assert.Equal(t, 3.14, af[0]) + assert.Equal(t, 0.0, af[1]) } func TestSlice(t *testing.T) { - ft := NewFloat64Indexed(3, 4, 5) + ft := NewFloat64(3, 4, 5) for r := range 3 { for y := range 4 { for x := range 5 { @@ -162,7 +160,7 @@ func TestSlice(t *testing.T) { assert.Equal(t, res, ft.Cells1D(1).String()) // fmt.Println(ft.String()) - sf := NewFloat64Indexed() + sf := NewFloat64() Slice(ft, sf, Range{}, Range{Start: 1, End: 2}) // fmt.Println(sf.String()) res = `Tensor: [3, 1, 5] diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index b7ce7a57dc..a4a6aaafbd 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -111,7 +111,7 @@ func (tb *Table) SliceIndex(i int) (si, vi int, invis bool) { si = tb.StartIndex + i vi = -1 if si < tb.Table.NumRows() { - vi = tb.Table.Index(si) + vi = tb.Table.RowIndex(si) } invis = vi < 0 return @@ -383,7 +383,7 @@ func (tb *Table) ColTensorBlank(cidx int, col tensor.Tensor) *tensor.Float64 { if ctb, has := tb.ColumnTensorBlank[cidx]; has { return ctb } - ctb := tensor.New[float64](col.Shape().Sizes...).(*tensor.Float64) + ctb := tensor.New[float64](col.ShapeInts()...).(*tensor.Float64) tb.ColumnTensorBlank[cidx] = ctb return ctb } @@ -677,7 +677,7 @@ func (tb *Table) CopySelectToMime() mimedata.Mimes { idx := tb.SelectedIndexesList(false) // ascending iidx := make([]int, len(idx)) for i, di := range idx { - iidx[i] = tb.Table.Index(di) + iidx[i] = tb.Table.RowIndex(di) } ix.Indexes = iidx var b bytes.Buffer @@ -712,7 +712,7 @@ func (tb *Table) PasteAssign(md mimedata.Mimes, idx int) { if len(recs) == 0 { return } - tb.Table.ReadCSVRow(recs[1], tb.Table.Index(idx)) + tb.Table.ReadCSVRow(recs[1], tb.Table.RowIndex(idx)) tb.UpdateChange() } @@ -727,7 +727,7 @@ func (tb *Table) PasteAtIndex(md mimedata.Mimes, idx int) { tb.Table.InsertRows(idx, nr) for ri := 0; ri < nr; ri++ { rec := recs[1+ri] - rw := tb.Table.Index(idx + ri) + rw := tb.Table.RowIndex(idx + ri) tb.Table.ReadCSVRow(rec, rw) } tb.SendChange() diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index ce9cdb3f4d..4771e5d312 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -3,11 +3,10 @@ package symbols import ( - "reflect" - "cogentcore.org/core/base/metadata" "cogentcore.org/core/tensor" "gonum.org/v1/gonum/mat" + "reflect" ) func init() { From c102aa64182b6b9489d725b7d96147e29a410ce5 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 20 Sep 2024 01:24:54 -0700 Subject: [PATCH 081/311] Indexed as Tensor mostly updated with tests working. need other views now. --- tensor/README.md | 2 +- tensor/convert.go | 88 +++++++- tensor/databrowser/filetree.go | 10 +- tensor/datafs/data.go | 14 +- tensor/datafs/dir.go | 22 +- tensor/datafs/fs.go | 2 +- tensor/datafs/list.go | 2 +- tensor/datafs/metadata.go | 4 +- tensor/examples/datafs-sim/sim.go | 44 ++-- tensor/funcs_test.go | 9 +- tensor/indexed.go | 18 +- tensor/matrix/matrix.go | 2 +- tensor/number.go | 26 --- tensor/shape.go | 2 +- tensor/stats/cluster/clust_test.go | 2 +- tensor/stats/cluster/cluster.go | 16 +- tensor/stats/glm/glm.go | 14 +- tensor/stats/metric/funcs.go | 74 +++---- tensor/stats/metric/matrix.go | 115 +++++----- tensor/stats/metric/metric_test.go | 31 ++- tensor/stats/metric/metrics.go | 4 +- tensor/stats/metric/misc.go | 11 +- tensor/stats/metric/vec.go | 42 ++-- tensor/stats/stats/describe.go | 6 +- tensor/stats/stats/funcs.go | 96 ++++---- tensor/stats/stats/group.go | 28 +-- tensor/stats/stats/group_test.go | 4 +- tensor/stats/stats/quantiles.go | 20 +- tensor/stats/stats/stats.go | 4 +- tensor/stats/stats/stats_test.go | 90 ++++---- tensor/stats/stats/vec.go | 41 ++-- tensor/table/indexes.go | 4 +- tensor/table/io.go | 6 +- tensor/table/io_test.go | 6 +- tensor/table/table.go | 14 +- tensor/table/table_test.go | 2 +- tensor/tensor_test.go | 82 +++---- tensor/tensormpi/tensor.go | 2 +- tensor/tmath/math.go | 339 +++++++++++++---------------- tensor/tmath/math_test.go | 6 +- tensor/tmath/norm.go | 24 +- tensor/tmath/ops.go | 169 +++++++------- tensor/tmath/ops_test.go | 16 +- tensor/vectorize.go | 14 +- 44 files changed, 756 insertions(+), 771 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index ce14836908..6637d6a171 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -106,7 +106,7 @@ ix.SetRowTensor(tsr, 4) // returns a flat, 1D Indexed view into n-dimensional tensor values at // given row. This is used in compute routines that operate generically // on the entire row as a flat pattern. -ci := ix.Cells1D(5) +ci := tensor.Cells1D(ix, 5) ``` ### Full N-dimensional Indexes diff --git a/tensor/convert.go b/tensor/convert.go index 9ff7de3267..c6ad2fb379 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -14,6 +14,13 @@ func New1DViewOf(tsr Tensor) Tensor { return vw } +// Cells1D returns a flat 1D [Tensor] view of the cells for given row +// index. This is useful for passing to other functions e.g., +// in stats or metrics that process a 1D tensor. +func Cells1D(tsr Tensor, row int) Tensor { + return New1DViewOf(tsr.SubSpace(row)) +} + // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. func NewFloat64Scalar(val float64) Tensor { @@ -57,34 +64,37 @@ func NewStringFromSlice(vals ...string) Tensor { return tsr } -// AsFloat64 returns the first value of tensor as an float64. Returns 0 if no values. -func AsFloat64(tsr Tensor) float64 { +// AsFloat64Scalar returns the first value of tensor as a float64 scalar. +// Returns 0 if no values. +func AsFloat64Scalar(tsr Tensor) float64 { if tsr.Len() == 0 { return 0 } return tsr.Float1D(0) } -// AsInt returns the first value of tensor as an int. Returns 0 if no values. -func AsInt(tsr Tensor) int { +// AsIntScalar returns the first value of tensor as an int scalar. +// Returns 0 if no values. +func AsIntScalar(tsr Tensor) int { if tsr.Len() == 0 { return 0 } return tsr.Int1D(0) } -// AsString returns the first value of tensor as an string. Returns "" if no values. -func AsString(tsr Tensor) string { +// AsStringScalar returns the first value of tensor as a string scalar. +// Returns "" if no values. +func AsStringScalar(tsr Tensor) string { if tsr.Len() == 0 { return "" } return tsr.String1D(0) } -// AsFloat64s returns all the tensor values as a slice of float64s. +// AsFloat64Slice returns all the tensor values as a slice of float64's. // This allocates a new slice for the return values, and is not // a good option for performance-critical code. -func AsFloat64s(tsr Tensor) []float64 { +func AsFloat64Slice(tsr Tensor) []float64 { if tsr.Len() == 0 { return nil } @@ -96,10 +106,10 @@ func AsFloat64s(tsr Tensor) []float64 { return slc } -// AsInts returns all the tensor values as a slice of ints. +// AsIntSlice returns all the tensor values as a slice of ints. // This allocates a new slice for the return values, and is not // a good option for performance-critical code. -func AsInts(tsr Tensor) []int { +func AsIntSlice(tsr Tensor) []int { if tsr.Len() == 0 { return nil } @@ -111,10 +121,10 @@ func AsInts(tsr Tensor) []int { return slc } -// AsStrings returns all the tensor values as a slice of strings. +// AsStringSlice returns all the tensor values as a slice of strings. // This allocates a new slice for the return values, and is not // a good option for performance-critical code. -func AsStrings(tsr Tensor) []string { +func AsStringSlice(tsr Tensor) []string { if tsr.Len() == 0 { return nil } @@ -125,3 +135,57 @@ func AsStrings(tsr Tensor) []string { } return slc } + +// AsFloat64Tensor returns the tensor as a [Float64] tensor. +// If already is a Float64, it is returned as such. +// Otherwise, a new Float64 tensor is created and values are copied. +// Use this function for interfacing with gonum or other apis that +// only operate on float64 types. +func AsFloat64Tensor(tsr Tensor) *Float64 { + if f, ok := tsr.(*Float64); ok { + return f + } + f := NewFloat64(tsr.ShapeInts()...) + f.CopyFrom(tsr) + return f +} + +// AsFloat32Tensor returns the tensor as a [Float32] tensor. +// If already is a Float32, it is returned as such. +// Otherwise, a new Float32 tensor is created and values are copied. +func AsFloat32Tensor(tsr Tensor) *Float32 { + if f, ok := tsr.(*Float32); ok { + return f + } + f := NewFloat32(AsIntSlice(tsr.ShapeSizes())...) + f.CopyFrom(tsr) + return f +} + +// AsStringTensor returns the tensor as a [String] tensor. +// If already is a String, it is returned as such. +// Otherwise, a new String tensor is created and values are copied. +// Use this function for interfacing with gonum or other apis that +// only operate on float64 types. +func AsStringTensor(tsr Tensor) *String { + if f, ok := tsr.(*String); ok { + return f + } + f := NewString(tsr.ShapeInts()...) + f.CopyFrom(tsr) + return f +} + +// AsIntTensor returns the tensor as a [Int] tensor. +// If already is a Int, it is returned as such. +// Otherwise, a new Int tensor is created and values are copied. +// Use this function for interfacing with gonum or other apis that +// only operate on float64 types. +func AsIntTensor(tsr Tensor) *Int { + if f, ok := tsr.(*Int); ok { + return f + } + f := NewInt(tsr.ShapeInts()...) + f.CopyFrom(tsr) + return f +} diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index 71d28f7c63..1a52968f4f 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -106,7 +106,7 @@ func (fn *FileNode) OpenFile() error { switch fn.Info.Known { case fileinfo.Tensor: d := DataFS(ofn) - br.NewTabTensorEditor(df, d.Data.Tensor) + br.NewTabTensorEditor(df, d.Data) // case fileinfo.Table: // d := DataFS(ofn) // dt := d.AsTable() @@ -222,13 +222,15 @@ func (fn *FileNode) PlotFile() { case fileinfo.Tensor: tsr := d.Data dt = table.NewTable(df) - dt.Columns.Rows = tsr.NumRows() - dt.Indexes = tsr.Indexes + dt.Columns.Rows = tsr.DimSize(0) + if ix, ok := tsr.(*tensor.Indexed); ok { + dt.Indexes = ix.Indexes + } rc := dt.AddIntColumn("Row") for r := range dt.Columns.Rows { rc.Values[r] = r } - dt.AddColumn(fn.Name, tsr.Tensor) + dt.AddColumn(fn.Name, tsr) // case fileinfo.Table: // dt = d.AsTable() default: diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 1c89261d79..fb5e3b4326 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -33,7 +33,7 @@ type Data struct { // represented using the universal [tensor] data type of // [tensor.Indexed], which can represent anything from a scalar // to n-dimensional data, in a range of data types. - Data *tensor.Indexed + Data tensor.Tensor // Dir is for directory nodes, with all the items in the directory. Dir *Dir @@ -61,8 +61,8 @@ func newData(dir *Data, name string) (*Data, error) { // of given data type, in given directory. // The names must be unique in the directory. // Returns the first item created, for immediate use of one value. -func NewScalar[T tensor.DataTypes](dir *Data, names ...string) *tensor.Indexed { - var first *tensor.Indexed +func NewScalar[T tensor.DataTypes](dir *Data, names ...string) tensor.Tensor { + var first tensor.Tensor for _, nm := range names { tsr := tensor.New[T](1) tsr.Metadata().SetName(nm) @@ -81,7 +81,7 @@ func NewScalar[T tensor.DataTypes](dir *Data, names ...string) *tensor.Indexed { // NewValue returns a new Data value as a [tensor.Indexed] [tensor.Tensor] // of given data type and shape sizes, in given directory Data item. // The name must be unique in the directory. -func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) *tensor.Indexed { +func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Tensor { tsr := tensor.New[T](sizes...) tsr.Metadata().SetName(name) d, err := newData(dir, name) @@ -96,7 +96,7 @@ func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) *tensor. // of given reflect.Kind type and shape sizes per dimension, in given directory Data item. // Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. // The name must be unique in the directory. -func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) *tensor.Indexed { +func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Tensor { tsr := tensor.NewOfType(typ, sizes...) tsr.Metadata().SetName(name) nd, err := newData(d, name) @@ -111,7 +111,7 @@ func (d *Data) KnownFileInfo() fileinfo.Known { if d.Data == nil { return fileinfo.Unknown } - tsr := d.Data.Tensor + tsr := d.Data if tsr.Len() > 1 { return fileinfo.Tensor } @@ -129,7 +129,7 @@ func (d *Data) Bytes() []byte { if d.Data == nil { return nil } - return d.Data.Tensor.Bytes() + return d.Data.Bytes() } // AsString returns data as scalar string. diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index a16705572e..b55a1945e4 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -52,7 +52,7 @@ func (d *Data) Item(name string) *Data { // within this directory. This will panic if item is not // found, and will return nil if it is not a Value // (i.e., it is a directory). -func (d *Data) Value(name string) *tensor.Indexed { +func (d *Data) Value(name string) tensor.Tensor { return d.Dir.At(name).Data } @@ -78,12 +78,12 @@ func (d *Data) Items(names ...string) ([]*Data, error) { // Values returns Value items (tensors) in given directory by name. // error reports any items not found, or if not a directory. -func (d *Data) Values(names ...string) ([]*tensor.Indexed, error) { +func (d *Data) Values(names ...string) ([]tensor.Tensor, error) { if err := d.mustDir("Values", ""); err != nil { return nil, err } var errs []error - var its []*tensor.Indexed + var its []tensor.Tensor for _, nm := range names { it := d.Dir.At(nm) if it != nil && it.Data != nil { @@ -118,11 +118,11 @@ func (d *Data) ItemsFunc(fun func(item *Data) bool) []*Data { // ValuesFunc returns Value items (tensors) in given directory // filtered by given function, in directory order (e.g., order added). // If func is nil, all values are returned. -func (d *Data) ValuesFunc(fun func(item *Data) bool) []*tensor.Indexed { +func (d *Data) ValuesFunc(fun func(item *Data) bool) []tensor.Tensor { if err := d.mustDir("ItemsFunc", ""); err != nil { return nil } - var its []*tensor.Indexed + var its []tensor.Tensor for _, it := range d.Dir.Values { if it.Data == nil { continue @@ -162,11 +162,11 @@ func (d *Data) ItemsAlphaFunc(fun func(item *Data) bool) []*Data { // (e.g., order added). // The function can filter out directories to prune the tree. // If func is nil, all Value items are returned. -func (d *Data) FlatValuesFunc(fun func(item *Data) bool) []*tensor.Indexed { +func (d *Data) FlatValuesFunc(fun func(item *Data) bool) []tensor.Tensor { if err := d.mustDir("FlatValuesFunc", ""); err != nil { return nil } - var its []*tensor.Indexed + var its []tensor.Tensor for _, it := range d.Dir.Values { if fun != nil && !fun(it) { continue @@ -211,12 +211,12 @@ func (d *Data) FlatItemsFunc(fun func(item *Data) bool) []*Data { // the entire subtree, filtered by given function, in alphabetical order. // The function can filter out directories to prune the tree. // If func is nil, all items are returned. -func (d *Data) FlatValuesAlphaFunc(fun func(item *Data) bool) []*tensor.Indexed { +func (d *Data) FlatValuesAlphaFunc(fun func(item *Data) bool) []tensor.Tensor { if err := d.mustDir("FlatValuesFunc", ""); err != nil { return nil } names := d.DirNamesAlpha() - var its []*tensor.Indexed + var its []tensor.Tensor for _, nm := range names { it := d.Dir.At(nm) if fun != nil && !fun(it) { @@ -373,7 +373,7 @@ func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { dt := table.NewTable(fsx.DirAndFile(string(d.Path()))) for _, it := range its { tsr := it.Data - rows := tsr.NumRows() + rows := tsr.DimSize(0) if dt.Columns.Rows < rows { dt.Columns.Rows = rows dt.SetNumRows(dt.Columns.Rows) @@ -382,7 +382,7 @@ func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { if it.Parent != d { nm = fsx.DirAndFile(string(it.Path())) } - dt.AddColumn(nm, tsr.Tensor) + dt.AddColumn(nm, tsr) } d.DirTable = dt return dt diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index 5e0d937e4f..75ffff2744 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -115,7 +115,7 @@ func (d *Data) Size() int64 { if d.Data == nil { return 0 } - return d.Data.Tensor.Sizeof() + return d.Data.Sizeof() } func (d *Data) IsDir() bool { diff --git a/tensor/datafs/list.go b/tensor/datafs/list.go index 9000d0306c..5fc375d5a6 100644 --- a/tensor/datafs/list.go +++ b/tensor/datafs/list.go @@ -18,7 +18,7 @@ const ( func (d *Data) String() string { if !d.IsDir() { - return d.Data.Tensor.Label() + return d.Data.Label() } return d.List(Short, DirOnly) } diff --git a/tensor/datafs/metadata.go b/tensor/datafs/metadata.go index 5ac08998d4..c9589f12e4 100644 --- a/tensor/datafs/metadata.go +++ b/tensor/datafs/metadata.go @@ -17,7 +17,7 @@ import ( func (d *Data) SetMetaItems(key string, value any, names ...string) error { tsrs, err := d.Values(names...) for _, tsr := range tsrs { - tsr.Tensor.Metadata().Set(key, value) + tsr.Metadata().Set(key, value) } return err } @@ -29,7 +29,7 @@ func (d *Data) CalcAll() error { var errs []error items := d.FlatValuesFunc(nil) for _, it := range items { - err := tensor.Calc(it.Tensor) + err := tensor.Calc(it) if err != nil { errs = append(errs, err) } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 7a6e74f60b..4323532da6 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -65,12 +65,12 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { ntrial := ss.Config.Item("NTrial").AsInt() sitems := ss.Stats.ValuesFunc(nil) for _, st := range sitems { - nm := st.Tensor.Metadata().Name() - lt := logd.NewOfType(nm, st.Tensor.DataType(), ntrial) - lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source - tensor.SetCalcFunc(lt.Tensor, func() error { + nm := st.Metadata().Name() + lt := logd.NewOfType(nm, st.DataType(), ntrial) + lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source + tensor.SetCalcFunc(lt, func() error { trl := ss.Stats.Item("Trial").AsInt() - if st.Tensor.IsString() { + if st.IsString() { lt.SetStringRow(st.StringRow(0), trl) } else { lt.SetFloatRow(st.FloatRow(0), trl) @@ -80,15 +80,15 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { } alllogd, _ := dir.Mkdir("AllTrials") for _, st := range sitems { - nm := st.Tensor.Metadata().Name() + nm := st.Metadata().Name() // allocate full size - lt := alllogd.NewOfType(nm, st.Tensor.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) - lt.Tensor.SetShape(0) // then truncate to 0 - lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) // key affordance: we get meta data from source - tensor.SetCalcFunc(lt.Tensor, func() error { - row := lt.Tensor.DimSize(0) - lt.Tensor.SetShape(row + 1) - if st.Tensor.IsString() { + lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) + lt.SetShapeInts(0) // then truncate to 0 + lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source + tensor.SetCalcFunc(lt, func() error { + row := lt.DimSize(0) + lt.SetShapeInts(row + 1) + if st.IsString() { lt.SetStringRow(st.StringRow(0), row) } else { lt.SetFloatRow(st.FloatRow(0), row) @@ -106,18 +106,18 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a nctr := ss.Config.Item("N" + level).AsInt() stout := tensor.NewFloat64Scalar(0) // tmp stat output for _, st := range sitems { - if st.Tensor.IsString() { + if st.IsString() { continue } - nm := st.Tensor.Metadata().Name() + nm := st.Metadata().Name() src := from.Value(nm) - if st.Tensor.DataType() >= reflect.Float32 { + if st.DataType() >= reflect.Float32 { // todo: pct correct etc dd, _ := logd.Mkdir(nm) for _, ag := range aggs { // key advantage of dir structure: multiple stats per item - lt := dd.NewOfType(ag.String(), st.Tensor.DataType(), nctr) - lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) - tensor.SetCalcFunc(lt.Tensor, func() error { + lt := dd.NewOfType(ag.String(), st.DataType(), nctr) + lt.Metadata().Copy(*st.Metadata()) + tensor.SetCalcFunc(lt, func() error { stats.Stat(ag, src, stout) ctr := ss.Stats.Item(level).AsInt() lt.SetFloatRow(stout.FloatRow(0), ctr) @@ -125,9 +125,9 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a }) } } else { - lt := logd.NewOfType(nm, st.Tensor.DataType(), nctr) - lt.Tensor.Metadata().Copy(*st.Tensor.Metadata()) - tensor.SetCalcFunc(lt.Tensor, func() error { + lt := logd.NewOfType(nm, st.DataType(), nctr) + lt.Metadata().Copy(*st.Metadata()) + tensor.SetCalcFunc(lt, func() error { v := st.FloatRow(0) ctr := ss.Stats.Item(level).AsInt() lt.SetFloatRow(v, ctr) diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 774230df2c..87096057d8 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -11,11 +11,10 @@ import ( "github.com/stretchr/testify/assert" ) -func abs(in, out *Indexed) { +func abs(in, out Tensor) { SetShapeFrom(out, in) - VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...*Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(i)), i) + VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...Tensor) { + tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) } @@ -44,5 +43,5 @@ func TestFuncs(t *testing.T) { out := CallOut("Abs", oned) // assert.NoError(t, err) - assert.Equal(t, AsFloat64(oneout), AsFloat64(out)) + assert.Equal(t, AsFloat64Scalar(oneout), AsFloat64Scalar(out)) } diff --git a/tensor/indexed.go b/tensor/indexed.go index 45bcfcf4ab..14d20dc043 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -45,6 +45,15 @@ func NewIndexed(tsr Tensor, idxs ...int) *Indexed { return ix } +// AsIndexed returns the tensor as an Indexed view. +// If it already is one, then it is returned, otherwise it is wrapped. +func AsIndexed(tsr Tensor) *Indexed { + if ix, ok := tsr.(*Indexed); ok { + return ix + } + return NewIndexed(tsr) +} + // SetTensor sets as indexes into given tensor with sequential initial indexes. func (ix *Indexed) SetTensor(tsr Tensor) { ix.Tensor = tsr @@ -144,7 +153,7 @@ func (ix *Indexed) SetNumRows(rows int) { // SetShape sets our shape to given sizes. // See [Indexed.SetShapeInts] for details. func (ix *Indexed) SetShape(sizes Tensor) { - ix.SetShapeInts(AsInts(sizes)...) + ix.SetShapeInts(AsIntSlice(sizes)...) } // Len returns the total number of elements in the tensor, @@ -722,13 +731,6 @@ func (ix *Indexed) SubSpace(offs ...int) Tensor { return ix.Tensor.SubSpace(offs...) } -// Cells1D returns a flat 1D [tensor.Indexed] view of the cells for given row -// index (indirected through our Indexes). This is useful for passing to -// other functions e.g., in stats or metrics that process a 1D tensor. -func (ix *Indexed) Cells1D(row int) *Indexed { - return NewIndexed(New1DViewOf(ix.SubSpace(row))) -} - // RowTensor is a convenience version of [Indexed.SubSpace] to return the // SubSpace for the outermost row dimension, indirected through the indexes. func (ix *Indexed) RowTensor(row int) Tensor { diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go index ab088407ec..0937112696 100644 --- a/tensor/matrix/matrix.go +++ b/tensor/matrix/matrix.go @@ -13,7 +13,7 @@ import ( // using standard Float64 interface func CopyDense(to tensor.Tensor, dm *mat.Dense) { nr, nc := dm.Dims() - to.SetShape(nr, nc) + to.SetShapeInts(nr, nc) idx := 0 for ri := 0; ri < nr; ri++ { for ci := 0; ci < nc; ci++ { diff --git a/tensor/number.go b/tensor/number.go index 796f185bfa..413449c14e 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -40,38 +40,12 @@ func NewFloat32(sizes ...int) *Float32 { return New[float32](sizes...).(*Float32) } -// AsFloat32 returns the tensor as a [Float32] tensor. -// If already is a Float32, it is returned as such. -// Otherwise, a new Float32 tensor is created and values are copied. -func AsFloat32(tsr Tensor) *Float32 { - if f, ok := tsr.(*Float32); ok { - return f - } - f := NewFloat32(AsInts(tsr.ShapeSizes())...) - f.CopyFrom(tsr) - return f -} - // NewFloat64 returns a new [Float64] tensor // with the given sizes per dimension (shape), and optional dimension names. func NewFloat64(sizes ...int) *Float64 { return New[float64](sizes...).(*Float64) } -// AsFloat64Tensor returns the tensor as a [Float64] tensor. -// If already is a Float64, it is returned as such. -// Otherwise, a new Float64 tensor is created and values are copied. -// Use this function for interfacing with gonum or other apis that -// only operate on float64 types. -func AsFloat64Tensor(tsr Tensor) *Float64 { - if f, ok := tsr.(*Float64); ok { - return f - } - f := NewFloat64(AsInts(tsr.ShapeSizes())...) - f.CopyFrom(tsr) - return f -} - // NewInt returns a new Int tensor // with the given sizes per dimension (shape), and optional dimension names. func NewInt(sizes ...int) *Int { diff --git a/tensor/shape.go b/tensor/shape.go index 40c044d35f..6031df3708 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -41,7 +41,7 @@ func (sh *Shape) SetShapeInts(sizes ...int) { // SetShape sets the shape sizes from int values from Tensor. // RowMajor ordering is used by default. func (sh *Shape) SetShape(shp Tensor) { - sh.SetShapeInts(AsInts(shp)...) + sh.SetShapeInts(AsIntSlice(shp)...) } // AsTensor returns shape sizes as an Int Tensor. diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 1bc7b63655..561579c32a 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -34,7 +34,7 @@ func TestClust(t *testing.T) { t.Error(err) } in := dt.Column("Input") - out := tensor.NewFloat64Indexed() + out := tensor.NewFloat64() metric.Matrix(metric.Euclidean.FuncName(), in, out) cl := Cluster(Avg.String(), out, dt.Column("Name")) diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index d504ce02de..6b9992a414 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -47,9 +47,9 @@ func (nn *Node) IsLeaf() bool { } // Sprint prints to string -func (nn *Node) Sprint(labels *tensor.Indexed, depth int) string { +func (nn *Node) Sprint(labels tensor.Tensor, depth int) string { if nn.IsLeaf() && labels != nil { - return labels.Tensor.String1D(nn.Index) + " " + return labels.String1D(nn.Index) + " " } sv := fmt.Sprintf("\n%v%v: ", indent.Tabs(depth), nn.Dist) for _, kn := range nn.Kids { @@ -89,8 +89,8 @@ func NewNode(na, nb *Node, dst float64) *Node { // and then Glom to do the iterative agglomerative clustering process. // If you want to start with pre-defined initial clusters, // then call Glom with a root node so-initialized. -func Cluster(funcName string, dmat, labels *tensor.Indexed) *Node { - ntot := dmat.Tensor.DimSize(0) // number of leaves +func Cluster(funcName string, dmat, labels tensor.Tensor) *Node { + ntot := dmat.DimSize(0) // number of leaves root := InitAllLeaves(ntot) return Glom(root, funcName, dmat) } @@ -111,10 +111,10 @@ func InitAllLeaves(ntot int) *Node { // with the starting clusters, which is all of the // leaves by default, but could be anything if you want // to start with predefined clusters. -func Glom(root *Node, funcName string, dmat *tensor.Indexed) *Node { - ntot := dmat.Tensor.DimSize(0) // number of leaves +func Glom(root *Node, funcName string, dmat tensor.Tensor) *Node { + ntot := dmat.DimSize(0) // number of leaves mout := tensor.NewFloat64Scalar(0) - stats.MaxFunc(tensor.NewIndexed(tensor.New1DViewOf(dmat.Tensor)), mout) + stats.MaxFunc(tensor.New1DViewOf(dmat), mout) maxd := mout.Float1D(0) // indexes in each group aidx := make([]int, ntot) @@ -131,7 +131,7 @@ func Glom(root *Node, funcName string, dmat *tensor.Indexed) *Node { bctr := 0 kb.Indexes(bidx, &bctr) bix := bidx[0:bctr] - dv := Call(funcName, aix, bix, ntot, maxd, dmat.Tensor) + dv := Call(funcName, aix, bix, ntot, maxd, dmat) if dv < mval { mval = dv ma = []int{ai} diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index a3bcd155b6..1045da7ae7 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -88,7 +88,7 @@ type GLM struct { Table *table.Table // tensor columns from table with the respective variables - IndepVars, DepVars, PredVars, ErrVars *tensor.Indexed + IndepVars, DepVars, PredVars, ErrVars tensor.Tensor // Number of independent and dependent variables NIndepVars, NDepVars int @@ -110,8 +110,8 @@ func (glm *GLM) Defaults() { func (glm *GLM) init(nIv, nDv int) { glm.NIndepVars = nIv glm.NDepVars = nDv - glm.Coeff.SetShape(nDv, nIv+1) - glm.Coeff.SetNames("DepVars", "IndepVars") + glm.Coeff.SetShapeInts(nDv, nIv+1) + // glm.Coeff.SetNames("DepVars", "IndepVars") glm.R2 = make([]float64, nDv) glm.ObsVariance = make([]float64, nDv) glm.ErrVariance = make([]float64, nDv) @@ -126,17 +126,17 @@ func (glm *GLM) init(nIv, nDv int) { func (glm *GLM) SetTable(dt *table.Table, indepVars, depVars, predVars, errVars string) error { iv := dt.Column(indepVars) dv := dt.Column(depVars) - var pv, ev *tensor.Indexed + var pv, ev tensor.Tensor if predVars != "" { pv = dt.Column(predVars) } if errVars != "" { ev = dt.Column(errVars) } - if pv != nil && !pv.Tensor.Shape().IsEqual(dv.Tensor.Shape()) { + if pv != nil && !pv.Shape().IsEqual(dv.Shape()) { return fmt.Errorf("predVars must have same shape as depVars") } - if ev != nil && !ev.Tensor.Shape().IsEqual(dv.Tensor.Shape()) { + if ev != nil && !ev.Shape().IsEqual(dv.Shape()) { return fmt.Errorf("errVars must have same shape as depVars") } _, nIv := iv.RowCellSize() @@ -190,7 +190,7 @@ func (glm *GLM) Run() { lrate *= 0.5 } for i := 0; i < n; i++ { - row := dt.Index(i) + row := dt.RowIndex(i) for di := 0; di < nDv; di++ { pred := 0.0 for ii := 0; ii < nIv; ii++ { diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 00d651d39e..4842d25a5d 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -28,12 +28,12 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type MetricFunc func(a, b, out *tensor.Indexed) +type MetricFunc func(a, b, out tensor.Tensor) // SumSquaresScaleFuncOut64 computes the sum of squares differences between tensor values, // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. -func SumSquaresScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { - scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func SumSquaresScaleFuncOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { + scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { return a - b }) @@ -43,9 +43,9 @@ func SumSquaresScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor. // SumSquaresFuncOut64 computes the sum of squares differences between tensor values, // and returns the Float64 output values for use in subsequent computations. -func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { +func SumSquaresFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) ss := ss64.Float1D(i) @@ -63,15 +63,15 @@ func SumSquaresFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { // SumSquaresFunc computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. -func SumSquaresFunc(a, b, out *tensor.Indexed) { +func SumSquaresFunc(a, b, out tensor.Tensor) { SumSquaresFuncOut64(a, b, out) } // EuclideanFunc computes the Euclidean square root of the sum of squares // differences between tensor values, aka the L2 Norm. -func EuclideanFunc(a, b, out *tensor.Indexed) { +func EuclideanFunc(a, b, out tensor.Tensor) { scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) ss := ss64.Float1D(i) @@ -89,8 +89,8 @@ func EuclideanFunc(a, b, out *tensor.Indexed) { // AbsFunc computes the sum of the absolute value of differences between the // tensor values, aka the L1 Norm. // See [MetricFunc] for general information. -func AbsFunc(a, b, out *tensor.Indexed) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func AbsFunc(a, b, out tensor.Tensor) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + math.Abs(a-b) }) @@ -100,8 +100,8 @@ func AbsFunc(a, b, out *tensor.Indexed) { // HammingFunc computes the sum of 1s for every element that is different, // i.e., "city block" distance. // See [MetricFunc] for general information. -func HammingFunc(a, b, out *tensor.Indexed) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func HammingFunc(a, b, out tensor.Tensor) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { if a != b { agg += 1 @@ -114,8 +114,8 @@ func HammingFunc(a, b, out *tensor.Indexed) { // SumSquaresBinTolScaleFuncOut64 computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. -func SumSquaresBinTolScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { - scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func SumSquaresBinTolScaleFuncOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { + scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { d := a - b if math.Abs(d) < 0.5 { @@ -130,9 +130,9 @@ func SumSquaresBinTolScaleFuncOut64(a, b, out *tensor.Indexed) (scale64, ss64 *t // EuclideanBinTolFunc computes the Euclidean square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. -func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { +func EuclideanBinTolFunc(a, b, out tensor.Tensor) { scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) ss := ss64.Float1D(i) @@ -149,9 +149,9 @@ func EuclideanBinTolFunc(a, b, out *tensor.Indexed) { // SumSquaresBinTolFunc computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { +func SumSquaresBinTolFunc(a, b, out tensor.Tensor) { scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) ss := ss64.Float1D(i) @@ -173,8 +173,8 @@ func SumSquaresBinTolFunc(a, b, out *tensor.Indexed) { // using Kullback-Leibler (KL) divergence. It is computed as: // a * log(a/b) + (1-a) * log(1-a/1-b). // See [MetricFunc] for general information. -func CrossEntropyFunc(a, b, out *tensor.Indexed) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func CrossEntropyFunc(a, b, out tensor.Tensor) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { b = math32.Clamp(b, 0.000001, 0.999999) if a >= 1.0 { @@ -191,8 +191,8 @@ func CrossEntropyFunc(a, b, out *tensor.Indexed) { // InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. // See [MetricFunc] for general information. -func InnerProductFunc(a, b, out *tensor.Indexed) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func InnerProductFunc(a, b, out tensor.Tensor) { + stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + a*b }) @@ -202,15 +202,15 @@ func InnerProductFunc(a, b, out *tensor.Indexed) { // CovarianceFunc computes the co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. -func CovarianceFunc(a, b, out *tensor.Indexed) { +func CovarianceFunc(a, b, out tensor.Tensor) { amean, acount := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - cov64 := stats.VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { + cov64 := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, am, bm, agg float64) float64 { return agg + (a-am)*(b-bm) }) }, a, b, amean, bmean, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { c := acount.Float1D(i) if c == 0 { @@ -228,10 +228,10 @@ func CovarianceFunc(a, b, out *tensor.Indexed) { // (i.e., the standardized covariance). // Equivalent to the cosine of mean-normalized vectors. // Returns the Float64 output values for subsequent use. -func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { +func CorrelationFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { amean, _ := stats.MeanFuncOut64(a, out) bmean, _ := stats.MeanFuncOut64(b, out) - ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { + ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { ad := a - am bd := b - bm @@ -242,7 +242,7 @@ func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { }) }, a, b, amean, bmean, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { ss := ss64.Float1D(i) vp := math.Sqrt(avar64.Float1D(i) * bvar64.Float1D(i)) @@ -261,7 +261,7 @@ func CorrelationFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized [CovarianceFunc]). // Equivalent to the [CosineFunc] of mean-normalized vectors. -func CorrelationFunc(a, b, out *tensor.Indexed) { +func CorrelationFunc(a, b, out tensor.Tensor) { CorrelationFuncOut64(a, b, out) } @@ -273,9 +273,9 @@ func CorrelationFunc(a, b, out *tensor.Indexed) { // Equivalent to the [CosineFunc] of mean-normalized vectors. // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCorrelationFunc(a, b, out *tensor.Indexed) { +func InvCorrelationFunc(a, b, out tensor.Tensor) { cor64 := CorrelationFuncOut64(a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { cor := cor64.Float1D(i) out.SetFloat1D(1-cor, i) @@ -285,8 +285,8 @@ func InvCorrelationFunc(a, b, out *tensor.Indexed) { // CosineFuncOut64 computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized [InnerProductFunc]: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc]. -func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { - ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func CosineFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { + ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { ss += a * b avar += a * a @@ -294,7 +294,7 @@ func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { return ss, avar, bvar }) }, a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { ss := ss64.Float1D(i) vp := math.Sqrt(avar64.Float1D(i) * bvar64.Float1D(i)) @@ -310,7 +310,7 @@ func CosineFuncOut64(a, b, out *tensor.Indexed) *tensor.Indexed { // CosineFunc computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc] -func CosineFunc(a, b, out *tensor.Indexed) { +func CosineFunc(a, b, out tensor.Tensor) { CosineFuncOut64(a, b, out) } @@ -319,9 +319,9 @@ func CosineFunc(a, b, out *tensor.Indexed) { // inner product / sqrt(ssA * ssB). // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCosineFunc(a, b, out *tensor.Indexed) { +func InvCosineFunc(a, b, out tensor.Tensor) { cos64 := CosineFuncOut64(a, b, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { cos := cos64.Float1D(i) out.SetFloat1D(1-cos, i) diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index d88f946d19..a408c89c28 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -9,6 +9,7 @@ import ( "cogentcore.org/core/math32/vecint" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/matrix" "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/tmath" "gonum.org/v1/gonum/mat" @@ -28,21 +29,21 @@ func init() { // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied // to the upper triangular region, for maximum efficiency. -func Matrix(funcName string, in, out *tensor.Indexed) { +func Matrix(funcName string, in, out tensor.Tensor) { rows, cells := in.RowCellSize() if rows == 0 || cells == 0 { return } - out.Tensor.SetShape(rows, rows) + out.SetShapeInts(rows, rows) mout := tensor.NewFloat64Scalar(0.0) coords := TriangularLIndicies(rows) nc := len(coords) // note: flops estimating 3 per item on average -- different for different metrics. - tensor.VectorizeThreaded(cells*3, func(tsr ...*tensor.Indexed) int { return nc }, - func(idx int, tsr ...*tensor.Indexed) { + tensor.VectorizeThreaded(cells*3, func(tsr ...tensor.Tensor) int { return nc }, + func(idx int, tsr ...tensor.Tensor) { c := coords[idx] - sa := tsr[0].Cells1D(c.X) - sb := tsr[0].Cells1D(c.Y) + sa := tensor.Cells1D(tsr[0], c.X) + sb := tensor.Cells1D(tsr[0], c.Y) tensor.Call(funcName, sa, sb, mout) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, in, out) @@ -50,7 +51,7 @@ func Matrix(funcName string, in, out *tensor.Indexed) { if c.X == c.Y { // exclude diag continue } - out.Tensor.SetFloat(out.Tensor.Float(c.X, c.Y), c.Y, c.X) + out.SetFloat(out.Float(c.X, c.Y), c.Y, c.X) } } @@ -63,7 +64,7 @@ func Matrix(funcName string, in, out *tensor.Indexed) { // The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. -func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { +func CrossMatrix(funcName string, a, b, out tensor.Tensor) { arows, acells := a.RowCellSize() if arows == 0 || acells == 0 { return @@ -72,17 +73,17 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { if brows == 0 || bcells == 0 { return } - out.Tensor.SetShape(arows, brows) + out.SetShapeInts(arows, brows) mout := tensor.NewFloat64Scalar(0.0) // note: flops estimating 3 per item on average -- different for different metrics. flops := min(acells, bcells) * 3 nc := arows * brows - tensor.VectorizeThreaded(flops, func(tsr ...*tensor.Indexed) int { return nc }, - func(idx int, tsr ...*tensor.Indexed) { + tensor.VectorizeThreaded(flops, func(tsr ...tensor.Tensor) int { return nc }, + func(idx int, tsr ...tensor.Tensor) { ar := idx / brows br := idx % brows - sa := tsr[0].Cells1D(ar) - sb := tsr[1].Cells1D(br) + sa := tensor.Cells1D(tsr[0], ar) + sb := tensor.Cells1D(tsr[1], br) tensor.Call(funcName, sa, sb, mout) tsr[2].SetFloat(mout.Float1D(0), ar, br) }, a, b, out) @@ -101,46 +102,46 @@ func CrossMatrix(funcName string, a, b, out *tensor.Indexed) { // which is typical in neural network models, and use // Correlation if they are on very different scales, because it effectively rescales). // The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. -func CovarianceMatrix(funcName string, in, out *tensor.Indexed) { +func CovarianceMatrix(funcName string, in, out tensor.Tensor) { rows, cells := in.RowCellSize() if rows == 0 || cells == 0 { return } - flatsz := []int{in.Tensor.DimSize(0), cells} - flatvw := in.Tensor.View() - flatvw.SetShape(flatsz...) - flatix := tensor.NewIndexed(flatvw) - flatix.Indexes = in.Indexes + flatsz := []int{in.DimSize(0), cells} + flatvw := in.View() + flatvw.SetShapeInts(flatsz...) + // flatix := tensor.NewIndexed(flatvw) + // flatix.Indexes = in.Indexes mout := tensor.NewFloat64Scalar(0.0) - out.Tensor.SetShape(cells, cells) - av := tensor.NewFloat64Indexed(rows) - bv := tensor.NewFloat64Indexed(rows) + out.SetShapeInts(cells, cells) + av := tensor.NewFloat64(rows) + bv := tensor.NewFloat64(rows) curCoords := vecint.Vector2i{-1, -1} coords := TriangularLIndicies(cells) nc := len(coords) // note: flops estimating 3 per item on average -- different for different metrics. - tensor.VectorizeThreaded(rows*3, func(tsr ...*tensor.Indexed) int { return nc }, - func(idx int, tsr ...*tensor.Indexed) { + tensor.VectorizeThreaded(rows*3, func(tsr ...tensor.Tensor) int { return nc }, + func(idx int, tsr ...tensor.Tensor) { c := coords[idx] if c.X != curCoords.X { - tensor.Slice(tsr[0], av, tensor.Range{}, tensor.Range{Start: c.X, End: c.X + 1}) + // tensor.Slice(tsr[0], av, tensor.Range{}, tensor.Range{Start: c.X, End: c.X + 1}) curCoords.X = c.X } if c.Y != curCoords.Y { - tensor.Slice(tsr[0], bv, tensor.Range{}, tensor.Range{Start: c.Y, End: c.Y + 1}) + // tensor.Slice(tsr[0], bv, tensor.Range{}, tensor.Range{Start: c.Y, End: c.Y + 1}) curCoords.Y = c.Y } tensor.Call(funcName, av, bv, mout) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) - }, flatix, out) + }, flatvw, out) for _, c := range coords { // copy to upper if c.X == c.Y { // exclude diag continue } - out.Tensor.SetFloat(out.Tensor.Float(c.X, c.Y), c.Y, c.X) + out.SetFloat(out.Float(c.X, c.Y), c.Y, c.X) } } @@ -151,13 +152,11 @@ func CovarianceMatrix(funcName string, in, out *tensor.Indexed) { // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *lowest* to *highest*. // Note that PCA produces results in the *opposite* order of [SVD]. -func PCA(covar, eigenvecs, vals *tensor.Indexed) { - n := covar.Tensor.DimSize(0) - cv := tensor.AsFloat64(covar.Tensor) - eigenvecs.Tensor.SetShape(n, n) - eigenvecs.Sequential() - vals.Tensor.SetShape(n) - vals.Sequential() +func PCA(covar, eigenvecs, vals tensor.Tensor) { + n := covar.DimSize(0) + cv := tensor.AsFloat64Tensor(covar) + eigenvecs.SetShapeInts(n, n) + vals.SetShapeInts(n) var eig mat.EigenSym ok := eig.Factorize(cv, true) if !ok { @@ -166,11 +165,11 @@ func PCA(covar, eigenvecs, vals *tensor.Indexed) { } var ev mat.Dense eig.VectorsTo(&ev) - tensor.CopyDense(eigenvecs.Tensor, &ev) - fv := tensor.AsFloat64(vals.Tensor) + matrix.CopyDense(eigenvecs, &ev) + fv := tensor.AsFloat64Tensor(vals) eig.Values(fv.Values) - if fv != vals.Tensor { - vals.Tensor.CopyFrom(fv) + if fv != vals { + vals.CopyFrom(fv) } } @@ -181,13 +180,11 @@ func PCA(covar, eigenvecs, vals *tensor.Indexed) { // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *highest* to *lowest*. // Note that SVD produces results in the *opposite* order of [PCA]. -func SVD(covar, eigenvecs, vals *tensor.Indexed) { - n := covar.Tensor.DimSize(0) - cv := tensor.AsFloat64(covar.Tensor) - eigenvecs.Tensor.SetShape(n, n) - eigenvecs.Sequential() - vals.Tensor.SetShape(n) - vals.Sequential() +func SVD(covar, eigenvecs, vals tensor.Tensor) { + n := covar.DimSize(0) + cv := tensor.AsFloat64Tensor(covar) + eigenvecs.SetShapeInts(n, n) + vals.SetShapeInts(n) var eig mat.SVD ok := eig.Factorize(cv, mat.SVDFull) // todo: benchmark different versions if !ok { @@ -196,11 +193,11 @@ func SVD(covar, eigenvecs, vals *tensor.Indexed) { } var ev mat.Dense eig.UTo(&ev) - tensor.CopyDense(eigenvecs.Tensor, &ev) - fv := tensor.AsFloat64(vals.Tensor) + matrix.CopyDense(eigenvecs, &ev) + fv := tensor.AsFloat64Tensor(vals) eig.Values(fv.Values) - if fv != vals.Tensor { - vals.Tensor.CopyFrom(fv) + if fv != vals { + vals.CopyFrom(fv) } } @@ -211,21 +208,21 @@ func SVD(covar, eigenvecs, vals *tensor.Indexed) { // and each row of cells is projected through the matrix column, producing a // 1D output with the number of rows. Otherwise a single number is produced. // This is typically done with results from SVD or PCA. -func ProjectOnMatrixColumn(mtx, vec, colindex, out *tensor.Indexed) { +func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) { ci := int(colindex.Float1D(0)) - col := tensor.NewFloat64Indexed() - tensor.Slice(mtx, col, tensor.Range{}, tensor.Range{Start: ci, End: ci + 1}) - // fmt.Println(mtx.Tensor.String(), col.Tensor.String()) + _ = ci + col := tensor.NewFloat64() + // tensor.Slice(mtx, col, tensor.Range{}, tensor.Range{Start: ci, End: ci + 1}) + // fmt.Println(mtx.String(), col.String()) rows, cells := vec.RowCellSize() - mout := tensor.NewFloat64Indexed() + mout := tensor.NewFloat64() if rows > 0 && cells > 0 { msum := tensor.NewFloat64Scalar(0) - out.Tensor.SetShape(rows) - out.Sequential() + out.SetShapeInts(rows) for i := range rows { - tmath.Mul(vec.Cells1D(i), col, mout) + tmath.Mul(tensor.Cells1D(vec, i), col, mout) stats.SumFunc(mout, msum) - // fmt.Println(vec.Cells1D(i).Tensor.String(), mout.Tensor.String(), msum.Tensor.String()) + // fmt.Println(tensor.Cells1D(vec, i).String(), mout.String(), msum.String()) out.SetFloat1D(msum.Float1D(0), i) } } else { diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 0ef95119cd..0922b5855a 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -72,8 +72,7 @@ func TestFuncs(t *testing.T) { } func TestMatrix(t *testing.T) { - var simres = `Tensor: [12, 12] -[0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 + var simres = `[12, 12] [0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 [1]: 3.4641016151377544 0 9.38083151964686 8.717797887081348 9.273618495495704 8.831760866327848 5.830951894845301 4.69041575982343 8.717797887081348 7.937253933193772 6.324555320336759 5.291502622129181 [2]: 8.831760866327848 9.38083151964686 0 3.4641016151377544 4.242640687119285 5.0990195135927845 9.38083151964686 9.899494936611665 4.47213595499958 5.744562646538029 9.38083151964686 9.899494936611665 [3]: 9.273618495495704 8.717797887081348 3.4641016151377544 0 5.477225575051661 3.7416573867739413 9.797958971132712 9.273618495495704 5.656854249492381 4.58257569495584 9.797958971132712 9.273618495495704 @@ -90,10 +89,10 @@ func TestMatrix(t *testing.T) { err := dt.OpenCSV("../cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) in := dt.Column("Input") - out := tensor.NewFloat64Indexed() + out := tensor.NewFloat64() Matrix(Euclidean.FuncName(), in, out) // fmt.Println(out.Tensor) - assert.Equal(t, simres, out.Tensor.String()) + assert.Equal(t, simres, out.String()) } func TestPCAIris(t *testing.T) { @@ -108,26 +107,26 @@ func TestPCAIris(t *testing.T) { t.Error(err) } data := dt.Column("data") - covar := tensor.NewFloat64Indexed() + covar := tensor.NewFloat64() CovarianceMatrix(Correlation.FuncName(), data, covar) - // fmt.Printf("correl: %s\n", covar.Tensor.String()) + // fmt.Printf("correl: %s\n", covar.String()) - vecs := tensor.NewFloat64Indexed() - vals := tensor.NewFloat64Indexed() + vecs := tensor.NewFloat64() + vals := tensor.NewFloat64() PCA(covar, vecs, vals) // fmt.Printf("correl vec: %v\n", vecs) // fmt.Printf("correl val: %v\n", vals) errtol := 1.0e-9 corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} - for i, v := range vals.Tensor.(*tensor.Float64).Values { + for i, v := range vals.Values { assert.InDelta(t, corvals[i], v, errtol) } colidx := tensor.NewFloat64Scalar(3) // strongest at end - prjns := tensor.NewFloat64Indexed() + prjns := tensor.NewFloat64() ProjectOnMatrixColumn(vecs, data, colidx, prjns) - // tensor.SaveCSV(prjns.Tensor, "testdata/pca_projection.csv", tensor.Comma) + // tensor.SaveCSV(prjns, "testdata/pca_projection.csv", tensor.Comma) trgprjns := []float64{ 2.6692308782935146, 2.696434011868953, @@ -140,8 +139,7 @@ func TestPCAIris(t *testing.T) { 2.4615836931965167, 2.6716628159090594, } - pj64 := tensor.AsFloat64(prjns.Tensor) - for i, v := range pj64.Values[:10] { + for i, v := range prjns.Values[:10] { assert.InDelta(t, trgprjns[i], v, errtol) } @@ -151,13 +149,13 @@ func TestPCAIris(t *testing.T) { SVD(covar, vecs, vals) // fmt.Printf("correl vec: %v\n", vecs) // fmt.Printf("correl val: %v\n", vals) - for i, v := range vals.Tensor.(*tensor.Float64).Values { + for i, v := range vals.Values { assert.InDelta(t, corvals[3-i], v, errtol) // opposite order } colidx.SetFloat1D(0, 0) // strongest at start ProjectOnMatrixColumn(vecs, data, colidx, prjns) - // tensor.SaveCSV(prjns.Tensor, "testdata/svd_projection.csv", tensor.Comma) + // tensor.SaveCSV(prjns, "testdata/svd_projection.csv", tensor.Comma) trgprjns = []float64{ -2.6692308782935172, -2.696434011868955, @@ -170,8 +168,7 @@ func TestPCAIris(t *testing.T) { -2.4615836931965185, -2.671662815909061, } - pj64 = tensor.AsFloat64(prjns.Tensor) - for i, v := range pj64.Values[:10] { + for i, v := range prjns.Values[:10] { assert.InDelta(t, trgprjns[i], v, errtol) } } diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 57691415e8..81a5969fcd 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -28,13 +28,13 @@ func init() { // Metric calls a standard Metrics enum function on given tensors. // Output results are in the out tensor. -func Metric(metric Metrics, a, b, out *tensor.Indexed) { +func Metric(metric Metrics, a, b, out tensor.Tensor) { tensor.Call(metric.FuncName(), a, b, out) } // MetricOut calls a standard Metrics enum function on given tensors, // returning output as a newly created tensor. -func MetricOut(metric Metrics, a, b *tensor.Indexed) *tensor.Indexed { +func MetricOut(metric Metrics, a, b tensor.Tensor) tensor.Tensor { return tensor.CallOut(metric.FuncName(), a, b) } diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 365fecd71c..23ecabae31 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -22,14 +22,14 @@ func init() { // Note: this does _not_ use any existing Indexes for the probe, // but does for the vocab, and the returned index is the logical index // into any existing Indexes. -func ClosestRow(funcName string, probe, vocab, out *tensor.Indexed) { - rows, _ := vocab.Tensor.RowCellSize() +func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) { + rows, _ := vocab.RowCellSize() mi := -1 mout := tensor.NewFloat64Scalar(0.0) mind := math.MaxFloat64 - pr1d := tensor.NewIndexed(tensor.New1DViewOf(probe.Tensor)) + pr1d := tensor.New1DViewOf(probe) for ri := range rows { - sub := vocab.Cells1D(ri) + sub := tensor.Cells1D(vocab, ri) tensor.Call(funcName, pr1d, sub, mout) d := mout.Float1D(0) if d < mind { @@ -37,8 +37,7 @@ func ClosestRow(funcName string, probe, vocab, out *tensor.Indexed) { mind = d } } - out.Tensor.SetShape(2) - out.Sequential() + out.SetShapeInts(2) out.SetFloat1D(float64(mi), 0) out.SetFloat1D(mind, 1) } diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index b0746fff67..90c316dbb5 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -14,17 +14,17 @@ import ( // Vectorize3Out64 is a version of the [tensor.Vectorize] function // for metrics, which makes three Float64 output tensors for aggregating // and computing values, returning them for final computation. -func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2, out3 *tensor.Indexed) { +func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (out1, out2, out3 tensor.Tensor) { n := nfunc(tsr...) if n <= 0 { return nil, nil, nil } nt := len(tsr) out := tsr[nt-1] - osz := out.Tensor.ShapeInts() - out1 = tensor.NewFloat64Indexed(osz...) - out2 = tensor.NewFloat64Indexed(osz...) - out3 = tensor.NewFloat64Indexed(osz...) + osz := out.ShapeInts() + out1 = tensor.NewFloat64(osz...) + out2 = tensor.NewFloat64(osz...) + out3 = tensor.NewFloat64(osz...) tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2, out3) for idx := range n { @@ -36,30 +36,30 @@ func Vectorize3Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t // OutShape returns the output shape based on given input // shape ints, with outer row dim = 1. func OutShape(ish ...int) []int { - ish[0] = 1 - return ish + osh := slices.Clone(ish) + osh[0] = 1 + return osh } // NFunc is the nfun for metrics functions, returning the min number of rows across the // two input tensors, and initializing the _last_ one to hold the output // with the first, row dimension set to 1. -func NFunc(tsr ...*tensor.Indexed) int { +func NFunc(tsr ...tensor.Tensor) int { nt := len(tsr) if nt < 3 { return 0 } a, b, out := tsr[0], tsr[1], tsr[nt-1] - na, nb := a.NumRows(), b.NumRows() - out.Tensor.SetShape(OutShape(a.Tensor.ShapeInts()...)) - out.Indexes = nil + na, nb := a.DimSize(0), b.DimSize(0) + out.SetShapeInts(OutShape(a.ShapeInts()...)...) return min(na, nb) } // VecFunc is a helper function for metrics functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // It also skips over NaN missing values. -func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg float64) float64) { - nsub := out.Tensor.Len() +func VecFunc(idx int, a, b, out tensor.Tensor, ini float64, fun func(a, b, agg float64) float64) { + nsub := out.Len() for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) @@ -80,8 +80,8 @@ func VecFunc(idx int, a, b, out *tensor.Indexed, ini float64, fun func(a, b, agg // the Cell subspace per row and initializing the aggregation values for first index. // This version does sum-of-squares integration over 2 output vectors, // It also skips over NaN missing values. -func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(a, b float64) float64) { - nsub := out2.Tensor.Len() +func VecSSFunc(idx int, a, b, out1, out2 tensor.Tensor, ini1, ini2 float64, fun func(a, b float64) float64) { + nsub := out2.Len() for i := range nsub { if idx == 0 { out1.SetFloat1D(ini1, i) @@ -116,8 +116,8 @@ func VecSSFunc(idx int, a, b, out1, out2 *tensor.Indexed, ini1, ini2 float64, fu // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input vectors, the second input being the output of another stat // e.g., the mean. It also skips over NaN missing values. -func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun func(a, b, a2, b2, agg float64) float64) { - nsub := out.Tensor.Len() +func Vec2inFunc(idx int, a, b, a2, b2, out tensor.Tensor, ini float64, fun func(a, b, a2, b2, agg float64) float64) { + nsub := out.Len() for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) @@ -140,8 +140,8 @@ func Vec2inFunc(idx int, a, b, a2, b2, out *tensor.Indexed, ini float64, fun fun // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 input, 3 output vectors. The second input being the output of another stat // e.g., the mean. It also skips over NaN missing values. -func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, a2, b2, out1, out2, out3 float64) (float64, float64, float64)) { - nsub := out1.Tensor.Len() +func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 tensor.Tensor, ini float64, fun func(a, b, a2, b2, out1, out2, out3 float64) (float64, float64, float64)) { + nsub := out1.Len() for i := range nsub { if idx == 0 { out1.SetFloat1D(ini, i) @@ -171,8 +171,8 @@ func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 *tensor.Indexed, ini // Vec3outFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // This version has 3 output vectors. It also skips over NaN missing values. -func Vec3outFunc(idx int, a, b, out1, out2, out3 *tensor.Indexed, ini float64, fun func(a, b, out1, out2, out3 float64) (float64, float64, float64)) { - nsub := out1.Tensor.Len() +func Vec3outFunc(idx int, a, b, out1, out2, out3 tensor.Tensor, ini float64, fun func(a, b, out1, out2, out3 float64) (float64, float64, float64)) { + nsub := out1.Len() for i := range nsub { if idx == 0 { out1.SetFloat1D(ini, i) diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go index 3776d5c823..83175da21d 100644 --- a/tensor/stats/stats/describe.go +++ b/tensor/stats/stats/describe.go @@ -23,17 +23,17 @@ var DescriptiveStats = []Stats{Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3} // This is an easy way to provide a comprehensive description of data. // The [DescriptiveStats] list is: [Count], [Mean], [Std], [Sem], // [Min], [Max], [Q1], [Median], [Q3] -func Describe(dir *datafs.Data, tsrs ...*tensor.Indexed) { +func Describe(dir *datafs.Data, tsrs ...tensor.Tensor) { dd, err := dir.RecycleDir("Describe") if errors.Log(err) != nil { return } for i, tsr := range tsrs { - nr := tsr.NumRows() + nr := tsr.DimSize(0) if nr == 0 { continue } - nm := tsr.Tensor.Metadata().Name() + nm := tsr.Metadata().Name() if nm == "" { nm = strconv.Itoa(i) } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index c21da113e9..8f8c4d81bb 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -25,12 +25,12 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type StatsFunc func(in, out *tensor.Indexed) +type StatsFunc func(in, out tensor.Tensor) // CountFuncOut64 computes the count of non-NaN tensor values, // and returns the Float64 output values for subsequent use. -func CountFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { - return VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func CountFuncOut64(in, out tensor.Tensor) tensor.Tensor { + return VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + 1 }) @@ -39,14 +39,14 @@ func CountFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { // CountFunc computes the count of non-NaN tensor values. // See [StatsFunc] for general information. -func CountFunc(in, out *tensor.Indexed) { +func CountFunc(in, out tensor.Tensor) { CountFuncOut64(in, out) } // SumFuncOut64 computes the sum of tensor values, // and returns the Float64 output values for subsequent use. -func SumFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { - return VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func SumFuncOut64(in, out tensor.Tensor) tensor.Tensor { + return VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + val }) @@ -55,15 +55,15 @@ func SumFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { // SumFunc computes the sum of tensor values. // See [StatsFunc] for general information. -func SumFunc(in, out *tensor.Indexed) { +func SumFunc(in, out tensor.Tensor) { SumFuncOut64(in, out) } // SumAbsFunc computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. -func SumAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func SumAbsFunc(in, out tensor.Tensor) { + VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) }) @@ -72,8 +72,8 @@ func SumAbsFunc(in, out *tensor.Indexed) { // ProdFunc computes the product of tensor values. // See [StatsFunc] for general information. -func ProdFunc(in, out *tensor.Indexed) { - VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func ProdFunc(in, out tensor.Tensor) { + VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { return agg * val }) @@ -82,8 +82,8 @@ func ProdFunc(in, out *tensor.Indexed) { // MinFunc computes the min of tensor values. // See [StatsFunc] for general information. -func MinFunc(in, out *tensor.Indexed) { - VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func MinFunc(in, out tensor.Tensor) { + VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, val) }) @@ -92,8 +92,8 @@ func MinFunc(in, out *tensor.Indexed) { // MaxFunc computes the max of tensor values. // See [StatsFunc] for general information. -func MaxFunc(in, out *tensor.Indexed) { - VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func MaxFunc(in, out tensor.Tensor) { + VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, val) }) @@ -102,8 +102,8 @@ func MaxFunc(in, out *tensor.Indexed) { // MinAbsFunc computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MinAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func MinAbsFunc(in, out tensor.Tensor) { + VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, math.Abs(val)) }) @@ -112,8 +112,8 @@ func MinAbsFunc(in, out *tensor.Indexed) { // MaxAbsFunc computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MaxAbsFunc(in, out *tensor.Indexed) { - VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func MaxAbsFunc(in, out tensor.Tensor) { + VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, math.Abs(val)) }) @@ -122,11 +122,11 @@ func MaxAbsFunc(in, out *tensor.Indexed) { // MeanFuncOut64 computes the mean of tensor values, // and returns the Float64 output values for subsequent use. -func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { +func MeanFuncOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor) { sum64 := SumFuncOut64(in, out) - count := tensor.NewIndexed(out.Tensor.Clone()) + count := tensor.NewIndexed(out.Clone()) count64 = CountFuncOut64(in, count) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { c := count64.Float1D(i) if c == 0 { @@ -140,15 +140,15 @@ func MeanFuncOut64(in, out *tensor.Indexed) (mean64, count64 *tensor.Indexed) { // MeanFunc computes the mean of tensor values. // See [StatsFunc] for general information. -func MeanFunc(in, out *tensor.Indexed) { +func MeanFunc(in, out tensor.Tensor) { MeanFuncOut64(in, out) } // SumSqDevFuncOut64 computes the sum of squared mean deviates of tensor values, // and returns the Float64 output values for subsequent use. -func SumSqDevFuncOut64(in, out *tensor.Indexed) (ssd64, mean64, count64 *tensor.Indexed) { +func SumSqDevFuncOut64(in, out tensor.Tensor) (ssd64, mean64, count64 tensor.Tensor) { mean64, count64 = MeanFuncOut64(in, out) - ssd64 = VectorizeOut64(NFunc, func(idx int, tsr ...*tensor.Indexed) { + ssd64 = VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { dv := val1 - val2 return agg + dv*dv @@ -159,9 +159,9 @@ func SumSqDevFuncOut64(in, out *tensor.Indexed) (ssd64, mean64, count64 *tensor. // VarFuncOut64 computes the sample variance of tensor values, // and returns the Float64 output values for subsequent use. -func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Indexed) { +func VarFuncOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor) { var64, mean64, count64 = SumSqDevFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { c := count64.Float1D(i) if c < 2 { @@ -177,15 +177,15 @@ func VarFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Index // VarFunc computes the sample variance of tensor values. // Squared deviations from mean, divided by n-1. See also [VarPopFunc]. // See [StatsFunc] for general information. -func VarFunc(in, out *tensor.Indexed) { +func VarFunc(in, out tensor.Tensor) { VarFuncOut64(in, out) } // StdFuncOut64 computes the sample standard deviation of tensor values. // and returns the Float64 output values for subsequent use. -func StdFuncOut64(in, out *tensor.Indexed) (std64, mean64, count64 *tensor.Indexed) { +func StdFuncOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor) { std64, mean64, count64 = VarFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { std := math.Sqrt(std64.Float1D(i)) std64.SetFloat1D(std, i) @@ -197,16 +197,16 @@ func StdFuncOut64(in, out *tensor.Indexed) (std64, mean64, count64 *tensor.Index // StdFunc computes the sample standard deviation of tensor values. // Sqrt of variance from [VarFunc]. See also [StdPopFunc]. // See [StatsFunc] for general information. -func StdFunc(in, out *tensor.Indexed) { +func StdFunc(in, out tensor.Tensor) { StdFuncOut64(in, out) } // SemFunc computes the sample standard error of the mean of tensor values. // Standard deviation [StdFunc] / sqrt(n). See also [SemPopFunc]. // See [StatsFunc] for general information. -func SemFunc(in, out *tensor.Indexed) { +func SemFunc(in, out tensor.Tensor) { var64, _, count64 := VarFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { c := count64.Float1D(i) if c < 2 { @@ -219,9 +219,9 @@ func SemFunc(in, out *tensor.Indexed) { // VarPopFuncOut64 computes the population variance of tensor values. // and returns the Float64 output values for subsequent use. -func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.Indexed) { +func VarPopFuncOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor) { var64, mean64, count64 = SumSqDevFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { c := count64.Float1D(i) if c == 0 { @@ -236,16 +236,16 @@ func VarPopFuncOut64(in, out *tensor.Indexed) (var64, mean64, count64 *tensor.In // VarPopFunc computes the population variance of tensor values. // Squared deviations from mean, divided by n. See also [VarFunc]. // See [StatsFunc] for general information. -func VarPopFunc(in, out *tensor.Indexed) { +func VarPopFunc(in, out tensor.Tensor) { VarPopFuncOut64(in, out) } // StdPopFunc computes the population standard deviation of tensor values. // Sqrt of variance from [VarPopFunc]. See also [StdFunc]. // See [StatsFunc] for general information. -func StdPopFunc(in, out *tensor.Indexed) { +func StdPopFunc(in, out tensor.Tensor) { var64, _, _ := VarPopFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { out.SetFloat1D(math.Sqrt(var64.Float1D(i)), i) } @@ -254,9 +254,9 @@ func StdPopFunc(in, out *tensor.Indexed) { // SemPopFunc computes the population standard error of the mean of tensor values. // Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. // See [StatsFunc] for general information. -func SemPopFunc(in, out *tensor.Indexed) { +func SemPopFunc(in, out tensor.Tensor) { var64, _, count64 := VarPopFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { c := count64.Float1D(i) if c < 2 { @@ -270,8 +270,8 @@ func SemPopFunc(in, out *tensor.Indexed) { // SumSqScaleFuncOut64 is a helper for sum-of-squares, returning scale and ss // factors aggregated separately for better numerical stability, per BLAS. // Returns the Float64 output values for subsequent use. -func SumSqScaleFuncOut64(in, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed) { - scale64, ss64 = Vectorize2Out64(NFunc, func(idx int, tsr ...*tensor.Indexed) { +func SumSqScaleFuncOut64(in, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { + scale64, ss64 = Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { if val == 0 { return scale, ss @@ -291,9 +291,9 @@ func SumSqScaleFuncOut64(in, out *tensor.Indexed) (scale64, ss64 *tensor.Indexed // SumSqFuncOut64 computes the sum of squares of tensor values, // and returns the Float64 output values for subsequent use. -func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { +func SumSqFuncOut64(in, out tensor.Tensor) tensor.Tensor { scale64, ss64 := SumSqScaleFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) ss := ss64.Float1D(i) @@ -311,16 +311,16 @@ func SumSqFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { // SumSqFunc computes the sum of squares of tensor values, // See [StatsFunc] for general information. -func SumSqFunc(in, out *tensor.Indexed) { +func SumSqFunc(in, out tensor.Tensor) { SumSqFuncOut64(in, out) } // L2NormFuncOut64 computes the square root of the sum of squares of tensor values, // known as the L2 norm, and returns the Float64 output values for // use in subsequent computations. -func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { +func L2NormFuncOut64(in, out tensor.Tensor) tensor.Tensor { scale64, ss64 := SumSqScaleFuncOut64(in, out) - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) ss := ss64.Float1D(i) @@ -339,6 +339,6 @@ func L2NormFuncOut64(in, out *tensor.Indexed) *tensor.Indexed { // L2NormFunc computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsFunc] for general information. -func L2NormFunc(in, out *tensor.Indexed) { +func L2NormFunc(in, out tensor.Tensor) { L2NormFuncOut64(in, out) } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 066b09efed..ebd1b416ba 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -31,7 +31,7 @@ import ( // rows, indirected through any existing indexes on the inputs, so that // the results can be used directly as Indexes into the corresponding tensor data. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { +func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) { gd, err := dir.RecycleDir("Groups") if errors.Log(err) != nil { return @@ -45,19 +45,19 @@ func Groups(dir *datafs.Data, tsrs ...*tensor.Indexed) { } for i, tsr := range tsrs { - nr := tsr.NumRows() + nr := tsr.DimSize(0) if nr == 0 { continue } - nm := tsr.Tensor.Metadata().Name() + nm := tsr.Metadata().Name() if nm == "" { nm = strconv.Itoa(i) } td, _ := gd.Mkdir(nm) - srt := tsr.CloneIndexes() + srt := tensor.AsIndexed(tsr).CloneIndexes() srt.SortStable(tensor.Ascending) start := 0 - if tsr.Tensor.IsString() { + if tsr.IsString() { lastVal := srt.StringRow(0) for r := range nr { v := srt.StringRow(r) @@ -99,12 +99,12 @@ func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { // into an "All/All" tensor in the given [datafs], which can then // be used with [GroupStats] to generate summary statistics across // all the data. See [Groups] for more general documentation. -func GroupAll(dir *datafs.Data, tsrs ...*tensor.Indexed) { +func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) { gd, err := dir.RecycleDir("Groups") if errors.Log(err) != nil { return } - tsr := tsrs[0] + tsr := tensor.AsIndexed(tsrs[0]) nr := tsr.NumRows() if nr == 0 { return @@ -112,7 +112,7 @@ func GroupAll(dir *datafs.Data, tsrs ...*tensor.Indexed) { td, _ := gd.Mkdir("All") it := datafs.NewValue[int](td, "All", nr) for j := range nr { - it.SetIntRow(tsr.Index(j), j) // key to indirect through any existing indexes + it.SetIntRow(tsr.RowIndex(j), j) // key to indirect through any existing indexes } } @@ -128,7 +128,7 @@ func GroupAll(dir *datafs.Data, tsrs ...*tensor.Indexed) { // a String tensor with the unique values of each source [Groups] tensor, // and a aligned Float64 tensor with the statistics results for each such // unique group value. See the README.md file for a diagram of the results. -func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { +func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) { gd, err := dir.RecycleDir("Groups") if errors.Log(err) != nil { return @@ -157,15 +157,15 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...*tensor.Indexed) { if gv == nil { gtsr := datafs.NewValue[string](sgd, gpnm, nv) for i, v := range vals { - gtsr.SetStringRow(v.Tensor.Metadata().Name(), i) + gtsr.SetStringRow(v.Metadata().Name(), i) } } for _, tsr := range tsrs { - vd, _ := sgd.RecycleDir(tsr.Tensor.Metadata().Name()) + vd, _ := sgd.RecycleDir(tsr.Metadata().Name()) sv := datafs.NewValue[float64](vd, stnm, nv) for i, v := range vals { - idx := v.Tensor.(*tensor.Int).Values - sg := tensor.NewIndexed(tsr.Tensor, idx) + idx := tensor.AsIntSlice(v) + sg := tensor.NewIndexed(tsr, idx...) tensor.Call(stat, sg, stout) sv.SetFloatRow(stout.Float1D(0), i) } @@ -180,7 +180,7 @@ func TableGroupStats(dir *datafs.Data, stat string, dt *table.Table, columns ... // GroupDescribe runs standard descriptive statistics on given tensor data // using [GroupStats] function, with [DescriptiveStats] list of stats. -func GroupDescribe(dir *datafs.Data, tsrs ...*tensor.Indexed) { +func GroupDescribe(dir *datafs.Data, tsrs ...tensor.Tensor) { for _, st := range DescriptiveStats { GroupStats(dir, st.FuncName(), tsrs...) } diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index 3696f0ebec..cb3ea6bd14 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -29,8 +29,8 @@ func TestGroup(t *testing.T) { TableGroups(dir, dt, "Name") ixs := dir.FlatValuesFunc(nil) - assert.Equal(t, []int{0, 1}, ixs[0].Tensor.(*tensor.Int).Values) - assert.Equal(t, []int{2, 3}, ixs[1].Tensor.(*tensor.Int).Values) + assert.Equal(t, []int{0, 1}, tensor.AsIntTensor(ixs[0]).Values) + assert.Equal(t, []int{2, 3}, tensor.AsIntTensor(ixs[1]).Values) TableGroupStats(dir, Mean.FuncName(), dt, "Value") diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 421469a48b..92a901f315 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -19,24 +19,24 @@ import ( // Uses linear interpolation. // Because this requires a sort, it is more efficient to get as many quantiles // as needed in one pass. -func QuantilesFunc(in, qs, out *tensor.Indexed) error { - if in.Tensor.NumDims() != 1 { +func QuantilesFunc(in, qs, out tensor.Tensor) error { + if in.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D input tensors allowed")) } - if qs.Tensor.NumDims() != 1 { + if qs.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } - sin := in.Clone() + sin := tensor.AsIndexed(in.Clone()) sin.ExcludeMissing() sin.Sort(tensor.Ascending) - out.Tensor.SetShapeFrom(qs.Tensor) + tensor.SetShapeFrom(out, qs) sz := len(sin.Indexes) - 1 // length of our own index list if sz <= 0 { - out.Tensor.SetZeros() + out.SetZeros() return nil } fsz := float64(sz) - nq := qs.Tensor.Len() + nq := qs.Len() for i := range nq { q := qs.Float1D(i) val := 0.0 @@ -60,18 +60,18 @@ func QuantilesFunc(in, qs, out *tensor.Indexed) error { // MedianFunc computes the median (50% quantile) of tensor values. // See [StatsFunc] for general information. -func MedianFunc(in, out *tensor.Indexed) { +func MedianFunc(in, out tensor.Tensor) { QuantilesFunc(in, tensor.NewFloat64Scalar(.5), out) } // Q1Func computes the first quantile (25%) of tensor values. // See [StatsFunc] for general information. -func Q1Func(in, out *tensor.Indexed) { +func Q1Func(in, out tensor.Tensor) { QuantilesFunc(in, tensor.NewFloat64Scalar(.25), out) } // Q3Func computes the third quantile (75%) of tensor values. // See [StatsFunc] for general information. -func Q3Func(in, out *tensor.Indexed) { +func Q3Func(in, out tensor.Tensor) { QuantilesFunc(in, tensor.NewFloat64Scalar(.75), out) } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index b901649b09..da4c0ed7c7 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -124,12 +124,12 @@ func StripPackage(name string) string { // Stat calls a standard Stats enum function on given tensors. // Output results are in the out tensor. -func Stat(stat Stats, in, out *tensor.Indexed) { +func Stat(stat Stats, in, out tensor.Tensor) { tensor.Call(stat.FuncName(), in, out) } // StatOut calls a standard Stats enum function on given tensor, // returning output as a newly created tensor. -func StatOut(stat Stats, in *tensor.Indexed) *tensor.Indexed { +func StatOut(stat Stats, in tensor.Tensor) tensor.Tensor { return tensor.CallOut(stat.FuncName(), in) } diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index ef21d13fbd..c71a85ee3b 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -14,76 +14,76 @@ import ( func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) + // ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) + ix := tensor.NewNumberFromSlice(vals...) out := tensor.NewFloat64(1) - oix := tensor.NewIndexed(out) results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} tol := 1.0e-8 - CountFunc(ix, oix) + CountFunc(ix, out) assert.Equal(t, results[Count], out.Values[0]) - SumFunc(ix, oix) + SumFunc(ix, out) assert.Equal(t, results[Sum], out.Values[0]) - SumAbsFunc(ix, oix) + SumAbsFunc(ix, out) assert.Equal(t, results[SumAbs], out.Values[0]) - ProdFunc(ix, oix) + ProdFunc(ix, out) assert.Equal(t, results[Prod], out.Values[0]) - MinFunc(ix, oix) + MinFunc(ix, out) assert.Equal(t, results[Min], out.Values[0]) - MaxFunc(ix, oix) + MaxFunc(ix, out) assert.Equal(t, results[Max], out.Values[0]) - MinAbsFunc(ix, oix) + MinAbsFunc(ix, out) assert.Equal(t, results[MinAbs], out.Values[0]) - MaxAbsFunc(ix, oix) + MaxAbsFunc(ix, out) assert.Equal(t, results[MaxAbs], out.Values[0]) - MeanFunc(ix, oix) + MeanFunc(ix, out) assert.Equal(t, results[Mean], out.Values[0]) - VarFunc(ix, oix) + VarFunc(ix, out) assert.InDelta(t, results[Var], out.Values[0], tol) - StdFunc(ix, oix) + StdFunc(ix, out) assert.InDelta(t, results[Std], out.Values[0], tol) - SemFunc(ix, oix) + SemFunc(ix, out) assert.InDelta(t, results[Sem], out.Values[0], tol) - VarPopFunc(ix, oix) + VarPopFunc(ix, out) assert.InDelta(t, results[VarPop], out.Values[0], tol) - StdPopFunc(ix, oix) + StdPopFunc(ix, out) assert.InDelta(t, results[StdPop], out.Values[0], tol) - SemPopFunc(ix, oix) + SemPopFunc(ix, out) assert.InDelta(t, results[SemPop], out.Values[0], tol) - SumSqFunc(ix, oix) + SumSqFunc(ix, out) assert.InDelta(t, results[SumSq], out.Values[0], tol) - L2NormFunc(ix, oix) + L2NormFunc(ix, out) assert.InDelta(t, results[L2Norm], out.Values[0], tol) - MedianFunc(ix, oix) + MedianFunc(ix, out) assert.InDelta(t, results[Median], out.Values[0], tol) - Q1Func(ix, oix) + Q1Func(ix, out) assert.InDelta(t, results[Q1], out.Values[0], tol) - Q3Func(ix, oix) + Q3Func(ix, out) assert.InDelta(t, results[Q3], out.Values[0], tol) for stat := Count; stat < StatsN; stat++ { - Stat(stat, ix, oix) + Stat(stat, ix, out) assert.InDelta(t, results[stat], out.Values[0], tol) } } @@ -93,63 +93,62 @@ func TestFuncsInt(t *testing.T) { tsr := tensor.NewNumberFromSlice(vals...) ix := tensor.NewIndexed(tsr) out := tensor.NewInt(1) - oix := tensor.NewIndexed(out) results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} - CountFunc(ix, oix) + CountFunc(ix, out) assert.Equal(t, results[Count], out.Values[0]) - SumFunc(ix, oix) + SumFunc(ix, out) assert.Equal(t, results[Sum], out.Values[0]) - SumAbsFunc(ix, oix) + SumAbsFunc(ix, out) assert.Equal(t, results[SumAbs], out.Values[0]) - ProdFunc(ix, oix) + ProdFunc(ix, out) assert.Equal(t, results[Prod], out.Values[0]) - MinFunc(ix, oix) + MinFunc(ix, out) assert.Equal(t, results[Min], out.Values[0]) - MaxFunc(ix, oix) + MaxFunc(ix, out) assert.Equal(t, results[Max], out.Values[0]) - MinAbsFunc(ix, oix) + MinAbsFunc(ix, out) assert.Equal(t, results[MinAbs], out.Values[0]) - MaxAbsFunc(ix, oix) + MaxAbsFunc(ix, out) assert.Equal(t, results[MaxAbs], out.Values[0]) - MeanFunc(ix, oix) + MeanFunc(ix, out) assert.Equal(t, results[Mean], out.Values[0]) - VarFunc(ix, oix) + VarFunc(ix, out) assert.Equal(t, results[Var], out.Values[0]) - StdFunc(ix, oix) + StdFunc(ix, out) assert.Equal(t, results[Std], out.Values[0]) - SemFunc(ix, oix) + SemFunc(ix, out) assert.Equal(t, results[Sem], out.Values[0]) - VarPopFunc(ix, oix) + VarPopFunc(ix, out) assert.Equal(t, results[VarPop], out.Values[0]) - StdPopFunc(ix, oix) + StdPopFunc(ix, out) assert.Equal(t, results[StdPop], out.Values[0]) - SemPopFunc(ix, oix) + SemPopFunc(ix, out) assert.Equal(t, results[SemPop], out.Values[0]) - SumSqFunc(ix, oix) + SumSqFunc(ix, out) assert.Equal(t, results[SumSq], out.Values[0]) - L2NormFunc(ix, oix) + L2NormFunc(ix, out) assert.Equal(t, results[L2Norm], out.Values[0]) for stat := Count; stat <= SemPop; stat++ { - Stat(stat, ix, oix) + Stat(stat, ix, out) assert.Equal(t, results[stat], out.Values[0]) } } @@ -166,18 +165,17 @@ func TestFuncsCell(t *testing.T) { ix := tensor.NewIndexed(tsr) out := tensor.NewFloat32(20, 10) - oix := tensor.NewIndexed(out) - CountFunc(ix, oix) + CountFunc(ix, out) nsub := out.Len() for i := range nsub { assert.Equal(t, 20.0, out.FloatRowCell(0, i)) } - MeanFunc(ix, oix) + MeanFunc(ix, out) for i := range nsub { assert.InDelta(t, vals[i], out.FloatRowCell(0, i), 1.0e-7) // lower tol, using float32 } - VarFunc(ix, oix) + VarFunc(ix, out) for i := range nsub { assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) } diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 3b4cb6bee2..1adb8deba8 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -19,21 +19,21 @@ import ( // and returns the Float64 output tensor for further processing as needed. // It uses the _last_ tensor as the output, allowing for multiple inputs, // as in the case of VarVecFun. -func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) *tensor.Indexed { +func VectorizeOut64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) tensor.Tensor { n := nfunc(tsr...) if n <= 0 { return nil } nt := len(tsr) out := tsr[nt-1] - osz := out.Tensor.ShapeInts() - o64 := tensor.NewIndexed(tensor.NewFloat64(osz...)) + osz := out.ShapeInts() + o64 := tensor.NewFloat64(osz...) etsr := slices.Clone(tsr) etsr[nt-1] = o64 for idx := range n { fun(idx, etsr...) } - nsub := out.Tensor.Len() + nsub := out.Len() for i := range nsub { out.SetFloat1D(o64.Float1D(i), i) } @@ -43,16 +43,16 @@ func VectorizeOut64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, ts // Vectorize2Out64 is a version of the [tensor.Vectorize] function // for stats, which makes two Float64 output tensors for aggregating // and computing values, returning them for final computation. -func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, tsr ...*tensor.Indexed), tsr ...*tensor.Indexed) (out1, out2 *tensor.Indexed) { +func Vectorize2Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (out1, out2 tensor.Tensor) { n := nfunc(tsr...) if n <= 0 { return nil, nil } nt := len(tsr) out := tsr[nt-1] - osz := out.Tensor.ShapeInts() - out1 = tensor.NewIndexed(tensor.NewFloat64(osz...)) - out2 = tensor.NewIndexed(tensor.NewFloat64(osz...)) + osz := out.ShapeInts() + out1 = tensor.NewFloat64(osz...) + out2 = tensor.NewFloat64(osz...) tsrs := slices.Clone(tsr[:nt-1]) tsrs = append(tsrs, out1, out2) for idx := range n { @@ -64,30 +64,31 @@ func Vectorize2Out64(nfunc func(tsr ...*tensor.Indexed) int, fun func(idx int, t // OutShape returns the output shape based on given input // shape ints, with outer row dim = 1. func OutShape(ish ...int) []int { - ish[0] = 1 - return ish + osh := slices.Clone(ish) + osh[0] = 1 + return osh } // NFunc is the nfun for stats functions, returning number of rows of the // first tensor, and initializing the _last_ one to hold the output // with the first, row dimension set to 1. -func NFunc(tsr ...*tensor.Indexed) int { +func NFunc(tsr ...tensor.Tensor) int { nt := len(tsr) if nt < 2 { return 0 } in, out := tsr[0], tsr[nt-1] - n := in.NumRows() - out.Tensor.SetShape(OutShape(in.Tensor.ShapeInts()...)...) - out.Indexes = nil + _ = out + n := in.DimSize(0) + out.SetShapeInts(OutShape(in.ShapeInts()...)...) return n } // VecFunc is a helper function for stats functions, dealing with iterating over // the Cell subspace per row and initializing the aggregation values for first index. // It also skips over NaN missing values. -func VecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg float64) float64) { - nsub := out.Tensor.Len() +func VecFunc(idx int, in, out tensor.Tensor, ini float64, fun func(val, agg float64) float64) { + nsub := out.Len() for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) @@ -105,8 +106,8 @@ func VecFunc(idx int, in, out *tensor.Indexed, ini float64, fun func(val, agg fl // This version has 2 input vectors, the second input being the output of another stat // e.g., the mean for Var. // It also skips over NaN missing values. -func Vec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(val1, val2, agg float64) float64) { - nsub := out.Tensor.Len() +func Vec2inFunc(idx int, in1, in2, out tensor.Tensor, ini float64, fun func(val1, val2, agg float64) float64) { + nsub := out.Len() for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) @@ -124,8 +125,8 @@ func Vec2inFunc(idx int, in1, in2, out *tensor.Indexed, ini float64, fun func(va // the Cell subspace per row and initializing the aggregation values for first index. // This version has 2 output vectors, for separate integration of scale sum squared // It also skips over NaN missing values. -func Vec2outFunc(idx int, in, out1, out2 *tensor.Indexed, ini1, ini2 float64, fun func(val, agg1, agg2 float64) (float64, float64)) { - nsub := out2.Tensor.Len() +func Vec2outFunc(idx int, in, out1, out2 tensor.Tensor, ini1, ini2 float64, fun func(val, agg1, agg2 float64) (float64, float64)) { + nsub := out2.Len() for i := range nsub { if idx == 0 { out1.SetFloat1D(ini1, i) diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 03e029c932..8542d1d020 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -12,9 +12,9 @@ import ( "cogentcore.org/core/tensor" ) -// Index returns the actual index into underlying tensor row based on given +// RowIndex returns the actual index into underlying tensor row based on given // index value. If Indexes == nil, index is passed through. -func (dt *Table) Index(idx int) int { +func (dt *Table) RowIndex(idx int) int { if dt.Indexes == nil { return idx } diff --git a/tensor/table/io.go b/tensor/table/io.go index da1c6d6641..15e0b3bf0c 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -401,7 +401,7 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { } rc++ } else { - csh := tensor.NewShape(tsr.ShapeInts[1:]...) // cell shape + csh := tensor.NewShape(tsr.ShapeInts()[1:]...) // cell shape tc := csh.Len() for ti := 0; ti < tc; ti++ { vl := "" @@ -433,7 +433,7 @@ func (dt *Table) TableHeaders() []string { if tsr.NumDims() == 1 { hdrs = append(hdrs, nm) } else { - csh := tensor.NewShape(tsr.ShapeInts[1:]...) // cell shape + csh := tensor.NewShape(tsr.ShapeInts()[1:]...) // cell shape tc := csh.Len() nd := csh.NumDims() fnm := nm + fmt.Sprintf("[%v:", nd) @@ -450,7 +450,7 @@ func (dt *Table) TableHeaders() []string { ffnm += "]" + dn + ">" hdrs = append(hdrs, ffnm) for ti := 1; ti < tc; ti++ { - idx := csh.Index(ti) + idx := csh.IndexFrom1D(ti) ffnm := fnm for di := 0; di < nd; di++ { ffnm += fmt.Sprintf("%v", idx[di]) diff --git a/tensor/table/io_test.go b/tensor/table/io_test.go index 12c46097a3..d4102cb0ac 100644 --- a/tensor/table/io_test.go +++ b/tensor/table/io_test.go @@ -9,6 +9,8 @@ import ( "reflect" "strings" "testing" + + "cogentcore.org/core/tensor" ) func TestTableHeaders(t *testing.T) { @@ -34,14 +36,14 @@ func TestTableHeaders(t *testing.T) { if cols[2].DataType() != reflect.Float32 { t.Errorf("TableHeaders: cols[2] != FLOAT32\n") } - shsz := AsInts(cols[1].ShapeSizes()) + shsz := tensor.AsIntSlice(cols[1].ShapeSizes()) if shsz[1] != 5 { t.Errorf("TableHeaders: cols[1].ShapeSizes[1] != 5\n") } if shsz[2] != 5 { t.Errorf("TableHeaders: cols[1].ShapeSizes[2] != 5\n") } - shsz = AsInts(cols[2].ShapeSizes()) + shsz = tensor.AsIntSlice(cols[2].ShapeSizes()) if shsz[1] != 5 { t.Errorf("TableHeaders: cols[2].ShapeSizes[1] != 5\n") } diff --git a/tensor/table/table.go b/tensor/table/table.go index 1f03351120..f666301795 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -86,7 +86,7 @@ func (dt *Table) Column(name string) *tensor.Indexed { if cl == nil { return nil } - return tensor.NewIndexed(cl, dt.Indexes) + return tensor.NewIndexed(cl, dt.Indexes...) } // ColumnTry is a version of [Table.Column] that also returns an error @@ -106,13 +106,13 @@ func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { // Will panic if out of range. func (dt *Table) ColumnByIndex(idx int) *tensor.Indexed { cl := dt.Columns.Values[idx] - return tensor.NewIndexed(cl, dt.Indexes) + return tensor.NewIndexed(cl, dt.Indexes...) } // ColumnList returns a list of tensors with given column names, // as [tensor.Indexed] with the shared [Table.Indexes] from this table. -func (dt *Table) ColumnList(names ...string) []*tensor.Indexed { - list := make([]*tensor.Indexed, 0, len(names)) +func (dt *Table) ColumnList(names ...string) []tensor.Tensor { + list := make([]tensor.Tensor, 0, len(names)) for _, nm := range names { cl := dt.Column(nm) if cl != nil { @@ -151,7 +151,7 @@ func AddColumn[T tensor.DataTypes](dt *Table, name string, cellSizes ...int) ten rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) tsr := tensor.New[T](sz...) - tsr.SetNames("Row") + // tsr.SetNames("Row") dt.AddColumn(name, tsr) return tsr } @@ -164,7 +164,7 @@ func InsertColumn[T tensor.DataTypes](dt *Table, name string, idx int, cellSizes rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) tsr := tensor.New[T](sz...) - tsr.SetNames("Row") + // tsr.SetNames("Row") dt.InsertColumn(idx, name, tsr) return tsr } @@ -192,7 +192,7 @@ func (dt *Table) AddColumnOfType(name string, typ reflect.Kind, cellSizes ...int rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) tsr := tensor.NewOfType(typ, sz...) - tsr.SetNames("Row") + // tsr.SetNames("Row") dt.AddColumn(name, tsr) return tsr } diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index 945755d0d8..55ba2b870a 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -126,7 +126,7 @@ func TestCells(t *testing.T) { // fmt.Println(s) ss := in.Tensor.SubSpace(i).(*tensor.Float32) // fmt.Println(ss.Values[:16]) - cl := in.Cells1D(i).Tensor.(*tensor.Float32) + cl := tensor.AsFloat32Tensor(tensor.Cells1D(in, i)) // fmt.Println(cl.Values[:16]) assert.Equal(t, vals, ss.Values[:16]) assert.Equal(t, vals, cl.Values[:16]) diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 30826ddf66..421f85d245 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -66,7 +66,7 @@ func TestTensorString(t *testing.T) { cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) - af := AsFloat64s(cln) + af := AsFloat64Slice(cln) assert.Equal(t, 3.14, af[0]) assert.Equal(t, 0.0, af[1]) } @@ -118,11 +118,12 @@ func TestTensorFloat64(t *testing.T) { cln.SetString1D("3.14", 0) assert.Equal(t, 3.14, cln.Float1D(0)) - af := AsFloat64s(cln) + af := AsFloat64Slice(cln) assert.Equal(t, 3.14, af[0]) assert.Equal(t, 0.0, af[1]) } +/* func TestSlice(t *testing.T) { ft := NewFloat64(3, 4, 5) for r := range 3 { @@ -135,62 +136,63 @@ func TestSlice(t *testing.T) { } res := `Tensor: [20] -[0]: 200 -[1]: 201 -[2]: 202 -[3]: 203 -[4]: 204 -[5]: 210 -[6]: 211 -[7]: 212 -[8]: 213 -[9]: 214 -[10]: 220 -[11]: 221 -[12]: 222 -[13]: 223 -[14]: 224 -[15]: 230 -[16]: 231 -[17]: 232 -[18]: 233 -[19]: 234 +[0]: 200 +[1]: 201 +[2]: 202 +[3]: 203 +[4]: 204 +[5]: 210 +[6]: 211 +[7]: 212 +[8]: 213 +[9]: 214 +[10]: 220 +[11]: 221 +[12]: 222 +[13]: 223 +[14]: 224 +[15]: 230 +[16]: 231 +[17]: 232 +[18]: 233 +[19]: 234 ` - // fmt.Println(ft.Cells1D(1)) - assert.Equal(t, res, ft.Cells1D(1).String()) + // fmt.Println(Cells1D(ft, 1)) + assert.Equal(t, res, Cells1D(ft, 1).String()) // fmt.Println(ft.String()) sf := NewFloat64() Slice(ft, sf, Range{}, Range{Start: 1, End: 2}) // fmt.Println(sf.String()) res = `Tensor: [3, 1, 5] -[0 0]: 110 111 112 113 114 -[1 0]: 210 211 212 213 214 -[2 0]: 310 311 312 313 314 +[0 0]: 110 111 112 113 114 +[1 0]: 210 211 212 213 214 +[2 0]: 310 311 312 313 314 ` assert.Equal(t, res, sf.Tensor.String()) Slice(ft, sf, Range{}, Range{}, Range{Start: 1, End: 2}) // fmt.Println(sf.String()) res = `Tensor: [3, 4, 1] -[0 0]: 101 -[0 1]: 111 -[0 2]: 121 -[0 3]: 131 -[1 0]: 201 -[1 1]: 211 -[1 2]: 221 -[1 3]: 231 -[2 0]: 301 -[2 1]: 311 -[2 2]: 321 -[2 3]: 331 +[0 0]: 101 +[0 1]: 111 +[0 2]: 121 +[0 3]: 131 +[1 0]: 201 +[1 1]: 211 +[1 2]: 221 +[1 3]: 231 +[2 0]: 301 +[2 1]: 311 +[2 2]: 321 +[2 3]: 331 ` assert.Equal(t, res, sf.Tensor.String()) } +*/ func TestSortFilter(t *testing.T) { - tsr := NewFloat64Indexed(5) + tsr := NewIndexed(NewFloat64(5)) for i := range 5 { tsr.SetFloatRowCell(float64(i), i, 0) } diff --git a/tensor/tensormpi/tensor.go b/tensor/tensormpi/tensor.go index c9f929c1bc..96f573b59d 100644 --- a/tensor/tensormpi/tensor.go +++ b/tensor/tensormpi/tensor.go @@ -119,7 +119,7 @@ func ReduceTensor(dest, src tensor.Tensor, comm *mpi.Comm, op mpi.Op) error { } slen := src.Len() if slen != dest.Len() { - dest.SetShapeFrom(src) + tensor.SetShapeFrom(dest, src) } var err error switch dt { diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index a9ad9a9eba..386efd9c97 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -50,304 +50,267 @@ func init() { tensor.AddFunc("Y1", Y1, 1) } -func Abs(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(i)), i) +func Abs(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) } -func Acos(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(i)), i) +func Acos(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(idx)), idx) }, in, out) } -func Acosh(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(i)), i) +func Acosh(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(idx)), idx) }, in, out) } -func Asin(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(i)), i) +func Asin(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(idx)), idx) }, in, out) } -func Asinh(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(i)), i) +func Asinh(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(idx)), idx) }, in, out) } -func Atan(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(i)), i) +func Atan(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(idx)), idx) }, in, out) } -func Atanh(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(i)), i) +func Atanh(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(idx)), idx) }, in, out) } -func Cbrt(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(i)), i) +func Cbrt(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(idx)), idx) }, in, out) } -func Ceil(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(i)), i) +func Ceil(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(idx)), idx) }, in, out) } -func Cos(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(i)), i) +func Cos(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(idx)), idx) }, in, out) } -func Cosh(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(i)), i) +func Cosh(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(idx)), idx) }, in, out) } -func Erf(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(i)), i) +func Erf(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(idx)), idx) }, in, out) } -func Erfc(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(i)), i) +func Erfc(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(idx)), idx) }, in, out) } -func Erfcinv(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(i)), i) +func Erfcinv(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(idx)), idx) }, in, out) } -func Erfinv(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(i)), i) +func Erfinv(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(idx)), idx) }, in, out) } -func Exp(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(i)), i) +func Exp(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(idx)), idx) }, in, out) } -func Exp2(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(i)), i) +func Exp2(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(idx)), idx) }, in, out) } -func Expm1(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(i)), i) +func Expm1(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(idx)), idx) }, in, out) } -func Floor(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(i)), i) +func Floor(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(idx)), idx) }, in, out) } -func Gamma(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(i)), i) +func Gamma(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(idx)), idx) }, in, out) } -func J0(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(i)), i) +func J0(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(idx)), idx) }, in, out) } -func J1(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(i)), i) +func J1(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(idx)), idx) }, in, out) } -func Log(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(i)), i) +func Log(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(idx)), idx) }, in, out) } -func Log10(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(i)), i) +func Log10(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(idx)), idx) }, in, out) } -func Log1p(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(i)), i) +func Log1p(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(idx)), idx) }, in, out) } -func Log2(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(i)), i) +func Log2(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(idx)), idx) }, in, out) } -func Logb(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(i)), i) +func Logb(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(idx)), idx) }, in, out) } -func Round(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(i)), i) +func Round(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(idx)), idx) }, in, out) } -func RoundToEven(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(i)), i) +func RoundToEven(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(idx)), idx) }, in, out) } -func Sin(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(i)), i) +func Sin(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(idx)), idx) }, in, out) } -func Sinh(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(i)), i) +func Sinh(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(idx)), idx) }, in, out) } -func Sqrt(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(i)), i) +func Sqrt(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(idx)), idx) }, in, out) } -func Tan(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(i)), i) +func Tan(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(idx)), idx) }, in, out) } -func Tanh(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(i)), i) +func Tanh(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(idx)), idx) }, in, out) } -func Trunc(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(i)), i) +func Trunc(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(idx)), idx) }, in, out) } -func Y0(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(i)), i) +func Y0(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(idx)), idx) }, in, out) } -func Y1(in, out *tensor.Indexed) { - out.SetShapeFrom(in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(i)), i) +func Y1(in, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(idx)), idx) }, in, out) } /* -func Atan2(y, in, out *tensor.Indexed) +func Atan2(y, in, out tensor.Tensor) func Copysign(f, sign float64) float64 func Dim(x, y float64) float64 func Hypot(p, q float64) float64 @@ -372,8 +335,8 @@ func Float64frombits(b uint64) float64 func FMA(x, y, z float64) float64 -func Jn(n int, in, out *tensor.Indexed) -func Yn(n int, in, out *tensor.Indexed) +func Jn(n int, in, out tensor.Tensor) +func Yn(n int, in, out tensor.Tensor) func Ldexp(frac float64, exp int) float64 diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index f04f56c5fb..40ae77f834 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -13,7 +13,7 @@ import ( ) type onef func(x float64) float64 -type tonef func(in, out *tensor.Indexed) +type tonef func(in, out tensor.Tensor) // Equal does equal testing taking into account NaN func Equal(t *testing.T, trg, val float64) { @@ -36,7 +36,7 @@ func TestMath(t *testing.T) { oneout := oned.Clone() cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { i, _, ci := cell2d.RowCellIndex(idx) cell2d.SetFloat1D(oned.Float1D(ci), i) }, cell2d) @@ -61,7 +61,7 @@ func TestMath(t *testing.T) { // fmt.Println(r) si := lv * r for c, v := range vals { - ov := cellout.Tensor.(*tensor.Float32).Values[si+c] + ov := tensor.AsFloat32Tensor(cellout).Values[si+c] Equal(t, fun(v), float64(ov)) } } diff --git a/tensor/tmath/norm.go b/tensor/tmath/norm.go index a2e65afd9c..e58e7661ec 100644 --- a/tensor/tmath/norm.go +++ b/tensor/tmath/norm.go @@ -12,7 +12,7 @@ import ( // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. -func ZScore(a, out *tensor.Indexed) { +func ZScore(a, out tensor.Tensor) { mout := tensor.NewIndexed(tensor.NewFloat64()) std, mean, _ := stats.StdFuncOut64(a, mout) Sub(a, mean, out) @@ -21,7 +21,7 @@ func ZScore(a, out *tensor.Indexed) { // UnitNorm computes unit normalized values into given output tensor, // subtracting the Min value and dividing by the Max of the remaining numbers. -func UnitNorm(a, out *tensor.Indexed) { +func UnitNorm(a, out tensor.Tensor) { mout := tensor.NewIndexed(tensor.NewFloat64()) stats.MinFunc(a, mout) Sub(a, mout, out) @@ -32,30 +32,28 @@ func UnitNorm(a, out *tensor.Indexed) { // Clamp ensures that all values are within min, max limits, clamping // values to those bounds if they exceed them. min and max args are // treated as scalars (first value used). -func Clamp(in, minv, maxv, out *tensor.Indexed) { - out.SetShapeFrom(in) +func Clamp(in, minv, maxv, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) mn := minv.Float1D(0) mx := maxv.Float1D(0) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - tsr[1].SetFloat1D(math32.Clamp(tsr[0].Float1D(i), mn, mx), i) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + tsr[1].SetFloat1D(math32.Clamp(tsr[0].Float1D(idx), mn, mx), idx) }, in, out) } // Binarize results in a binary-valued output by setting // values >= the threshold to 1, else 0. threshold is // treated as a scalar (first value used). -func Binarize(in, threshold, out *tensor.Indexed) { - out.SetShapeFrom(in) +func Binarize(in, threshold, out tensor.Tensor) { + tensor.SetShapeFrom(out, in) thr := threshold.Float1D(0) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - i, _, _ := tsr[0].RowCellIndex(idx) - v := tsr[0].Float1D(i) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + v := tsr[0].Float1D(idx) if v >= thr { v = 1 } else { v = 0 } - tsr[1].SetFloat1D(v, i) + tsr[1].SetFloat1D(v, idx) }, in, out) } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 6bce47c6cd..3e04ef693c 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -21,7 +21,7 @@ func init() { // then the SubSpace is added to each row of the other. // Otherwise an element-wise addition is performed // for overlapping cells. -func Add(a, b, out *tensor.Indexed) { +func Add(a, b, out tensor.Tensor) { if b.Len() == 1 { AddScalar(b.FloatRow(0), a, out) return @@ -30,8 +30,8 @@ func Add(a, b, out *tensor.Indexed) { AddScalar(a.FloatRow(0), b, out) return } - arows, acells := a.Tensor.RowCellSize() - brows, bcells := b.Tensor.RowCellSize() + arows, acells := a.RowCellSize() + brows, bcells := b.RowCellSize() if brows*bcells == acells { AddSubSpace(a, b, out) return @@ -41,34 +41,31 @@ func Add(a, b, out *tensor.Indexed) { return } // just do element-wise - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return tensor.NMinLen(2, tsr...) }, - func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := tsr[0].RowCellIndex(idx) - ib, _, _ := tsr[1].RowCellIndex(idx) - out.SetFloat1D(tsr[0].Float1D(ia)+tsr[1].Float1D(ib), ia) + func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(tsr[0].Float1D(idx)+tsr[1].Float1D(idx), idx) }, a, b, out) } // AddScalar adds a scalar to given tensor into output. -func AddScalar(scalar float64, a, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := a.RowCellIndex(idx) - out.SetFloat1D(a.Float1D(ia)+scalar, ia) +func AddScalar(scalar float64, a, out tensor.Tensor) { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(a.Float1D(idx)+scalar, idx) }, a, out) } // AddSubSpace adds the subspace tensor to each row in the given tensor, // into the output tensor. -func AddSubSpace(a, sub, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, ci := a.RowCellIndex(idx) - si, _, _ := sub.RowCellIndex(ci) - out.SetFloat1D(a.Float1D(ia)+sub.Float1D(si), ia) +func AddSubSpace(a, sub, out tensor.Tensor) { + _, acells := a.RowCellSize() + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + si := idx % acells + out.SetFloat1D(a.Float1D(idx)+sub.Float1D(si), idx) }, a, sub, out) } @@ -81,7 +78,7 @@ func AddSubSpace(a, sub, out *tensor.Indexed) { // then the SubSpace is subtracted from each row of the other. // Otherwise an element-wise subtractition is performed // for overlapping cells. -func Sub(a, b, out *tensor.Indexed) { +func Sub(a, b, out tensor.Tensor) { if b.Len() == 1 { SubScalar(1, b.FloatRow(0), a, out) return @@ -90,8 +87,8 @@ func Sub(a, b, out *tensor.Indexed) { SubScalar(-1, a.FloatRow(0), b, out) return } - arows, acells := a.Tensor.RowCellSize() - brows, bcells := b.Tensor.RowCellSize() + arows, acells := a.RowCellSize() + brows, bcells := b.RowCellSize() if brows*bcells == acells { SubSubSpace(1, a, b, out) return @@ -101,36 +98,33 @@ func Sub(a, b, out *tensor.Indexed) { return } // just do element-wise - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return tensor.NMinLen(2, tsr...) }, - func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := tsr[0].RowCellIndex(idx) - ib, _, _ := tsr[1].RowCellIndex(idx) - out.SetFloat1D(tsr[0].Float1D(ia)-tsr[1].Float1D(ib), ia) + func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(tsr[0].Float1D(idx)-tsr[1].Float1D(idx), idx) }, a, b, out) } // SubScalar subtracts a scalar from given tensor into output. // sign determines which way the subtraction goes: 1 = a-scalar, -1 = scalar-a -func SubScalar(sign float64, scalar float64, a, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := a.RowCellIndex(idx) - out.SetFloat1D(sign*(a.Float1D(ia)-scalar), ia) +func SubScalar(sign float64, scalar float64, a, out tensor.Tensor) { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(sign*(a.Float1D(idx)-scalar), idx) }, a, out) } // SubSubSpace subtracts the subspace tensor to each row in the given tensor, // into the output tensor. // sign determines which way the subtraction goes: 1 = a-sub, -1 = sub-a -func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, ci := a.RowCellIndex(idx) - si, _, _ := sub.RowCellIndex(ci) - out.SetFloat1D(sign*(a.Float1D(ia)-sub.Float1D(si)), ia) +func SubSubSpace(sign float64, a, sub, out tensor.Tensor) { + _, acells := a.RowCellSize() + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + si := idx % acells + out.SetFloat1D(sign*(a.Float1D(idx)-sub.Float1D(si)), idx) }, a, sub, out) } @@ -143,7 +137,7 @@ func SubSubSpace(sign float64, a, sub, out *tensor.Indexed) { // then the SubSpace multiplies each row of the other. // Otherwise an element-wise multiplication is performed // for overlapping cells. -func Mul(a, b, out *tensor.Indexed) { +func Mul(a, b, out tensor.Tensor) { if b.Len() == 1 { MulScalar(b.FloatRow(0), a, out) return @@ -152,8 +146,8 @@ func Mul(a, b, out *tensor.Indexed) { MulScalar(a.FloatRow(0), b, out) return } - arows, acells := a.Tensor.RowCellSize() - brows, bcells := b.Tensor.RowCellSize() + arows, acells := a.RowCellSize() + brows, bcells := b.RowCellSize() if brows*bcells == acells { MulSubSpace(a, b, out) return @@ -163,34 +157,31 @@ func Mul(a, b, out *tensor.Indexed) { return } // just do element-wise - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return tensor.NMinLen(2, tsr...) }, - func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := tsr[0].RowCellIndex(idx) - ib, _, _ := tsr[1].RowCellIndex(idx) - out.SetFloat1D(tsr[0].Float1D(ia)*tsr[1].Float1D(ib), ia) + func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(tsr[0].Float1D(idx)*tsr[1].Float1D(idx), idx) }, a, b, out) } // MulScalar multiplies a scalar to given tensor into output. -func MulScalar(scalar float64, a, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := a.RowCellIndex(idx) - out.SetFloat1D(a.Float1D(ia)*scalar, ia) +func MulScalar(scalar float64, a, out tensor.Tensor) { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(a.Float1D(idx)*scalar, idx) }, a, out) } // MulSubSpace multiplies the subspace tensor to each row in the given tensor, // into the output tensor. -func MulSubSpace(a, sub, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, ci := a.RowCellIndex(idx) - si, _, _ := sub.RowCellIndex(ci) - out.SetFloat1D(a.Float1D(ia)*sub.Float1D(si), ia) +func MulSubSpace(a, sub, out tensor.Tensor) { + _, acells := a.RowCellSize() + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + si := idx % acells + out.SetFloat1D(a.Float1D(idx)*sub.Float1D(si), idx) }, a, sub, out) } @@ -203,7 +194,7 @@ func MulSubSpace(a, sub, out *tensor.Indexed) { // then the SubSpace divides each row of the other. // Otherwise an element-wise division is performed // for overlapping cells. -func Div(a, b, out *tensor.Indexed) { +func Div(a, b, out tensor.Tensor) { if b.Len() == 1 { DivScalar(b.FloatRow(0), a, out) return @@ -212,8 +203,8 @@ func Div(a, b, out *tensor.Indexed) { DivScalarInv(a.FloatRow(0), b, out) return } - arows, acells := a.Tensor.RowCellSize() - brows, bcells := b.Tensor.RowCellSize() + arows, acells := a.RowCellSize() + brows, bcells := b.RowCellSize() if brows*bcells == acells { DivSubSpace(a, b, out) return @@ -223,54 +214,50 @@ func Div(a, b, out *tensor.Indexed) { return } // just do element-wise - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, func(tsr ...*tensor.Indexed) int { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return tensor.NMinLen(2, tsr...) }, - func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := tsr[0].RowCellIndex(idx) - ib, _, _ := tsr[1].RowCellIndex(idx) - out.SetFloat1D(tsr[0].Float1D(ia)/tsr[1].Float1D(ib), ia) + func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(tsr[0].Float1D(idx)/tsr[1].Float1D(idx), idx) }, a, b, out) } // DivScalar divides given tensor elements by scalar into output. -func DivScalar(scalar float64, a, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := a.RowCellIndex(idx) - out.SetFloat1D(a.Float1D(ia)/scalar, ia) +func DivScalar(scalar float64, a, out tensor.Tensor) { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(a.Float1D(idx)/scalar, idx) }, a, out) } // DivScalarInv divides scalar by given tensor elements into output // (inverse of [DivScalar]). -func DivScalarInv(scalar float64, a, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, _ := a.RowCellIndex(idx) - out.SetFloat1D(scalar/a.Float1D(ia), ia) +func DivScalarInv(scalar float64, a, out tensor.Tensor) { + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(scalar/a.Float1D(idx), idx) }, a, out) } // DivSubSpace divides each row of the given tensor by the subspace tensor elements, // into the output tensor. -func DivSubSpace(a, sub, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, ci := a.RowCellIndex(idx) - si, _, _ := sub.RowCellIndex(ci) - out.SetFloat1D(a.Float1D(ia)/sub.Float1D(si), ia) +func DivSubSpace(a, sub, out tensor.Tensor) { + _, acells := a.RowCellSize() + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + si := idx % acells + out.SetFloat1D(a.Float1D(idx)/sub.Float1D(si), idx) }, a, sub, out) } // DivSubSpaceInv divides the subspace tensor by each row of the given tensor, // into the output tensor (inverse of [DivSubSpace]) -func DivSubSpaceInv(a, sub, out *tensor.Indexed) { - out.SetShapeFrom(a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { - ia, _, ci := a.RowCellIndex(idx) - si, _, _ := sub.RowCellIndex(ci) - out.SetFloat1D(sub.Float1D(si)/a.Float1D(ia), ia) +func DivSubSpaceInv(a, sub, out tensor.Tensor) { + _, acells := a.RowCellSize() + tensor.SetShapeFrom(out, a) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + si := idx % acells + out.SetFloat1D(sub.Float1D(si)/a.Float1D(idx), idx) }, a, sub, out) } diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index bd57440716..a43b9d5028 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -24,7 +24,7 @@ func TestAdd(t *testing.T) { oneout := oned.Clone() cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...*tensor.Indexed) { + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { i, _, ci := cell2d.RowCellIndex(idx) cell2d.SetFloat1D(oned.Float1D(ci), i) }, cell2d) @@ -47,7 +47,7 @@ func TestAdd(t *testing.T) { Add(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v+v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v+v, cellout.FloatRowCell(ri, i), 1.0e-6) } } @@ -75,7 +75,7 @@ func TestAdd(t *testing.T) { Sub(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v-v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v-v, cellout.FloatRowCell(ri, i), 1.0e-6) } } @@ -95,7 +95,7 @@ func TestAdd(t *testing.T) { Mul(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v*v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v*v, cellout.FloatRowCell(ri, i), 1.0e-6) } } @@ -123,7 +123,7 @@ func TestAdd(t *testing.T) { Div(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v/v, cellout.Tensor.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v/v, cellout.FloatRowCell(ri, i), 1.0e-6) } } @@ -138,7 +138,7 @@ func TestAdd(t *testing.T) { assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) stats.MaxFunc(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) - // fmt.Println(oneout.Tensor) + // fmt.Println(oneout) minv := tensor.NewFloat64Scalar(0) maxv := tensor.NewFloat64Scalar(1) @@ -147,7 +147,7 @@ func TestAdd(t *testing.T) { assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) stats.MaxFunc(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) - // fmt.Println(oneout.Tensor) + // fmt.Println(oneout) thr := tensor.NewFloat64Scalar(0.5) Binarize(oned, thr, oneout) @@ -155,5 +155,5 @@ func TestAdd(t *testing.T) { assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) stats.MaxFunc(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) - // fmt.Println(oneout.Tensor) + // fmt.Println(oneout) } diff --git a/tensor/vectorize.go b/tensor/vectorize.go index 9fa6a5390a..01b0978109 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -35,7 +35,7 @@ var ( // the outermost row dimension of the tensor. // This version runs purely sequentially on on this go routine. // See VectorizeThreaded and VectorizeGPU for other versions. -func Vectorize(nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { +func Vectorize(nfun func(tsr ...Tensor) int, fun func(idx int, tsr ...Tensor), tsr ...Tensor) { n := nfun(tsr...) if n <= 0 { return @@ -51,7 +51,7 @@ func Vectorize(nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed // (floating point operations) for the function exceeds the [ThreadingThreshold]. // Heuristically, numbers below this threshold do not result // in an overall speedup, due to overhead costs. -func VectorizeThreaded(flops int, nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { +func VectorizeThreaded(flops int, nfun func(tsr ...Tensor) int, fun func(idx int, tsr ...Tensor), tsr ...Tensor) { n := nfun(tsr...) if n <= 0 { return @@ -80,7 +80,7 @@ func DefaultNumThreads() int { // it is likely to be beneficial, in terms of the ThreadingThreshold. // If threads is 0, then the [DefaultNumThreads] will be used: // GOMAXPROCS subject to NumThreads constraint if non-zero. -func VectorizeOnThreads(threads int, nfun func(tsr ...*Indexed) int, fun func(idx int, tsr ...*Indexed), tsr ...*Indexed) { +func VectorizeOnThreads(threads int, nfun func(tsr ...Tensor) int, fun func(idx int, tsr ...Tensor), tsr ...Tensor) { if threads == 0 { threads = DefaultNumThreads() } @@ -108,16 +108,16 @@ func VectorizeOnThreads(threads int, nfun func(tsr ...*Indexed) int, fun func(id // NFirstRows is an N function for Vectorize that returns the number of // outer-dimension rows (or Indexes) of the first tensor. -func NFirstRows(tsr ...*Indexed) int { +func NFirstRows(tsr ...Tensor) int { if len(tsr) == 0 { return 0 } - return tsr[0].NumRows() + return tsr[0].DimSize(0) } // NFirstLen is an N function for Vectorize that returns the number of // elements in the tensor, taking into account the Indexes view. -func NFirstLen(tsr ...*Indexed) int { +func NFirstLen(tsr ...Tensor) int { if len(tsr) == 0 { return 0 } @@ -127,7 +127,7 @@ func NFirstLen(tsr ...*Indexed) int { // NMinLen is an N function for Vectorize that returns the min number of // elements across given number of tensors in the list. Use a closure // to call this with the nt. -func NMinLen(nt int, tsr ...*Indexed) int { +func NMinLen(nt int, tsr ...Tensor) int { nt = min(len(tsr), nt) if nt == 0 { return 0 From e71d11fe83cd66a6b6f3e533e6b620c4021d0590 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 20 Sep 2024 15:35:18 -0700 Subject: [PATCH 082/311] finally did my rtfm on numpy and have a solid plan for how everything relates. it is good to first confront the basic issues and think independently about solutions before seeing how they did it -- making a few different choices here that seem better. --- goal/README.md | 22 +- tensor/README.md | 97 ++++- tensor/broadcast.go | 0 tensor/convert.go | 22 + tensor/datafs/README.md | 2 +- tensor/datafs/data.go | 16 +- tensor/funcs.go | 10 +- tensor/indexed.go | 38 +- tensor/ranges.go | 77 ++++ tensor/slice.go | 186 -------- tensor/sliced.go | 812 +++++++++++++++++++++++++++++++++++ tensor/stats/cluster/plot.go | 4 +- tensor/stats/metric/doc.go | 2 +- tensor/stats/stats/README.md | 4 +- tensor/table/indexes.go | 4 +- tensor/table/table.go | 4 +- tensor/tensorcore/table.go | 2 +- tensor/tmath/math_test.go | 11 +- tensor/tmath/ops_test.go | 10 +- 19 files changed, 1052 insertions(+), 271 deletions(-) create mode 100644 tensor/broadcast.go create mode 100644 tensor/ranges.go delete mode 100644 tensor/slice.go create mode 100644 tensor/sliced.go diff --git a/goal/README.md b/goal/README.md index 7fcd3fd5c4..52d47d2452 100644 --- a/goal/README.md +++ b/goal/README.md @@ -223,7 +223,15 @@ TODO: update aboven In general, _Goal_ is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. -All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor, and directly supports an indexed view of the outermost row dimension, for optimized sorting and filtering operations. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information (_Goal_ does not support `matrix`). +All elements of a _Goal_ math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that _Goal_ does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately. + +The _view_ versions of `Tensor` include `tensor.Sliced`, `tensor.Masked`, and `tensor.Indexed`, each of which wraps around another "source" `Tensor`, and provides its own way of accessing the underlying data: + +* `Sliced` has an arbitrary set of indexes for each dimension, so access to values along that dimension go through the indexes. Thus, you could reverse the order of the columns (dimension 1), or only operate on a subset of them. + +* `Masked` has a `tensor.Bool` tensor that filters access to the underlying source tensor through a mask: anywhere the bool value is `false`, the corresponding source value is not settable, and returns `NaN` (missing value) when accessed. + +* `Indexed` is an optimized version of `Sliced` with indexes only for the first, outermost, _row_ dimension, which Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) @@ -259,10 +267,10 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | | | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | | | | | -| **Access** | | | -| (returns a _copy_, use `copy` to assign back) | (returns a _reference_, changes modify original) | (returns a _copy_) | | -| | `y = x.copy()` | `y=x` | NumPy assigns by reference | -| | `y = x[1, :].copy()` | `y=x(2,:)` | NumPy slices are by reference | +| **Access and Slicing** | | | +| (returns a _view_, changes modify original) | (returns a _reference_, changes modify original) | (returns a _copy_) | | +| `y = x.copy()` or `y = x.Clone()` | `y = x.copy()` | `y=x` | `y=x` just assigns `y` to point to `x`, so that changes to `y` also change `x`; need to make a `copy` to get distinct values | +| `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | | | `a[-1]` | `a(end)` | access last element | | | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | | | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | @@ -290,8 +298,8 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of `a` where column vector `v` > 0.5 | | | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of `a` where vector `v` > 0.5 | | | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | -| | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D tensor, a | -| | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, a | +| | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D tensor, `a` | +| | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, `a` | | | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the tensor `a` as tensor `b` with rows sorted by the first column | | | `np.unique(a)` | `unique(a)` | a vector of unique values in tensor `a` | | | | | diff --git a/tensor/README.md b/tensor/README.md index 6637d6a171..ca68cc4646 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -1,58 +1,60 @@ # Tensor -Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used `numpy` and `pandas` libraries in python, and the commercial MATLAB framework. +Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used [NumPy](https://numpy.org/doc/stable/index.html) and [pandas](https://pandas.pydata.org/) libraries in Python, and the commercial MATLAB framework. -The [Goal](../goal) augmented version of the _Go_ language directly supports numpy-like operations on tensors. A `Tensor` is comparable to the numpy `array` type, and it provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix and beyond. All functions take and return `Tensor` arguments. +The [Goal](../goal) augmented version of the _Go_ language directly supports numpy-like operations on tensors. A `Tensor` is comparable to the NumPy `ndarray` type, and it provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix and beyond. All functions take and return `Tensor` arguments. -The `Tensor` interface is implemented at the basic level with n-dimensional indexing into flat Go slices of any numeric data type (by `Number`), along with `String`, and `Bool` (which uses [bitslice](bitslice) for maximum efficiency). The `Shape` type provides all the n-dimensional indexing with arbitrary strides to allow any ordering, although _row major_ is the default and other orders have to be manually imposed. +The `Tensor` interface is implemented at the basic level with n-dimensional indexing into flat Go slices of any numeric data type (by `Number`), along with `String`, and `Bool` (which uses [bitslice](bitslice) for maximum efficiency). The `Shape` type provides all the n-dimensional indexing with arbitrary strides to allow any ordering, although _row major_ is the default and other orders have to be manually imposed. -In addition, there are three important "view" implementations of `Tensor` that wrap another Tensor to provide more flexible and efficient access to the data, consistent with the numpy functionality: +In addition, there are three important "view" implementations of `Tensor` that wrap another Tensor to provide more flexible and efficient access to the data, consistent with the NumPy functionality. See [Basic and Advanced Indexing](#basic-and-advanced-indexing) for more info. -* `Indexed` provides an index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension. Organizing data systematically along the row dimension provides a natural and efficient constraint that is leveraged throughout the `tensor` package: it eliminates many sources of ambiguity about how to process higher-dimensional data, and any given n-dimensional structure can be reshaped to fit in this row-based format. +* `Rows` provides an index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension. Organizing data systematically along the row dimension eliminates many sources of ambiguity about how to process higher-dimensional data, and any given n-dimensional structure can be reshaped to fit in this row-based format. -* `Sliced` is a more general version of `Indexed` that provides a sub-sliced view into the wrapped `Tensor`, using an indexed list for access along each dimension. +* `Sliced` is a more general version of `Rows` that provides a sub-sliced view into the wrapped `Tensor`, using an indexed list along each dimension, not just the outermost one. Thus, it can provide a systematic, reordered and filtered view onto the raw data, and it has a well-defined shape in terms of the number of indexes per dimension. This corresponds well with the NumPy basic sliced indexing model. -* `Bitmask` provides a `Bool` masked view onto each element in the wrapped `Tensor` (the two maintain the same shape), such that any cell with a `false` state returns a `NaN` (missing data) and `Set` functions are no-ops. +* `Masked` provides a `Bool` masked view onto each element in the wrapped `Tensor`, where the two maintain the same shape). Any cell with a `false` value in the bool mask returns a `NaN` (missing data), and `Set` functions are no-ops. + +* `Indexed` uses a tensor of indexes where the final, innermost dimension is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. The `[Set]FloatRow[Cell]` methods are used wherever possible, for the most efficient and natural indirection for row-major organized data. See [Standard shapes](#standard-shapes) for more info. -The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. It is completely up to the compute function how to interpret the index. There is a Threaded version of this for parallelizable functions, and a GPU version. +The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. In general it is completely up to the compute function how to interpret the index, although we also support the [broadcasting](#broadcasting) principles from NumPy for binary functions operating on two tensors. There is a Threaded version of this for parallelizable functions, and a GPU version. -All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensor for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. +All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensor for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. These functions are the equivalent of the [ufunc](https://numpy.org/doc/stable/user/basics.ufuncs.html) universal functions in NumPy. -* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. +* [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including [plot/plotcore](../plot/plotcore). -* [datafs](datafs) provides a virtual filesystem (FS) for organizing arbitrary collections of data, supporting interactive, ad-hoc (notebook style) as well as systematic data processing. Interactive [goal](../goal) shell commands (`cd`, `ls`, `mkdir` etc) can be used to navigate the data space, with numerical expressions immediately available to operate on the data and save results back to the filesystem. Furthermore, the data can be directly copied to / from the OS filesystem to persist it, and `goal` can transparently access data on remote systems through ssh. Furthermore, the [databrowser](databrowser) provides a fully interactive GUI for inspecting and plotting data. +* [datafs](datafs) provides a virtual filesystem (FS) for organizing arbitrary collections of data, supporting interactive, ad-hoc (notebook style) as well as systematic data processing. Interactive [goal](../goal) shell commands (`cd`, `ls`, `mkdir` etc) can be used to navigate the data space, with numerical expressions immediately available to operate on the data and save results back to the filesystem. Furthermore, the data can be directly copied to / from the OS filesystem to persist it, and `goal` can transparently access data on remote systems through ssh. Furthermore, the [databrowser](databrowser) provides a fully interactive GUI for inspecting and plotting data. * [tensorcore](tensorcore) provides core widgets for graphically displaying the `Tensor` and `Table` data, which are used in `datafs`. -* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data, including the standard `+, -, *, /` operators. `goal` then calls these functions. +* [tmath](tmath) implements all standard math functions on `tensor.Indexed` data, including the standard `+, -, *, /` operators. `goal` then calls these functions. * [plot/plotcore](../plot/plotcore) supports interactive plotting of `Table` data. -* [bitslice](bitslice) is a Go slice of bytes `[]byte` that has methods for setting individual bits, as if it was a slice of bools, while being 8x more memory efficient. This is used for encoding null entries in `etensor`, and as a Tensor of bool / bits there as well, and is generally very useful for binary (boolean) data. +* [bitslice](bitslice) is a Go slice of bytes `[]byte` that has methods for setting individual bits, as if it was a slice of bools, while being 8x more memory efficient. This is used for encoding null entries in `etensor`, and as a Tensor of bool / bits there as well, and is generally very useful for binary (boolean) data. * [stats](stats) implements a number of different ways of analyzing tensor and table data, including: - [cluster](cluster) implements agglomerative clustering of items based on [metric](metric) distance / similarity matrix data. - [convolve](convolve) convolves data (e.g., for smoothing). - - [glm](glm) fits a general linear model for one or more dependent variables as a function of one or more independent variables. This encompasses all forms of regression. + - [glm](glm) fits a general linear model for one or more dependent variables as a function of one or more independent variables. This encompasses all forms of regression. - [histogram](histogram) bins data into groups and reports the frequency of elements in the bins. - [metric](metric) computes similarity / distance metrics for comparing two tensors, and associated distance / similarity matrix functions, including PCA and SVD analysis functions that operate on a covariance matrix. - - [stats](stats) provides a set of standard summary statistics on a range of different data types, including basic slices of floats, to tensor and table data. It also includes the ability to extract Groups of values and generate statistics for each group, as in a "pivot table" in a spreadsheet. + - [stats](stats) provides a set of standard summary statistics on a range of different data types, including basic slices of floats, to tensor and table data. It also includes the ability to extract Groups of values and generate statistics for each group, as in a "pivot table" in a spreadsheet. # Standard shapes -In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: +In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: -* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data. Any `Tensor` can be accessed via a flat 1D index, which goes directly into the underlying Go slice for the basic types, and is appropriately (though somewhat expensively in some cases) indirected through the effective geometry in `Sliced` and `Indexed` types. +* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data. Any `Tensor` can be accessed via a flat 1D index, which goes directly into the underlying Go slice for the basic types, and is appropriately (though somewhat expensively in some cases) indirected through the effective geometry in `Sliced` and `Indexed` types. -* **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Indexed` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In most packages including [tmath](tmath) and [stats](stats), 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. +* **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Indexed` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In most packages including [tmath](tmath) and [stats](stats), 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. - A higher-dimensional tensor can also be re-shaped into this row, cell format by collapsing any number of additional outer dimensions into a longer, effective "row" index, with the remaining inner dimensions forming the cell-wise patterns. You can decide where to make the cut, and the `RowCellSplit` function makes it easy to create a new view of an existing tensor with this split made at a given dimension. + A higher-dimensional tensor can also be re-shaped into this row, cell format by collapsing any number of additional outer dimensions into a longer, effective "row" index, with the remaining inner dimensions forming the cell-wise patterns. You can decide where to make the cut, and the `RowCellSplit` function makes it easy to create a new view of an existing tensor with this split made at a given dimension. * **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations. @@ -104,7 +106,7 @@ ix.SetRowTensor(tsr, 4) ```Go // returns a flat, 1D Indexed view into n-dimensional tensor values at -// given row. This is used in compute routines that operate generically +// given row. This is used in compute routines that operate generically // on the entire row as a flat pattern. ci := tensor.Cells1D(ix, 5) ``` @@ -116,9 +118,62 @@ ci := tensor.Cells1D(ix, 5) val := ix.Float(3,2,1) ``` +# `Tensor` vs. Python NumPy + +The [Goal](../goal) language provides a reasonably faithful translation of NumPy `array` syntax into the corresponding Go tensor package implementations. For those already familiar with NumPy, it should mostly "just work", but the following provides a more in-depth explanation for how the two relate, and when you might get different results. + +## Basic and Advanced Indexing + +NumPy distinguishes between _basic indexing_ (using a single index or sliced ranges of indexes along each dimension) versus _advanced indexing_ (using an array of indexes or bools) indexing. Basic indexing returns a **view** into the original data (where changes to the view directly affect the underlying type), while advanced indexing returns a **copy**. + +However, rather confusingly (per this [stack overflow question](https://stackoverflow.com/questions/15691740/does-assignment-with-advanced-indexing-copy-array-data)), you can do direct assignment through advanced indexing (more on this below): +```Python +a[np.array([1,2])] = 5 # or: +a[a > 0.5] = 1 # boolean advanced indexing +``` + +Although powerful, the semantics of all of this is a bit confusing. In the `tensor` package, we provide what are hopefully more clear and concrete _view_ types that have well-defined semantics, and cover the relevant functionality, while hopefully being a bit easier to reason with. These were described at the start of this README. The correspondence to NumPy indexing is as follows: + +* Basic indexing by individual integer index coordinate values is supported by the basic `Number`, `String`, `Bool` value `Tensor`s. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) view then completes the basic indexing with arbitrary reordering and filtering along entire dimension values. + +* The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `CloneValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `CloneIndexes` method returns a 1D list of indexes of the `true` values, which can be used for the `Indexed` view. + +* The `Indexed` view corresponds to the array-based advanced indexing case in NumPy, but again it is a view, not a copy, so the assignment semantics are as expected from a view (and how NumPy behaves some of the time). Note that the NumPy version uses `n` separate index tensors, where each such tensor specifies the value of a corresponding dimension index, and all such tensors _must have the same shape_; that form can be converted into the single Indexes form with a utility function. Also, NumPy advanced indexing has a somewhat confusing property where it de-duplicates index references during some operations, such that `+=1` only increments +1 even when there are multiple elements in the view. The tensor version does not implement that special case, due to its direct view semantics. + +To reiterate, all view tensors have a `CloneValues` function, equivalent to the `copy` function in NumPy, which turns the view into a corresponding basic concrete value Tensor, so the copy semantics of advanced indexing (modulo the direct assignment behavior) can be achieved when assigning to a new variable. + +## Broadcasting + +The NumPy concept of [broadcasting](https://numpy.org/doc/stable/user/basics.broadcasting.html) is critical for flexibly defining the semantics for how functions taking two n-dimensional Tensor arguments behave when they have different shapes. If both tensors are 1D and the same length, then a simple matched iteration over both can take place. However, the broadcasting logic defines what happens when there is a systematic relationship between the two, enabling powerful (but sometimes difficult to understand) computations to be specified. + +The following examples demonstrate the logic: + +Innermost dimensions that match in dimension are iterated over as you'd expect: +``` +Image (3d array): 256 x 256 x 3 +Scale (1d array): 3 +Result (3d array): 256 x 256 x 3 +``` + +Anything with a dimension size of 1 will match against any other sized dimension: +``` +A (4d array): 8 x 1 x 6 x 1 +B (3d array): 7 x 1 x 5 +Result (4d array): 8 x 7 x 6 x 5 +``` +In the innermost dimension here, the single value in A acts like a "scalar" in relationship to the 5 values in B along that same dimension, operating on each one in turn. + +Any non-1 mismatch represents an error: +``` +A (2d array): 2 x 1 +B (3d array): 8 x 4 x 3 # second from last dimensions mismatched +``` + +Computationally, the broadcast logic is straightforward to implement, in terms of computing the resulting shape. Any missing outer dimensions can be replaced with 1s, and the full 1D product index on the result shape can be applied directly to the source shapes, using the modulo operator on length so it just repeatedly samples as needed. + # History -This package was originally developed as [etable](https://github.com/emer/etable) as part of the _emergent_ software framework. It always depended on the GUI framework that became Cogent Core, and having it integrated within the Core monorepo makes it easier to integrate updates, and also makes it easier to build advanced data management and visualization applications. For example, the [plot/plotcore](../plot/plotcore) package uses the `Table` to support flexible and powerful plotting functionality. +This package was originally developed as [etable](https://github.com/emer/etable) as part of the _emergent_ software framework. It always depended on the GUI framework that became Cogent Core, and having it integrated within the Core monorepo makes it easier to integrate updates, and also makes it easier to build advanced data management and visualization applications. For example, the [plot/plotcore](../plot/plotcore) package uses the `Table` to support flexible and powerful plotting functionality. It was completely rewritten in Sept 2024 to use a single data type (`tensor.Indexed`) and call signature for compute functions taking these args, to provide a simple and efficient data processing framework that greatly simplified the code and enables the [goal](../goal) language to directly transpile simplified math expressions into corresponding tensor compute code. diff --git a/tensor/broadcast.go b/tensor/broadcast.go new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tensor/convert.go b/tensor/convert.go index c6ad2fb379..a508e4d905 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -21,6 +21,28 @@ func Cells1D(tsr Tensor, row int) Tensor { return New1DViewOf(tsr.SubSpace(row)) } +// RowCellSplit splits the given tensor into a standard 2D row, cell +// shape at the given split dimension index. All dimensions prior to +// split are collapsed into the row dimension, and from split onward +// form the cells dimension. The resulting tensor is a re-shaped view +// of the original tensor, sharing the same underlying data. +func RowCellSplit(tsr Tensor, split int) Tensor { + sizes := tsr.ShapeInts() + rows := sizes[:split] + cells := sizes[split:] + nr := 1 + for _, r := range rows { + nr *= r + } + nc := 1 + for _, c := range cells { + nc *= c + } + vw := tsr.View() + vw.SetShapeInts(nr, nc) + return vw +} + // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. func NewFloat64Scalar(val float64) Tensor { diff --git a/tensor/datafs/README.md b/tensor/datafs/README.md index cae33f83e5..78d9fc848c 100644 --- a/tensor/datafs/README.md +++ b/tensor/datafs/README.md @@ -2,7 +2,7 @@ `datafs` is a virtual file system that implements the Go `fs` interface, and can be accessed using fs-general tools, including the cogent core `filetree` and the `goal` shell. -Data is represented using the [tensor] package universal data type: the `tensor.Indexed` `Tensor`, which can represent everything from a single scalar value up to n-dimensional collections of patterns, in a range of data types. +Data is represented using the [tensor] package universal data type: the `tensor.Tensor`, which can represent everything from a single scalar value up to n-dimensional collections of patterns, in a range of data types. A given `Data` node is either: * A _Value_, with a tensor encoding its `Data` value. These are terminal "leaves" in the hierarchical data tree, equivalent to "files" in a standard filesystem. diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index fb5e3b4326..7d1c48bf24 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -15,7 +15,7 @@ import ( ) // Data is a single item of data, the "file" or "directory" in the data filesystem. -// Data is represented using the [tensor] package universal data type: the [tensor.Indexed] +// Data is represented using the [tensor] package universal data type: the // [tensor.Tensor], which can represent everything from a single scalar value up to // n-dimensional collections of patterns, in a range of data types. // Directories have an ordered map of items. @@ -31,7 +31,7 @@ type Data struct { // Data is the data value for "files" / "leaves" in the FS, // represented using the universal [tensor] data type of - // [tensor.Indexed], which can represent anything from a scalar + // [tensor.Tensor], which can represent anything from a scalar // to n-dimensional data, in a range of data types. Data tensor.Tensor @@ -57,7 +57,7 @@ func newData(dir *Data, name string) (*Data, error) { return d, err } -// NewScalar returns new scalar Data value(s) (as a [tensor.Indexed]) +// NewScalar returns new scalar Data value(s) (as a [tensor.Tensor]) // of given data type, in given directory. // The names must be unique in the directory. // Returns the first item created, for immediate use of one value. @@ -70,7 +70,7 @@ func NewScalar[T tensor.DataTypes](dir *Data, names ...string) tensor.Tensor { if errors.Log(err) != nil { return nil } - d.Data = tensor.NewIndexed(tsr) + d.Data = tsr if first == nil { first = d.Data } @@ -78,7 +78,7 @@ func NewScalar[T tensor.DataTypes](dir *Data, names ...string) tensor.Tensor { return first } -// NewValue returns a new Data value as a [tensor.Indexed] [tensor.Tensor] +// NewValue returns a new Data value as a [tensor.Tensor] // of given data type and shape sizes, in given directory Data item. // The name must be unique in the directory. func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Tensor { @@ -88,11 +88,11 @@ func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.T if errors.Log(err) != nil { return nil } - d.Data = tensor.NewIndexed(tsr) + d.Data = tsr return d.Data } -// NewOfType returns a new Data value as a [tensor.Indexed] +// NewOfType returns a new Data value as a [tensor.Tensor] // of given reflect.Kind type and shape sizes per dimension, in given directory Data item. // Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. // The name must be unique in the directory. @@ -103,7 +103,7 @@ func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Ten if errors.Log(err) != nil { return nil } - nd.Data = tensor.NewIndexed(tsr) + nd.Data = tsr return nd.Data } diff --git a/tensor/funcs.go b/tensor/funcs.go index f6100f78b7..5865e2fbb0 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -17,15 +17,15 @@ import ( const StringFirstArg = true // Func represents a registered tensor function, which has -// In number of input *tensor.Indexed arguments, and Out -// number of output arguments (typically 1). There can also be an optional +// In number of input Tensor arguments, and Out number of output +// arguments (typically 1). There can also be an optional // string first argument, which is used to specify the name of // another function in some cases (e.g., a stat or metric function). type Func struct { // Name is the original CamelCase Go name for function Name string - // Fun is the function, which must _only_ take some number of *tensor.Indexed + // Fun is the function, which must _only_ take some number of Tensor // args, with an optional first string arg per [StringFirst]. Fun any @@ -60,8 +60,8 @@ func NewFunc(name string, fun any, out int, stringFirst ...bool) (*Func, error) } // Funcs is the global tensor named function registry. -// All functions must be of the form: func(a, b *tensor.Indexed) -// i.e., taking some specific number of Indexed arguments, +// All functions must be of the form: func(a, b tensor.Tensor) +// i.e., taking some specific number of Tensor arguments, // with the number of output vs. input arguments registered. var Funcs map[string]*Func diff --git a/tensor/indexed.go b/tensor/indexed.go index 14d20dc043..6cdac6f251 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -18,13 +18,13 @@ import ( "gonum.org/v1/gonum/mat" ) -// Indexed is an indexed wrapper around a tensor.Tensor that provides a -// specific view onto the Tensor defined by the set of indexes, which -// apply to the outermost row dimension (with default row-major indexing). +// Indexed is an indexed wrapper around another [Tensor] that provides a +// specific view onto the Tensor defined by the set of [Indexed.Indexes], +// which apply to the outermost row dimension (with default row-major indexing). // Sorting and filtering a tensor only requires updating the indexes while // leaving the underlying Tensor alone. // To produce a new [Tensor] that has its raw data actually organized according -// to the indexed order (i.e., the copy function of numpy), call the [NewTensor] method. +// to the indexed order (i.e., the copy function of numpy), call [Indexed.NewTensor]. // Use the [Set]FloatRow[Cell] methods wherever possible, for the most efficient // and natural indirection through the indexes. type Indexed struct { //types:add @@ -38,14 +38,14 @@ type Indexed struct { //types:add Indexes []int } -// NewIndexed returns a new Indexed based on given tensor, +// NewIndexed returns a new [Indexed] view of given tensor, // with optional list of indexes (none / nil = sequential). func NewIndexed(tsr Tensor, idxs ...int) *Indexed { ix := &Indexed{Tensor: tsr, Indexes: idxs} return ix } -// AsIndexed returns the tensor as an Indexed view. +// AsIndexed returns the tensor as an [Indexed[] view. // If it already is one, then it is returned, otherwise it is wrapped. func AsIndexed(tsr Tensor) *Indexed { if ix, ok := tsr.(*Indexed); ok { @@ -178,7 +178,7 @@ func (ix *Indexed) DimSize(dim int) int { } // RowCellSize returns the size of the outermost Row shape dimension -// (via [Indexed.Rows] method), and the size of all the remaining +// (via [Indexed.NumRows] method), and the size of all the remaining // inner dimensions (the "cell" size). func (ix *Indexed) RowCellSize() (rows, cells int) { _, cells = ix.Tensor.RowCellSize() @@ -186,20 +186,10 @@ func (ix *Indexed) RowCellSize() (rows, cells int) { return } -// RowCellIndex returns the direct Values index into underlying tensor -// based on given overall row * cell index. -func (ix *Indexed) RowCellIndex(idx int) (i1d, ri, ci int) { - _, cells := ix.Tensor.RowCellSize() - ri = idx / cells - ci = idx % cells - i1d = ix.RowIndex(ri)*cells + ci - return -} - -// DeleteInvalid deletes all invalid indexes from the list. +// ValidIndexes deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. -func (ix *Indexed) DeleteInvalid() { - if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { +func (ix *Indexed) ValidIndexes() { + if ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { ix.Indexes = nil return } @@ -220,8 +210,8 @@ func (ix *Indexed) Sequential() { //types:add // e.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise // current indexes are left as is. Use Sequential, then IndexesNeeded to ensure // all rows are represented. -func (ix *Indexed) IndexesNeeded() { //types:add - if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { +func (ix *Indexed) IndexesNeeded() { + if ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return } @@ -237,7 +227,7 @@ func (ix *Indexed) IndexesNeeded() { //types:add // ExcludeMissing deletes indexes where the values are missing, as indicated by NaN. // Uses first cell of higher dimensional data. func (ix *Indexed) ExcludeMissing() { //types:add - if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + if ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return } @@ -254,7 +244,7 @@ func (ix *Indexed) ExcludeMissing() { //types:add // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated func (ix *Indexed) Permuted() { - if ix.Tensor == nil || ix.Tensor.DimSize(0) <= 0 { + if ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return } diff --git a/tensor/ranges.go b/tensor/ranges.go new file mode 100644 index 0000000000..ff593c0c9c --- /dev/null +++ b/tensor/ranges.go @@ -0,0 +1,77 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "fmt" + "slices" +) + +// todo: make a version of these functions that takes +// a standard Indexed tensor with n x 3 shape, where the 3 inner values +// specify the Range Start, End, Incr values, across n ranges. +// these would convert to the current Range-based format that does the impl, +// using the Range helper functions, which are also easier and more explicit +// to use in Go code. + +// Range represents a range of values, for extracting slices of data, +// using standard for loop logic with a Start and exclusive End value, +// and an increment: for i := Start; i < End; i += Incr. +// The zero value means all values in the dimension. +type Range struct { + // Starting value. + Start int + + // End value. 0 default = size of relevant dimension. + End int + + // Increment. must be positive, 1 or greater. 0 default = 1. + Incr int +} + +// EndActual is the actual end value given the size of the dimension. +func (rn *Range) EndActual(size int) int { + if rn.End == 0 { + return size + } + return min(rn.End, size) // preserves -1 for no values. +} + +// IncrActual is the actual increment value. +func (rn *Range) IncrActual() int { + return max(1, rn.Incr) +} + +// Size is the number of elements in the actual range given +// size of the dimension. +func (rn *Range) Size(size int) int { + e := rn.EndActual(size) + if e <= rn.Start { + return 0 + } + i := rn.IncrActual() + return (e - rn.Start) / i +} + +// RangeSizes returns a set of sizes applying the ranges, in order, to +// the given dimension sizes. It is important that all dimensions +// are non-zero, otherwise nothing will be included. +// An error is returned if this is the case. +// Dimensions beyond the ranges specified are automatically included. +func RangeSizes(sizes []int, ranges ...Range) ([]int, error) { + nsz := slices.Clone(sizes) + mx := min(len(ranges), len(sizes)) + var zd []int + for i := range mx { + nsz[i] = ranges[i].Size(sizes[i]) + if nsz[i] == 0 { + zd = append(zd, i) + } + } + if len(zd) > 0 { + return nsz, fmt.Errorf("tensor.Shape Slice has zero size for following dimensions: %v", zd) + } + return nsz, nil +} diff --git a/tensor/slice.go b/tensor/slice.go deleted file mode 100644 index 9fba593f46..0000000000 --- a/tensor/slice.go +++ /dev/null @@ -1,186 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tensor - -import ( - "fmt" - "slices" -) - -// Range represents a range of values, for extracting slices of data, -// using standard for loop logic with a Start and exclusive End value, -// and an increment: for i := Start; i < End; i += Incr. -// The zero value means all values in the dimension. -type Range struct { - // Starting value. - Start int - - // End value. 0 default = size of relevant dimension. - End int - - // Increment. must be positive, 1 or greater. 0 default = 1. - Incr int -} - -// EndActual is the actual end value given the size of the dimension. -func (rn *Range) EndActual(size int) int { - if rn.End == 0 { - return size - } - return min(rn.End, size) // preserves -1 for no values. -} - -// IncrActual is the actual increment value. -func (rn *Range) IncrActual() int { - return max(1, rn.Incr) -} - -// Size is the number of elements in the actual range given -// size of the dimension. -func (rn *Range) Size(size int) int { - e := rn.EndActual(size) - if e <= rn.Start { - return 0 - } - i := rn.IncrActual() - return (e - rn.Start) / i -} - -// SliceSize returns a set of sizes applying the ranges, in order, to -// the given dimension sizes. It is important that all dimensions -// are non-zero, otherwise nothing will be included. -// An error is returned if this is the case. -// Dimensions beyond the ranges specified are automatically included. -func SliceSize(sizes []int, ranges ...Range) ([]int, error) { - nsz := slices.Clone(sizes) - mx := min(len(ranges), len(sizes)) - var zd []int - for i := range mx { - nsz[i] = ranges[i].Size(sizes[i]) - if nsz[i] == 0 { - zd = append(zd, i) - } - } - if len(zd) > 0 { - return nsz, fmt.Errorf("tensor.Shape Slice has zero size for following dimensions: %v", zd) - } - return nsz, nil -} - -// note: the only way to allow arbitrary slicing with shared access -// is with a bitmask. but bitmask is not computationally or memory -// efficient, relative to indexes, and it is simpler to only support one. -// also, the need for direct shared access is limited. - -// todo: make a version of these functions that takes -// a standard Indexed tensor with n x 3 shape, where the 3 inner values -// specify the Range Start, End, Incr values, across n ranges. -// these would convert to the current Range-based format that does the impl, -// using the Range helper functions, which are also easier and more explicit -// to use in Go code. - -// Slice extracts a subset of values from the given tensor into the -// output tensor, according to the provided ranges. -// Dimensions beyond the ranges specified are automatically included. -// Unlike the [Tensor.SubSlice] function, the values extracted here are -// copies of the original, not a slice pointer into them, -// which is necessary to allow discontinuous ranges to be extracted. -// Use the [SliceSet] function to copy sliced values back to the original. -func Slice(tsr, out *Indexed, ranges ...Range) error { - sizes := slices.Clone(tsr.Tensor.ShapeInts()) - sizes[0] = tsr.NumRows() // takes into account indexes - nsz, err := SliceSize(sizes, ranges...) - if err != nil { - return err - } - ndim := len(nsz) - out.Tensor.SetShapeInts(nsz...) - out.Sequential() - nl := out.Len() - oc := make([]int, ndim) // orig coords - nr := len(ranges) - for ni := range nl { - nc := out.Tensor.Shape().IndexFrom1D(ni) - for i := range ndim { - c := nc[i] - if i < nr { - r := ranges[i] - oc[i] = r.Start + c*r.IncrActual() - } else { - oc[i] = c - } - } - oc[0] = tsr.RowIndex(oc[0]) - oi := tsr.Tensor.Shape().IndexTo1D(oc...) - if out.Tensor.IsString() { - out.Tensor.SetString1D(tsr.Tensor.String1D(oi), ni) - } else { - out.SetFloat1D(tsr.Float1D(oi), ni) - } - } - return nil -} - -// SliceSet sets values from the slice into the given tensor. -// Slice tensor must have been created with the [Slice] -// function using the same Range sizes (Start offsets -// can be different). -func SliceSet(tsr, slc *Indexed, ranges ...Range) error { - sizes := slices.Clone(tsr.Tensor.ShapeInts()) - sizes[0] = tsr.NumRows() // takes into account indexes - nsz, err := SliceSize(sizes, ranges...) - if err != nil { - return err - } - if slices.Compare(nsz, slc.Tensor.ShapeInts()) != 0 { - return fmt.Errorf("tensor.SliceSet size from ranges is not the same as the slice tensor") - } - ndim := len(nsz) - nl := slc.Len() - oc := make([]int, ndim) // orig coords - nr := len(ranges) - for ni := range nl { - nc := slc.Tensor.Shape().IndexFrom1D(ni) - for i := range ndim { - c := nc[i] - if i < nr { - r := ranges[i] - oc[i] = r.Start + c*r.IncrActual() - } else { - oc[i] = c - } - } - oc[0] = tsr.RowIndex(oc[0]) - oi := tsr.Tensor.Shape().IndexTo1D(oc...) - if slc.Tensor.IsString() { - tsr.Tensor.SetString1D(slc.Tensor.String1D(ni), oi) - } else { - tsr.SetFloat1D(slc.Float1D(ni), oi) - } - } - return nil -} - -// RowCellSplit splits the given tensor into a standard 2D row, cell -// shape at the given split dimension index. All dimensions prior to -// split are collapsed into the row dimension, and from split onward -// form the cells dimension. The resulting tensor is a re-shaped view -// of the original tensor, sharing the same underlying data. -func RowCellSplit(tsr Tensor, split int) Tensor { - sizes := tsr.ShapeInts() - rows := sizes[:split] - cells := sizes[split:] - nr := 1 - for _, r := range rows { - nr *= r - } - nc := 1 - for _, c := range cells { - nc *= c - } - vw := tsr.View() - vw.SetShapeInts(nr, nc) - return vw -} diff --git a/tensor/sliced.go b/tensor/sliced.go new file mode 100644 index 0000000000..47d8252482 --- /dev/null +++ b/tensor/sliced.go @@ -0,0 +1,812 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "log" + "math/rand" + "reflect" + "slices" + "sort" + "strings" + + "cogentcore.org/core/base/metadata" + "cogentcore.org/core/base/slicesx" + "gonum.org/v1/gonum/mat" +) + +// Slice extracts a subset of values from the given tensor into the +// output tensor, according to the provided ranges. +// Dimensions beyond the ranges specified are automatically included. +// Unlike the [Tensor.SubSlice] function, the values extracted here are +// copies of the original, not a slice pointer into them, +// which is necessary to allow discontinuous ranges to be extracted. +// Use the [SliceSet] function to copy sliced values back to the original. +func Slice(tsr, out *Sliced, ranges ...Range) error { + sizes := slices.Clone(tsr.Tensor.ShapeInts()) + sizes[0] = tsr.NumRows() // takes into account indexes + nsz, err := SliceSize(sizes, ranges...) + if err != nil { + return err + } + ndim := len(nsz) + out.Tensor.SetShapeInts(nsz...) + out.Sequential() + nl := out.Len() + oc := make([]int, ndim) // orig coords + nr := len(ranges) + for ni := range nl { + nc := out.Tensor.Shape().IndexFrom1D(ni) + for i := range ndim { + c := nc[i] + if i < nr { + r := ranges[i] + oc[i] = r.Start + c*r.IncrActual() + } else { + oc[i] = c + } + } + oc[0] = tsr.RowIndex(oc[0]) + oi := tsr.Tensor.Shape().IndexTo1D(oc...) + if out.Tensor.IsString() { + out.Tensor.SetString1D(tsr.Tensor.String1D(oi), ni) + } else { + out.SetFloat1D(tsr.Float1D(oi), ni) + } + } + return nil +} + +// Sliced is a fully indexed wrapper around another [Tensor] that provides a +// re-sliced view onto the Tensor defined by the set of [SlicedIndexes], +// for each dimension (must have at least 1 per dimension). +// Thus, every dimension can be transformed in arbitrary ways relative +// to the original tensor. There is some additional cost for every +// access operation associated with the additional indexed indirection. +// See also [Indexed] for a version that only indexes the outermost row dimension, +// which is much more efficient for this common use-case. +// To produce a new [Tensor] that has its raw data actually organized according +// to the indexed order (i.e., the copy function of numpy), call [Sliced.NewTensor]. +type Sliced struct { //types:add + + // Tensor that we are an indexed view onto. + Tensor Tensor + + // Indexes are the indexes for each dimension, with dimensions as the outer + // slice (enforced to be the same length as the NumDims of the source Tensor), + // and a list of dimension index values (within range of DimSize(d)). + // A nil list of indexes automatically provides a full, sequential view of that + // dimension. + Indexes [][]int +} + +// NewSliced returns a new [Sliced] view of given tensor, +// with optional list of indexes (none / nil = sequential). +func NewSliced(tsr Tensor, idxs ...[]int) *Sliced { + sl := &Sliced{Tensor: tsr, Indexes: idxs} + sl.ValidIndexes() + return sl +} + +// AsSliced returns the tensor as a [Sliced] view. +// If it already is one, then it is returned, otherwise it is wrapped. +func AsSliced(tsr Tensor) *Sliced { + if sl, ok := tsr.(*Sliced); ok { + return sl + } + return NewSliced(tsr) +} + +// SetTensor sets as indexes into given tensor with sequential initial indexes. +func (sl *Sliced) SetTensor(tsr Tensor) { + sl.Tensor = tsr + sl.Sequential() +} + +// ValidIndexes ensures that [Sliced.Indexes] are valid, +// removing any out-of-range values and setting the view to nil (full sequential) +// for any dimension with no indexes (which is an invalid condition). +// Call this when any structural changes are made to underlying Tensor. +func (sl *Sliced) ValidIndexes() { + nd := sl.Tensor.NumDims() + sl.Indexes = slicesx.SetLength(sl.Indexes, nd) + for d := range nd { + ni := len(sl.Indexes[d]) + if ni == 0 { // invalid + sl.Indexes[d] = nil // full + continue + } + ds := sl.Tensor.DimSize(d) + ix := sl.Indexes[d] + for i := ni - 1; i >= 0; i-- { + if ix[i] >= ds { + ix = append(ix[:i], ix[i+1:]...) + } + } + sl.Indexes[d] = ix + } +} + +// Sequential sets all Indexes to nil, resulting in full sequential access into tensor. +func (sl *Sliced) Sequential() { //types:add + nd := sl.Tensor.NumDims() + sl.Indexes = slicesx.SetLength(sl.Indexes, nd) + for d := range nd { + sl.Indexes[d] = nil + } +} + +// IndexesNeeded is called prior to an operation that needs actual indexes, +// on given dimension. If Indexes == nil, they are set to all items, otherwise +// current indexes are left as is. Use Sequential, then IndexesNeeded to ensure +// all dimension indexes are represented. +func (sl *Sliced) IndexesNeeded(d int) { + ix := sl.Indexes[d] + if ix != nil { + return + } + ix = make([]int, sl.Tensor.DimSize(d)) + for i := range ix { + ix[i] = i + } + sl.Indexes[d] = ix +} + +// Label satisfies the core.Labeler interface for a summary description of the tensor. +func (sl *Sliced) Label() string { + return sl.Tensor.Label() +} + +// Metadata returns the metadata for this tensor, which can be used +// to encode plotting options, etc. +func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } + +// For each dimension, we return the effective shape sizes using +// the current number of indexes per dimension. +func (sl *Sliced) ShapeInts() []int { + nd := sl.Tensor.NumDims() + if nd == 0 { + return sl.Tensor.ShapeInts() + } + sh := slices.Clone(sl.Tensor.ShapeInts()) + for d := range nd { + if sl.Indexes[d] != nil { + sh[d] = len(sl.Indexes[d]) + } + } + return sh +} + +func (sl *Sliced) ShapeSizes() Tensor { + return NewIntFromSlice(sl.ShapeInts()...) +} + +// Shape() returns a [Shape] representation of the tensor shape +// (dimension sizes). If we have Indexes, this is the effective +// shape using the current number of indexes per dimension. +func (sl *Sliced) Shape() *Shape { + return NewShape(sl.ShapeInts()...) +} + +// SetShapeInts sets the shape of the underlying wrapped tensor +// to the given sizes per dimension, and resets our indexes +// which are now invalid. +func (sl *Sliced) SetShapeInts(sizes ...int) { + sl.Tensor.SetShapeInts(sizes...) + sl.Sequential() +} + +// SetShape sets our shape to given sizes. +// See [Sliced.SetShapeInts] for details. +func (sl *Sliced) SetShape(sizes Tensor) { + sl.SetShapeInts(AsIntSlice(sizes)...) +} + +// Len returns the total number of elements in our view of the tensor. +func (sl *Sliced) Len() int { + return sl.Shape().Len() +} + +// NumDims returns the total number of dimensions. +func (sl *Sliced) NumDims() int { return sl.Tensor.NumDims() } + +// DimSize returns the effective view size of given dimension. +func (sl *Sliced) DimSize(dim int) int { + if sl.Indexes[dim] != nil { + return len(sl.Indexes[dim]) + } + return sl.Tensor.DimSize(dim) +} + +// RowCellSize returns the size of the outermost Row shape dimension +// (via [Sliced.Rows] method), and the size of all the remaining +// inner dimensions (the "cell" size). +func (sl *Sliced) RowCellSize() (rows, cells int) { + rows = sl.DimSize(0) + nd := sl.Tensor.NumDims() + if nd == 1 { + cells = 1 + } else if rows > 0 { + cells = sl.Len() / rows + } else { + ln := 1 + for d := 1; d < nd; d++ { + ln *= sl.DimSize() + } + cells = ln + } + return +} + +// Permuted sets indexes in given dimension to a permuted order. +// If indexes already exist then existing list of indexes is permuted, +// otherwise a new set of permuted indexes are generated +func (sl *Sliced) Permuted(dim int) { + ix := sl.Indexes[dim] + if ix == nil { + ix = rand.Perm(sl.Tensor.DimSize(dim)) + } else { + rand.Shuffle(len(ix), func(i, j int) { + ix[i], ix[j] = ix[j], ix[i] + }) + } + sl.Indexes[dim] = ix +} + +// SortFunc sorts the indexes along given dimension using given compare function. +// The compare function operates directly on indexes into the Tensor +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +func (sl *Sliced) SortFunc(dim int, cmp func(tsr Tensor, dim, i, j int) int) { + sl.IndexesNeeded(dim) + ix := sl.Indexes[dim] + slices.SortFunc(ix, func(a, b int) int { + return cmp(sl.Tensor, dim, a, b) // key point: these are already indirected through indexes!! + }) + sl.Indexes[dim] = ix +} + +// SortIndexes sorts the indexes along given dimension directly in +// numerical order, producing the native ordering, while preserving +// any filtering that might have occurred. +func (sl *Sliced) SortIndexes(dim int) { + ix := sl.Indexes[dim] + if ix == nil { + return + } + sort.Ints(ix) + sl.Indexes[dim] = ix +} + +// Sort does default alpha or numeric sort along given dimension of data. +func (sl *Sliced) Sort(dim int, ascending bool) { + if sl.Tensor.IsString() { + sl.SortFunc(dim, func(tsr Tensor, dim, i, j int) int { + return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) + }) + } else { + sl.SortFunc(func(tsr Tensor, i, j int) int { + return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) + }) + } +} + +// SortStableFunc stably sorts the row-wise indexes using given compare function. +// The compare function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +// It is *essential* that it always returns 0 when the two are equal +// for the stable function to actually work. +func (sl *Sliced) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { + sl.IndexesNeeded() + slices.SortStableFunc(sl.Indexes, func(a, b int) int { + return cmp(sl.Tensor, a, b) // key point: these are already indirected through indexes!! + }) +} + +// SortStable does stable default alpha or numeric sort. +// Uses first cell of higher dimensional data. +func (sl *Sliced) SortStable(ascending bool) { + if sl.Tensor.IsString() { + sl.SortStableFunc(func(tsr Tensor, i, j int) int { + return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) + }) + } else { + sl.SortStableFunc(func(tsr Tensor, i, j int) int { + return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) + }) + } +} + +// Filter filters the indexes using given Filter function. +// The Filter function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +func (sl *Sliced) Filter(filterer func(tsr Tensor, row int) bool) { + sl.IndexesNeeded() + sz := len(sl.Indexes) + for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering + if !filterer(sl.Tensor, sl.Indexes[i]) { // delete + sl.Indexes = append(sl.Indexes[:i], sl.Indexes[i+1:]...) + } + } +} + +// FilterString filters the indexes using string values compared to given +// string. Includes rows with matching values unless exclude is set. +// If contains, only checks if row contains string; if ignoreCase, ignores case. +// Use the named const args [Include], [Exclude], [Contains], [Equals], +// [IgnoreCase], [UseCase] for greater clarity. +// Uses first cell of higher dimensional data. +func (sl *Sliced) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add + lowstr := strings.ToLower(str) + sl.Filter(func(tsr Tensor, row int) bool { + val := tsr.StringRowCell(row, 0) + has := false + switch { + case contains && ignoreCase: + has = strings.Contains(strings.ToLower(val), lowstr) + case contains: + has = strings.Contains(val, str) + case ignoreCase: + has = strings.EqualFold(val, str) + default: + has = (val == str) + } + if exclude { + return !has + } + return has + }) +} + +// NewTensor returns a new tensor with column data organized according to +// the Indexes. If Indexes are nil, a clone of the current tensor is returned +// but this function is only sensible if there is an indexed view in place. +func (sl *Sliced) NewTensor() Tensor { + nt := sl.Tensor.Clone() + if sl.Indexes == nil { + return nt + } + rows := len(sl.Indexes) + nt.SetNumRows(rows) + _, cells := sl.Tensor.RowCellSize() + str := sl.Tensor.IsString() + for r := range rows { + for c := range cells { + if str { + nt.SetStringRowCell(sl.StringRowCell(r, c), r, c) + } else { + nt.SetFloatRowCell(sl.FloatRowCell(r, c), r, c) + } + } + } + return nt +} + +// Clone returns a copy of the current Sliced view with a cloned copy of +// the underlying Tensor and copy of the indexes. +func (sl *Sliced) Clone() Tensor { + nix := &Sliced{} + nix.Tensor = sl.Tensor.Clone() + nix.CopyIndexes(sl) + return nix +} + +func (sl *Sliced) View() Tensor { + nix := &Sliced{} + nix.Tensor = sl.Tensor.View() + nix.CopyIndexes(sl) + return nix +} + +// CloneIndexes returns a copy of the current Sliced view with new indexes, +// with a pointer to the same underlying Tensor as the source. +func (sl *Sliced) CloneIndexes() *Sliced { + nix := &Sliced{} + nix.Tensor = sl.Tensor + nix.CopyIndexes(sl) + return nix +} + +// CopyIndexes copies indexes from other Sliced view. +func (sl *Sliced) CopyIndexes(oix *Sliced) { + if oix.Indexes == nil { + sl.Indexes = nil + } else { + sl.Indexes = slices.Clone(oix.Indexes) + } +} + +// AddRows adds n rows to end of underlying Tensor, and to the indexes in this view +func (sl *Sliced) AddRows(n int) { //types:add + stidx := sl.Tensor.DimSize(0) + sl.Tensor.SetNumRows(stidx + n) + if sl.Indexes != nil { + for i := stidx; i < stidx+n; i++ { + sl.Indexes = append(sl.Indexes, i) + } + } +} + +// InsertRows adds n rows to end of underlying Tensor, and to the indexes starting at +// given index in this view +func (sl *Sliced) InsertRows(at, n int) { + stidx := sl.Tensor.DimSize(0) + sl.IndexesNeeded() + sl.Tensor.SetNumRows(stidx + n) + nw := make([]int, n, n+len(sl.Indexes)-at) + for i := 0; i < n; i++ { + nw[i] = stidx + i + } + sl.Indexes = append(sl.Indexes[:at], append(nw, sl.Indexes[at:]...)...) +} + +// DeleteRows deletes n rows of indexes starting at given index in the list of indexes +func (sl *Sliced) DeleteRows(at, n int) { + sl.IndexesNeeded() + sl.Indexes = append(sl.Indexes[:at], sl.Indexes[at+n:]...) +} + +// Swap switches the indexes for i and j +func (sl *Sliced) Swap(i, j int) { + if sl.Indexes == nil { + return + } + sl.Indexes[i], sl.Indexes[j] = sl.Indexes[j], sl.Indexes[i] +} + +/////////////////////////////////////////////// +// Sliced access + +///////////////////// Floats + +// Float returns the value of given index as a float64. +// The first index value is indirected through the indexes. +func (sl *Sliced) Float(i ...int) float64 { + if sl.Indexes == nil { + return sl.Tensor.Float(i...) + } + ic := slices.Clone(i) + ic[0] = sl.Indexes[ic[0]] + return sl.Tensor.Float(ic...) +} + +// SetFloat sets the value of given index as a float64 +// The first index value is indirected through the [Sliced.Indexes]. +func (sl *Sliced) SetFloat(val float64, i ...int) { + if sl.Indexes == nil { + sl.Tensor.SetFloat(val, i...) + return + } + ic := slices.Clone(i) + ic[0] = sl.Indexes[ic[0]] + sl.Tensor.SetFloat(val, ic...) +} + +// FloatRowCell returns the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Sliced.Indexes]. +// This is the preferred interface for all Sliced operations. +func (sl *Sliced) FloatRowCell(row, cell int) float64 { + return sl.Tensor.FloatRowCell(sl.RowIndex(row), cell) +} + +// SetFloatRowCell sets the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Sliced.Indexes]. +// This is the preferred interface for all Sliced operations. +func (sl *Sliced) SetFloatRowCell(val float64, row, cell int) { + sl.Tensor.SetFloatRowCell(val, sl.RowIndex(row), cell) +} + +// Float1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (sl *Sliced) Float1D(i int) float64 { + if sl.Indexes == nil { + return sl.Tensor.Float1D(i) + } + return sl.Float(sl.Tensor.Shape().IndexFrom1D(i)...) +} + +// SetFloat1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (sl *Sliced) SetFloat1D(val float64, i int) { + if sl.Indexes == nil { + sl.Tensor.SetFloat1D(val, i) + } + sl.SetFloat(val, sl.Tensor.Shape().IndexFrom1D(i)...) +} + +func (sl *Sliced) FloatRow(row int) float64 { + return sl.FloatRowCell(row, 0) +} + +func (sl *Sliced) SetFloatRow(val float64, row int) { + sl.SetFloatRowCell(val, row, 0) +} + +///////////////////// Strings + +// StringValue returns the value of given index as a string. +// The first index value is indirected through the indexes. +func (sl *Sliced) StringValue(i ...int) string { + if sl.Indexes == nil { + return sl.Tensor.StringValue(i...) + } + ic := slices.Clone(i) + ic[0] = sl.Indexes[ic[0]] + return sl.Tensor.StringValue(ic...) +} + +// SetString sets the value of given index as a string +// The first index value is indirected through the [Sliced.Indexes]. +func (sl *Sliced) SetString(val string, i ...int) { + if sl.Indexes == nil { + sl.Tensor.SetString(val, i...) + } + ic := slices.Clone(i) + ic[0] = sl.Indexes[ic[0]] + sl.Tensor.SetString(val, ic...) +} + +// StringRowCell returns the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Sliced.Indexes]. +// This is the preferred interface for all Sliced operations. +func (sl *Sliced) StringRowCell(row, cell int) string { + return sl.Tensor.StringRowCell(sl.RowIndex(row), cell) +} + +// SetStringRowCell sets the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Sliced.Indexes]. +// This is the preferred interface for all Sliced operations. +func (sl *Sliced) SetStringRowCell(val string, row, cell int) { + sl.Tensor.SetStringRowCell(val, sl.RowIndex(row), cell) +} + +// String1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (sl *Sliced) String1D(i int) string { + if sl.Indexes == nil { + return sl.Tensor.String1D(i) + } + return sl.StringValue(sl.Tensor.Shape().IndexFrom1D(i)...) +} + +// SetString1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (sl *Sliced) SetString1D(val string, i int) { + if sl.Indexes == nil { + sl.Tensor.SetString1D(val, i) + } + sl.SetString(val, sl.Tensor.Shape().IndexFrom1D(i)...) +} + +func (sl *Sliced) StringRow(row int) string { + return sl.StringRowCell(row, 0) +} + +func (sl *Sliced) SetStringRow(val string, row int) { + sl.SetStringRowCell(val, row, 0) +} + +///////////////////// Ints + +// Int returns the value of given index as an int. +// The first index value is indirected through the indexes. +func (sl *Sliced) Int(i ...int) int { + if sl.Indexes == nil { + return sl.Tensor.Int(i...) + } + ic := slices.Clone(i) + ic[0] = sl.Indexes[ic[0]] + return sl.Tensor.Int(ic...) +} + +// SetInt sets the value of given index as an int +// The first index value is indirected through the [Sliced.Indexes]. +func (sl *Sliced) SetInt(val int, i ...int) { + if sl.Indexes == nil { + sl.Tensor.SetInt(val, i...) + return + } + ic := slices.Clone(i) + ic[0] = sl.Indexes[ic[0]] + sl.Tensor.SetInt(val, ic...) +} + +// IntRowCell returns the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Sliced.Indexes]. +// This is the preferred interface for all Sliced operations. +func (sl *Sliced) IntRowCell(row, cell int) int { + return sl.Tensor.IntRowCell(sl.RowIndex(row), cell) +} + +// SetIntRowCell sets the value at given row and cell, +// where row is outermost dim, and cell is 1D index into remaining inner dims. +// Row is indirected through the [Sliced.Indexes]. +// This is the preferred interface for all Sliced operations. +func (sl *Sliced) SetIntRowCell(val int, row, cell int) { + sl.Tensor.SetIntRowCell(val, sl.RowIndex(row), cell) +} + +// Int1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (sl *Sliced) Int1D(i int) int { + if sl.Indexes == nil { + return sl.Tensor.Int1D(i) + } + return sl.Int(sl.Tensor.Shape().IndexFrom1D(i)...) +} + +// SetInt1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (sl *Sliced) SetInt1D(val int, i int) { + if sl.Indexes == nil { + sl.Tensor.SetInt1D(val, i) + } + sl.SetInt(val, sl.Tensor.Shape().IndexFrom1D(i)...) +} + +func (sl *Sliced) IntRow(row int) int { + return sl.IntRowCell(row, 0) +} + +func (sl *Sliced) SetIntRow(val int, row int) { + sl.SetIntRowCell(val, row, 0) +} + +///////////////////// SubSpaces + +// SubSpace returns a new tensor with innermost subspace at given +// offset(s) in outermost dimension(s) (len(offs) < NumDims). +// The new tensor points to the values of the this tensor (i.e., modifications +// will affect both), as its Values slice is a view onto the original (which +// is why only inner-most contiguous supsaces are supported). +// Use Clone() method to separate the two. +// Sliced version does indexed indirection of the outermost row dimension +// of the offsets. +func (sl *Sliced) SubSpace(offs ...int) Tensor { + if len(offs) == 0 { + return nil + } + offs[0] = sl.RowIndex(offs[0]) + return sl.Tensor.SubSpace(offs...) +} + +// RowTensor is a convenience version of [Sliced.SubSpace] to return the +// SubSpace for the outermost row dimension, indirected through the indexes. +func (sl *Sliced) RowTensor(row int) Tensor { + return sl.Tensor.RowTensor(sl.RowIndex(row)) +} + +// SetRowTensor sets the values of the SubSpace at given row to given values, +// with row indirected through the indexes. +func (sl *Sliced) SetRowTensor(val Tensor, row int) { + sl.Tensor.SetRowTensor(val, sl.RowIndex(row)) +} + +// CopyFrom copies all values from other tensor into this tensor. +// Checks if source is an Sliced and copies indexes too, +// otherwise underlying tensor copies from and indexes are reset. +func (sl *Sliced) CopyFrom(from Tensor) { + if fix, ok := from.(*Sliced); ok { + sl.Tensor.CopyFrom(fix.Tensor) + sl.CopyIndexes(fix) + return + } + sl.Sequential() + sl.Tensor.CopyFrom(from) +} + +// AppendFrom appends all values from other tensor into this tensor. +// This invalidates the indexes which are reset. +func (sl *Sliced) AppendFrom(from Tensor) error { + sl.Sequential() + return sl.Tensor.AppendFrom(from) +} + +// CopyCellsFrom copies given range of values from other tensor into this tensor, +// This invalidates the indexes which are reset. +func (sl *Sliced) CopyCellsFrom(from Tensor, to, start, n int) { + sl.Sequential() + sl.Tensor.CopyCellsFrom(from, to, start, n) +} + +func (sl *Sliced) Sizeof() int64 { + return sl.Tensor.Sizeof() // todo: could be out of sync with shape! +} + +func (sl *Sliced) Bytes() []byte { + return sl.Tensor.Bytes() // todo: could be out of sync with shape! +} + +func (sl *Sliced) IsString() bool { + return sl.Tensor.IsString() +} + +func (sl *Sliced) DataType() reflect.Kind { + return sl.Tensor.DataType() +} + +func (sl *Sliced) Range() (min, max float64, minIndex, maxIndex int) { + return sl.Tensor.Range() +} + +func (sl *Sliced) SetZeros() { + sl.Tensor.SetZeros() +} + +////////////////////////// gonum matrix api + +// Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the +// 2D Matrix. Assumes Row-major ordering and logs an error if NumDims < 2. +func (sl *Sliced) Dims() (r, c int) { + nd := sl.NumDims() + if nd < 2 { + log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") + return 0, 0 + } + return sl.DimSize(nd - 2), sl.DimSize(nd - 1) +} + +// Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric +// 2D Matrix. +func (sl *Sliced) Symmetric() (r int) { + nd := sl.NumDims() + if nd < 2 { + log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") + return 0 + } + if sl.DimSize(nd-2) != sl.DimSize(nd-1) { + log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") + return 0 + } + return sl.DimSize(nd - 1) +} + +// SymmetricDim returns the number of rows/columns in the matrix. +func (sl *Sliced) SymmetricDim() int { + nd := sl.NumDims() + if nd < 2 { + log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") + return 0 + } + if sl.DimSize(nd-2) != sl.DimSize(nd-1) { + log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") + return 0 + } + return sl.DimSize(nd - 1) +} + +// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given +// row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. +func (sl *Sliced) At(i, j int) float64 { + nd := sl.NumDims() + if nd < 2 { + log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") + return 0 + } else if nd == 2 { + return sl.Float(i, j) + } else { + nix := make([]int, nd) + nix[nd-2] = i + nix[nd-1] = j + return sl.Float(nix...) + } +} + +// T is the gonum/mat.Matrix transpose method. +// It performs an implicit transpose by returning the receiver inside a Transpose. +func (sl *Sliced) T() mat.Matrix { + return mat.Transpose{sl} +} + +// check for interface impl +var _ Tensor = (*Sliced)(nil) diff --git a/tensor/stats/cluster/plot.go b/tensor/stats/cluster/plot.go index e253b2236e..6ab209ef4c 100644 --- a/tensor/stats/cluster/plot.go +++ b/tensor/stats/cluster/plot.go @@ -12,7 +12,7 @@ import ( // Plot sets the rows of given data table to trace out lines with labels that // will render cluster plot starting at root node when plotted with a standard plotting package. // The lines double-back on themselves to form a continuous line to be plotted. -func Plot(pt *table.Table, root *Node, dmat, labels *tensor.Indexed) { +func Plot(pt *table.Table, root *Node, dmat, labels tensor.Tensor) { pt.DeleteAll() pt.AddFloat64Column("X") pt.AddFloat64Column("Y") @@ -26,7 +26,7 @@ func Plot(pt *table.Table, root *Node, dmat, labels *tensor.Indexed) { // Plot sets the rows of given data table to trace out lines with labels that // will render this node in a cluster plot when plotted with a standard plotting package. // The lines double-back on themselves to form a continuous line to be plotted. -func (nn *Node) Plot(pt *table.Table, dmat, labels *tensor.Indexed) { +func (nn *Node) Plot(pt *table.Table, dmat, labels tensor.Tensor) { row := pt.NumRows() xc := pt.ColumnByIndex(0) yc := pt.ColumnByIndex(1) diff --git a/tensor/stats/metric/doc.go b/tensor/stats/metric/doc.go index 7c7496dd4c..2375767852 100644 --- a/tensor/stats/metric/doc.go +++ b/tensor/stats/metric/doc.go @@ -3,6 +3,6 @@ // license that can be found in the LICENSE file. /* -Package metric provides various similarity / distance metrics for comparing tensors, operating on the tensor.Indexed standard data representation. +Package metric provides various similarity / distance metrics for comparing tensors, operating on the tensor.Tensor standard data representation. */ package metric diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index da5262e2a2..61350bd229 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -1,8 +1,8 @@ # stats -The `stats` package provides standard statistic computations operating on the `tensor.Indexed` standard data representation, using this standard function: +The `stats` package provides standard statistic computations operating on the `tensor.Tensor` standard data representation, using this standard function: ```Go -type StatsFunc func(in, out *tensor.Indexed) +type StatsFunc func(in, out tensor.Tensor) ``` For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outermost row dimension set to 1. diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 8542d1d020..2cd13bfa68 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -56,9 +56,9 @@ func (dt *Table) IndexesFromTensor(ix *tensor.Indexed) { dt.Indexes = ix.Indexes } -// DeleteInvalid deletes all invalid indexes from the list. +// ValidIndexes deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from table. -func (dt *Table) DeleteInvalid() { +func (dt *Table) ValidIndexes() { if dt.Columns.Rows <= 0 || dt.Indexes == nil { dt.Indexes = nil return diff --git a/tensor/table/table.go b/tensor/table/table.go index f666301795..f424dd8d91 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -19,7 +19,7 @@ import ( // Use the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a // [tensor.Indexed] view of the column, using the shared [Table.Indexes] of the Table. // Thus, a coordinated sorting and filtered view of the column data is automatically -// available for any of the tensor package functions that use [tensor.Indexed] as the one +// available for any of the tensor package functions that use [tensor.Tensor] as the one // common data representation for all operations. type Table struct { //types:add // Columns has the list of column tensor data for this table. @@ -275,7 +275,7 @@ func (dt *Table) SetNumRows(rows int) *Table { //types:add dt.Indexes = append(dt.Indexes, strow+i) } } else { - dt.DeleteInvalid() + dt.ValidIndexes() } return dt } diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index a4a6aaafbd..830c092a6b 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -183,7 +183,7 @@ func (tb *Table) SetIndexed(ix *table.Table) *Table { } func (tb *Table) UpdateSliceSize() int { - tb.Table.DeleteInvalid() // table could have changed + tb.Table.ValidIndexes() // table could have changed if tb.Table.NumRows() == 0 { tb.Table.Sequential() } diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 40ae77f834..42bd1958f3 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -32,15 +32,16 @@ func TestMath(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) + oned := tensor.NewNumberFromSlice(vals...) oneout := oned.Clone() - cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) + cell2d := tensor.NewFloat32(5, 2, 6) + _, cells := cell2d.RowCellSize() + assert.Equal(t, cells, 12) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - i, _, ci := cell2d.RowCellIndex(idx) - cell2d.SetFloat1D(oned.Float1D(ci), i) + ci := idx % cells + cell2d.SetFloat1D(oned.Float1D(ci), idx) }, cell2d) - // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() mfuncs := []onef{math.Abs, math.Acos, math.Acosh, math.Asin, math.Asinh, math.Atan, math.Atanh, math.Cbrt, math.Ceil, math.Cos, math.Cosh, math.Erf, math.Erfc, math.Erfcinv, math.Erfinv, math.Exp, math.Exp2, math.Expm1, math.Floor, math.Gamma, math.J0, math.J1, math.Log, math.Log10, math.Log1p, math.Log2, math.Logb, math.Round, math.RoundToEven, math.Sin, math.Sinh, math.Sqrt, math.Tan, math.Tanh, math.Trunc, math.Y0, math.Y1} diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index a43b9d5028..bbc90d1959 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -20,13 +20,15 @@ func TestAdd(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) + oned := tensor.NewNumberFromSlice(vals...) oneout := oned.Clone() - cell2d := tensor.NewIndexed(tensor.NewFloat32(5, 2, 6)) + cell2d := tensor.NewFloat32(5, 2, 6) + _, cells := cell2d.RowCellSize() + assert.Equal(t, cells, 12) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - i, _, ci := cell2d.RowCellIndex(idx) - cell2d.SetFloat1D(oned.Float1D(ci), i) + ci := idx % cells + cell2d.SetFloat1D(oned.Float1D(ci), idx) }, cell2d) // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() From 44c3d2733efb593db874d496ebc1896d58bff6dc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 20 Sep 2024 16:21:55 -0700 Subject: [PATCH 083/311] replaced Indexed -> Rows --- goal/README.md | 8 +- goal/transpile/math.go | 2 +- goal/transpile/transpile_test.go | 2 +- plot/plotcore/ploteditor.go | 10 +- plot/plotcore/tablexy.go | 4 +- tensor/README.md | 16 +- tensor/base.go | 20 +-- tensor/bits.go | 2 +- tensor/broadcast.go | 5 + tensor/datafs/dir.go | 2 +- tensor/funcs.go | 2 +- tensor/number.go | 4 +- tensor/ranges.go | 2 +- tensor/{indexed.go => rows.go} | 241 +++++++++++++++---------------- tensor/sliced.go | 175 ++++++---------------- tensor/stats/glm/README.md | 2 +- tensor/stats/metric/README.md | 2 +- tensor/stats/metric/matrix.go | 2 - tensor/stats/stats/README.md | 6 +- tensor/stats/stats/doc.go | 10 +- tensor/stats/stats/funcs.go | 2 +- tensor/stats/stats/group.go | 10 +- tensor/stats/stats/stats_test.go | 5 +- tensor/string.go | 2 +- tensor/table/README.md | 10 +- tensor/table/indexes.go | 4 +- tensor/table/table.go | 22 +-- tensor/tensor.go | 20 +-- tensor/tensorcore/table.go | 24 +-- tensor/tmath/README.md | 2 +- tensor/tmath/doc.go | 2 +- tensor/tmath/norm.go | 4 +- tensor/tmath/ops_test.go | 2 +- tensor/vectorize.go | 2 +- 34 files changed, 252 insertions(+), 376 deletions(-) rename tensor/{indexed.go => rows.go} (77%) diff --git a/goal/README.md b/goal/README.md index 52d47d2452..285030c0db 100644 --- a/goal/README.md +++ b/goal/README.md @@ -37,7 +37,7 @@ for _, x := range #[1,2,3]# { } ``` -In general, the math mode syntax in _Goal_ is designed to be as compatible with Python NumPy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a _Goal_ math expression are [tensors](../tensor), specifically `*tensor.Indexed`, which can represent everything from a scalar to an n-dimenstional tensor. These are called an "array" in NumPy terms. +In general, the math mode syntax in _Goal_ is designed to be as compatible with Python NumPy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a _Goal_ math expression are [tensors](../tensor), which can represent everything from a scalar to an n-dimenstional tensor. These are called an "ndarray" in NumPy terms. The rationale and mnemonics for using `$` and `#` are as follows: @@ -225,13 +225,15 @@ In general, _Goal_ is designed to be as compatible with Python NumPy / SciPy syn All elements of a _Goal_ math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that _Goal_ does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately. -The _view_ versions of `Tensor` include `tensor.Sliced`, `tensor.Masked`, and `tensor.Indexed`, each of which wraps around another "source" `Tensor`, and provides its own way of accessing the underlying data: +The _view_ versions of `Tensor` include `tensor.Sliced`, `tensor.Rows`, `tensor.Masked`, and `tensor.Indexed`, each of which wraps around another "source" `Tensor`, and provides its own way of accessing the underlying data: * `Sliced` has an arbitrary set of indexes for each dimension, so access to values along that dimension go through the indexes. Thus, you could reverse the order of the columns (dimension 1), or only operate on a subset of them. +* `Rows` is an optimized version of `Sliced` with indexes only for the first, outermost, _row_ dimension, which + * `Masked` has a `tensor.Bool` tensor that filters access to the underlying source tensor through a mask: anywhere the bool value is `false`, the corresponding source value is not settable, and returns `NaN` (missing value) when accessed. -* `Indexed` is an optimized version of `Sliced` with indexes only for the first, outermost, _row_ dimension, which +* `Indexed` uses a tensor of indexes where the final, innermost dimension is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 03018be454..fe6d2306d4 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -341,7 +341,7 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { } var numpyFuncs = map[string]funWrap{ - "zeros": {"tensor.NewFloat64Indexed", ""}, + "zeros": {"tensor.NewFloat64", ""}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 33b059513b..7cafac4bd0 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -206,7 +206,7 @@ func TestMath(t *testing.T) { // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, // {"# a := [1,2,3,4]", `a := tensor.NewIntFromSlice([]int { 1, 2, 3, 4 } ...)`}, // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64Indexed(3, 4)`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64Tensor(3, 4)`}, } st := NewState() diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 95febfe8f6..91a4cd9d6a 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -69,7 +69,7 @@ func (pl *PlotEditor) CopyFieldsFrom(frm tree.Node) { fr := frm.(*PlotEditor) pl.Frame.CopyFieldsFrom(&fr.Frame) pl.Options = fr.Options - pl.setIndexed(fr.table) + pl.setTable(fr.table) mx := min(len(pl.Columns), len(fr.Columns)) for i := 0; i < mx; i++ { *pl.Columns[i] = *fr.Columns[i] @@ -133,11 +133,11 @@ func (pl *PlotEditor) Init() { }) } -// setIndexed sets the table to view and does Update +// setTable sets the table to view and does Update // to update the Column list, which will also trigger a Layout // and updating of the plot on next render pass. // This is safe to call from a different goroutine. -func (pl *PlotEditor) setIndexed(tab *table.Table) *PlotEditor { +func (pl *PlotEditor) setTable(tab *table.Table) *PlotEditor { pl.table = tab pl.Update() return pl @@ -273,7 +273,7 @@ func (pl *PlotEditor) xLabel() string { return "X" } -// GoUpdatePlot updates the display based on current Indexed into table. +// GoUpdatePlot updates the display based on current Indexed view into table. // This version can be called from goroutines. It does Sequential() on // the [table.Table], under the assumption that it is used for tracking a // the latest updates of a running process. @@ -294,7 +294,7 @@ func (pl *PlotEditor) GoUpdatePlot() { pl.Scene.AsyncUnlock() } -// UpdatePlot updates the display based on current Indexed into table. +// UpdatePlot updates the display based on current Indexed view into table. // It does not automatically update the [table.Table] unless it is // nil or out date. func (pl *PlotEditor) UpdatePlot() { diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index e8e4e647f1..060daeac37 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -47,7 +47,7 @@ var _ plot.Valuer = &tableXY{} var _ plot.Labeler = &tableXY{} var _ plots.YErrorer = &tableXY{} -// newTableXY returns a new XY plot view onto the given Indexed of table.Table (makes a copy), +// newTableXY returns a new XY plot view onto the given Indexed view of table.Table (makes a copy), // from given column indexes, and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. func newTableXY(dt *table.Table, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { @@ -55,7 +55,7 @@ func newTableXY(dt *table.Table, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minm return txy, txy.validate() } -// newTableXYName returns a new XY plot view onto the given Indexed of table.Table (makes a copy), +// newTableXYName returns a new XY plot view onto the given Indexed view of table.Table (makes a copy), // from given column name and tensor indexes within each cell. // Column indexes are enforced to be valid, with an error message if they are not. func newTableXYName(dt *table.Table, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { diff --git a/tensor/README.md b/tensor/README.md index ca68cc4646..8ff75f3b33 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -50,23 +50,23 @@ All tensor package functions are registered using a global name-to-function map In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: -* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one dimension. `Indexed` views of this 1D data provide fine-grained filtering and sorting of all the data. Any `Tensor` can be accessed via a flat 1D index, which goes directly into the underlying Go slice for the basic types, and is appropriately (though somewhat expensively in some cases) indirected through the effective geometry in `Sliced` and `Indexed` types. +* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one dimension. `Rows` views of this 1D data provide fine-grained filtering and sorting of all the data. Any `Tensor` can be accessed via a flat 1D index, which goes directly into the underlying Go slice for the basic types, and is appropriately (though somewhat expensively in some cases) indirected through the effective geometry in `Sliced` and `Rows` types. -* **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Indexed` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In most packages including [tmath](tmath) and [stats](stats), 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. +* **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Rows` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In the [stats](stats) and [metric](metric) packages, 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. The [tmath](tmath) package, which defines binary functions, uses the [broadcasting](#broadcasting) logic to align n-dimensional data, and the row, cell structure provides a concrete simplification for thinking about how that works. A higher-dimensional tensor can also be re-shaped into this row, cell format by collapsing any number of additional outer dimensions into a longer, effective "row" index, with the remaining inner dimensions forming the cell-wise patterns. You can decide where to make the cut, and the `RowCellSplit` function makes it easy to create a new view of an existing tensor with this split made at a given dimension. -* **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations. +* **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations, as in the [matrix](matrix) package. -* **Matrix 3D**: For functions that specifically process 2D matricies, a 3D shape can be used as well, which iterates over the outer row-wise dimension to process the inner 2D matricies. +* **Matrix 3+D**: For functions that specifically process 2D matricies, a 3+D shape can be used as well, which iterates over the outer dimensions to process the inner 2D matricies. ## Dynamic row sizing (e.g., for logs) -The `SetNumRows` method can be used to progressively increase the number of rows to fit more data, as is typically the case when logging data (often using a [table](table)). You can set the row dimension to 0 to start -- that is (now) safe. However, for greatest efficiency, it is best to set the number of rows to the largest expected size first, and _then_ set it back to 0. The underlying slice of data retains its capacity when sized back down. During incremental increasing of the slice size, if it runs out of capacity, all the elements need to be copied, so it is more efficient to establish the capacity up front instead of having multiple incremental re-allocations. +The `SetNumRows` function can be used to progressively increase the number of rows to fit more data, as is typically the case when logging data (often using a [table](table)). You can set the row dimension to 0 to start -- that is (now) safe. However, for greatest efficiency, it is best to set the number of rows to the largest expected size first, and _then_ set it back to 0. The underlying slice of data retains its capacity when sized back down. During incremental increasing of the slice size, if it runs out of capacity, all the elements need to be copied, so it is more efficient to establish the capacity up front instead of having multiple incremental re-allocations. # Cheat Sheet -`ix` is the `Indexed` tensor for these examples: +`ix` is the `Rows` tensor for these examples: ## Tensor Access @@ -86,7 +86,7 @@ str := ix.String1D(2) ```Go // value at row 3, cell 2 (flat index into entire `SubSpace` tensor for this row) -// The row index will be indirected through any `Indexes` present on the Indexed view. +// The row index will be indirected through any `Indexes` present on the Rows view. val := ix.FloatRowCell(3, 2) // string value at row 2, cell 0. this is safe for 1D and 2D+ shapes // and is a robust way to get 1D data from tensors of unknown shapes. @@ -105,7 +105,7 @@ ix.SetRowTensor(tsr, 4) ``` ```Go -// returns a flat, 1D Indexed view into n-dimensional tensor values at +// returns a flat, 1D Rows view into n-dimensional tensor values at // given row. This is used in compute routines that operate generically // on the entire row as a flat pattern. ci := tensor.Cells1D(ix, 5) diff --git a/tensor/base.go b/tensor/base.go index 6a256e21a4..87f99b6223 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -206,19 +206,11 @@ func (tsr *Base[T]) SymmetricDim() int { // with a maximum length of as given: output is terminated // when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. func Sprint(tsr Tensor, maxLen int) string { - return stringIndexed(tsr, maxLen, nil) + return sprint(tsr, maxLen) } -// SprintIndexed returns a string representation of the given Indexed tensor, -// with a maximum length of as given: output is terminated -// when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. -func SprintIndexed(tsr *Indexed, maxLen int) string { - return stringIndexed(tsr.Tensor, maxLen, tsr.Indexes) -} - -// stringIndexed is the underlying impl of String that works for indexed -// data as well. -func stringIndexed(tsr Tensor, maxLen int, idxs []int) string { +// sprint is the underlying impl of String +func sprint(tsr Tensor, maxLen int) string { if maxLen == 0 { maxLen = MaxSprintLength } @@ -227,17 +219,11 @@ func stringIndexed(tsr Tensor, maxLen int, idxs []int) string { b.WriteString(sh.String() + " ") oddRow := false rows, cols, _, _ := Projection2DShape(sh, oddRow) - if idxs != nil { - rows = min(rows, len(idxs)) - } ctr := 0 for r := range rows { rc, _ := Projection2DCoords(sh, oddRow, r, 0) b.WriteString(fmt.Sprintf("%v: ", rc)) ri := r - if idxs != nil { - ri = idxs[r] - } for c := 0; c < cols; c++ { if tsr.IsString() { b.WriteString(fmt.Sprintf("%s ", Projection2DString(tsr, oddRow, ri, c))) diff --git a/tensor/bits.go b/tensor/bits.go index 63831957df..58222b3ada 100644 --- a/tensor/bits.go +++ b/tensor/bits.go @@ -66,7 +66,7 @@ func BoolToInt(bv bool) int { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *Bits) String() string { - return stringIndexed(tsr, 0, nil) + return sprint(tsr, 0) } func (tsr *Bits) IsString() bool { diff --git a/tensor/broadcast.go b/tensor/broadcast.go index e69de29bb2..f5543518e0 100644 --- a/tensor/broadcast.go +++ b/tensor/broadcast.go @@ -0,0 +1,5 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index b55a1945e4..73eae7b8cd 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -48,7 +48,7 @@ func (d *Data) Item(name string) *Data { return d.Dir.At(name) } -// Value returns the [tensor.Indexed] Value for given item +// Value returns the [tensor.Tensor] Value for given item // within this directory. This will panic if item is not // found, and will return nil if it is not a Value // (i.e., it is a directory). diff --git a/tensor/funcs.go b/tensor/funcs.go index 5865e2fbb0..69530325a5 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -277,7 +277,7 @@ func (fn *Func) CallOut(tsr ...Tensor) ([]Tensor, error) { } outs := make([]Tensor, fn.Out) for i := range outs { - outs[i] = NewIndexed(NewOfType(typ)) + outs[i] = NewOfType(typ) } tsr = append(tsr, outs...) err := fn.Call(tsr...) diff --git a/tensor/number.go b/tensor/number.go index 413449c14e..f5f7a41103 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -98,7 +98,7 @@ func NewNumberFromSlice[T num.Number](vals ...T) Tensor { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *Number[T]) String() string { - return stringIndexed(tsr, 0, nil) + return sprint(tsr, 0) } func (tsr *Number[T]) IsString() bool { @@ -367,7 +367,7 @@ func (tsr *Number[T]) SubSpace(offs ...int) Tensor { } // RowTensor is a convenience version of [Tensor.SubSpace] to return the -// SubSpace for the outermost row dimension. [Indexed] defines a version +// SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. func (tsr *Number[T]) RowTensor(row int) Tensor { return tsr.SubSpace(row) diff --git a/tensor/ranges.go b/tensor/ranges.go index ff593c0c9c..7d3a5b9c32 100644 --- a/tensor/ranges.go +++ b/tensor/ranges.go @@ -10,7 +10,7 @@ import ( ) // todo: make a version of these functions that takes -// a standard Indexed tensor with n x 3 shape, where the 3 inner values +// a tensor with n x 3 shape, where the 3 inner values // specify the Range Start, End, Incr values, across n ranges. // these would convert to the current Range-based format that does the impl, // using the Range helper functions, which are also easier and more explicit diff --git a/tensor/indexed.go b/tensor/rows.go similarity index 77% rename from tensor/indexed.go rename to tensor/rows.go index 6cdac6f251..128fabfe82 100644 --- a/tensor/indexed.go +++ b/tensor/rows.go @@ -18,16 +18,16 @@ import ( "gonum.org/v1/gonum/mat" ) -// Indexed is an indexed wrapper around another [Tensor] that provides a -// specific view onto the Tensor defined by the set of [Indexed.Indexes], +// Rows is an indexed wrapper around another [Tensor] that provides a +// specific view onto the Tensor defined by the set of [Rows.Indexes], // which apply to the outermost row dimension (with default row-major indexing). // Sorting and filtering a tensor only requires updating the indexes while // leaving the underlying Tensor alone. // To produce a new [Tensor] that has its raw data actually organized according -// to the indexed order (i.e., the copy function of numpy), call [Indexed.NewTensor]. +// to the indexed order (i.e., the copy function of numpy), call [Rows.NewTensor]. // Use the [Set]FloatRow[Cell] methods wherever possible, for the most efficient // and natural indirection through the indexes. -type Indexed struct { //types:add +type Rows struct { //types:add // Tensor that we are an indexed view onto. Tensor Tensor @@ -38,41 +38,41 @@ type Indexed struct { //types:add Indexes []int } -// NewIndexed returns a new [Indexed] view of given tensor, +// NewRows returns a new [Rows] view of given tensor, // with optional list of indexes (none / nil = sequential). -func NewIndexed(tsr Tensor, idxs ...int) *Indexed { - ix := &Indexed{Tensor: tsr, Indexes: idxs} +func NewRows(tsr Tensor, idxs ...int) *Rows { + ix := &Rows{Tensor: tsr, Indexes: idxs} return ix } -// AsIndexed returns the tensor as an [Indexed[] view. +// AsRows returns the tensor as an [Rows[] view. // If it already is one, then it is returned, otherwise it is wrapped. -func AsIndexed(tsr Tensor) *Indexed { - if ix, ok := tsr.(*Indexed); ok { +func AsRows(tsr Tensor) *Rows { + if ix, ok := tsr.(*Rows); ok { return ix } - return NewIndexed(tsr) + return NewRows(tsr) } // SetTensor sets as indexes into given tensor with sequential initial indexes. -func (ix *Indexed) SetTensor(tsr Tensor) { +func (ix *Rows) SetTensor(tsr Tensor) { ix.Tensor = tsr ix.Sequential() } // RowIndex returns the actual index into underlying tensor row based on given // index value. If Indexes == nil, index is passed through. -func (ix *Indexed) RowIndex(idx int) int { +func (ix *Rows) RowIndex(idx int) int { if ix.Indexes == nil { return idx } return ix.Indexes[idx] } -// NumRows returns the effective number of rows in this Indexed view, +// NumRows returns the effective number of rows in this Rows view, // which is the length of the index list or number of outer // rows dimension of tensor if no indexes (full sequential view). -func (ix *Indexed) NumRows() int { +func (ix *Rows) NumRows() int { if ix.Indexes == nil { return ix.Tensor.DimSize(0) } @@ -80,22 +80,22 @@ func (ix *Indexed) NumRows() int { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (ix *Indexed) String() string { - return stringIndexed(ix.Tensor, 0, ix.Indexes) // todo: no need +func (ix *Rows) String() string { + return sprint(ix.Tensor, 0) // todo: no need } // Label satisfies the core.Labeler interface for a summary description of the tensor. -func (ix *Indexed) Label() string { +func (ix *Rows) Label() string { return ix.Tensor.Label() } // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. -func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } +func (ix *Rows) Metadata() *metadata.Data { return ix.Tensor.Metadata() } // If we have Indexes, this is the effective shape sizes using // the current number of indexes as the outermost row dimension size. -func (ix *Indexed) ShapeInts() []int { +func (ix *Rows) ShapeInts() []int { if ix.Indexes == nil || ix.Tensor.NumDims() == 0 { return ix.Tensor.ShapeInts() } @@ -104,7 +104,7 @@ func (ix *Indexed) ShapeInts() []int { return sh } -func (ix *Indexed) ShapeSizes() Tensor { +func (ix *Rows) ShapeSizes() Tensor { if ix.Indexes == nil { return ix.Tensor.ShapeSizes() } @@ -114,7 +114,7 @@ func (ix *Indexed) ShapeSizes() Tensor { // Shape() returns a [Shape] representation of the tensor shape // (dimension sizes). If we have Indexes, this is the effective // shape using the current number of indexes as the outermost row dimension size. -func (ix *Indexed) Shape() *Shape { +func (ix *Rows) Shape() *Shape { if ix.Indexes == nil { return ix.Tensor.Shape() } @@ -129,7 +129,7 @@ func (ix *Indexed) Shape() *Shape { // e.g., for computational routines that use a 1D cell view. // Otherwise, we reset the indexes and then set the wrapped shape, // because our current indexes are now invalidated. -func (ix *Indexed) SetShapeInts(sizes ...int) { +func (ix *Rows) SetShapeInts(sizes ...int) { if ix.Indexes == nil || ix.Tensor.NumDims() == 0 { ix.Tensor.SetShapeInts(sizes...) return @@ -145,32 +145,32 @@ func (ix *Indexed) SetShapeInts(sizes ...int) { // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. // This invalidates the indexes. -func (ix *Indexed) SetNumRows(rows int) { +func (ix *Rows) SetNumRows(rows int) { ix.Sequential() ix.Tensor.SetNumRows(rows) } // SetShape sets our shape to given sizes. -// See [Indexed.SetShapeInts] for details. -func (ix *Indexed) SetShape(sizes Tensor) { +// See [Rows.SetShapeInts] for details. +func (ix *Rows) SetShape(sizes Tensor) { ix.SetShapeInts(AsIntSlice(sizes)...) } // Len returns the total number of elements in the tensor, // taking into account the Indexes via [Rows], // as NumRows() * cell size. -func (ix *Indexed) Len() int { +func (ix *Rows) Len() int { rows := ix.NumRows() _, cells := ix.Tensor.RowCellSize() return cells * rows } // NumDims returns the total number of dimensions. -func (ix *Indexed) NumDims() int { return ix.Tensor.NumDims() } +func (ix *Rows) NumDims() int { return ix.Tensor.NumDims() } // DimSize returns size of given dimension, returning NumRows() // for first dimension. -func (ix *Indexed) DimSize(dim int) int { +func (ix *Rows) DimSize(dim int) int { if dim == 0 { return ix.NumRows() } @@ -178,9 +178,9 @@ func (ix *Indexed) DimSize(dim int) int { } // RowCellSize returns the size of the outermost Row shape dimension -// (via [Indexed.NumRows] method), and the size of all the remaining +// (via [Rows.NumRows] method), and the size of all the remaining // inner dimensions (the "cell" size). -func (ix *Indexed) RowCellSize() (rows, cells int) { +func (ix *Rows) RowCellSize() (rows, cells int) { _, cells = ix.Tensor.RowCellSize() rows = ix.NumRows() return @@ -188,7 +188,7 @@ func (ix *Indexed) RowCellSize() (rows, cells int) { // ValidIndexes deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. -func (ix *Indexed) ValidIndexes() { +func (ix *Rows) ValidIndexes() { if ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { ix.Indexes = nil return @@ -202,7 +202,7 @@ func (ix *Indexed) ValidIndexes() { } // Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor. -func (ix *Indexed) Sequential() { //types:add +func (ix *Rows) Sequential() { //types:add ix.Indexes = nil } @@ -210,7 +210,7 @@ func (ix *Indexed) Sequential() { //types:add // e.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise // current indexes are left as is. Use Sequential, then IndexesNeeded to ensure // all rows are represented. -func (ix *Indexed) IndexesNeeded() { +func (ix *Rows) IndexesNeeded() { if ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return @@ -226,7 +226,7 @@ func (ix *Indexed) IndexesNeeded() { // ExcludeMissing deletes indexes where the values are missing, as indicated by NaN. // Uses first cell of higher dimensional data. -func (ix *Indexed) ExcludeMissing() { //types:add +func (ix *Rows) ExcludeMissing() { //types:add if ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return @@ -243,7 +243,7 @@ func (ix *Indexed) ExcludeMissing() { //types:add // Permuted sets indexes to a permuted order. If indexes already exist // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated -func (ix *Indexed) Permuted() { +func (ix *Rows) Permuted() { if ix.Tensor.DimSize(0) <= 0 { ix.Indexes = nil return @@ -276,7 +276,7 @@ const ( // as these row numbers have already been projected through the indexes. // cmp(a, b) should return a negative number when a < b, a positive // number when a > b and zero when a == b. -func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) { +func (ix *Rows) SortFunc(cmp func(tsr Tensor, i, j int) int) { ix.IndexesNeeded() slices.SortFunc(ix.Indexes, func(a, b int) int { return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! @@ -286,7 +286,7 @@ func (ix *Indexed) SortFunc(cmp func(tsr Tensor, i, j int) int) { // SortIndexes sorts the indexes into our Tensor directly in // numerical order, producing the native ordering, while preserving // any filtering that might have occurred. -func (ix *Indexed) SortIndexes() { +func (ix *Rows) SortIndexes() { if ix.Indexes == nil { return } @@ -302,7 +302,7 @@ func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { // Sort does default alpha or numeric sort of row-wise data. // Uses first cell of higher dimensional data. -func (ix *Indexed) Sort(ascending bool) { +func (ix *Rows) Sort(ascending bool) { if ix.Tensor.IsString() { ix.SortFunc(func(tsr Tensor, i, j int) int { return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) @@ -321,7 +321,7 @@ func (ix *Indexed) Sort(ascending bool) { // number when a > b and zero when a == b. // It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { +func (ix *Rows) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { ix.IndexesNeeded() slices.SortStableFunc(ix.Indexes, func(a, b int) int { return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! @@ -330,7 +330,7 @@ func (ix *Indexed) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { // SortStable does stable default alpha or numeric sort. // Uses first cell of higher dimensional data. -func (ix *Indexed) SortStable(ascending bool) { +func (ix *Rows) SortStable(ascending bool) { if ix.Tensor.IsString() { ix.SortStableFunc(func(tsr Tensor, i, j int) int { return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) @@ -350,7 +350,7 @@ type FilterFunc func(tsr Tensor, row int) bool // Filter filters the indexes using given Filter function. // The Filter function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. -func (ix *Indexed) Filter(filterer func(tsr Tensor, row int) bool) { +func (ix *Rows) Filter(filterer func(tsr Tensor, row int) bool) { ix.IndexesNeeded() sz := len(ix.Indexes) for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering @@ -382,7 +382,7 @@ const ( // Use the named const args [Include], [Exclude], [Contains], [Equals], // [IgnoreCase], [UseCase] for greater clarity. // Uses first cell of higher dimensional data. -func (ix *Indexed) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add +func (ix *Rows) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add lowstr := strings.ToLower(str) ix.Filter(func(tsr Tensor, row int) bool { val := tsr.StringRowCell(row, 0) @@ -407,7 +407,7 @@ func (ix *Indexed) FilterString(str string, exclude, contains, ignoreCase bool) // NewTensor returns a new tensor with column data organized according to // the Indexes. If Indexes are nil, a clone of the current tensor is returned // but this function is only sensible if there is an indexed view in place. -func (ix *Indexed) NewTensor() Tensor { +func (ix *Rows) NewTensor() Tensor { nt := ix.Tensor.Clone() if ix.Indexes == nil { return nt @@ -428,33 +428,33 @@ func (ix *Indexed) NewTensor() Tensor { return nt } -// Clone returns a copy of the current Indexed view with a cloned copy of +// Clone returns a copy of the current Rows view with a cloned copy of // the underlying Tensor and copy of the indexes. -func (ix *Indexed) Clone() Tensor { - nix := &Indexed{} +func (ix *Rows) Clone() Tensor { + nix := &Rows{} nix.Tensor = ix.Tensor.Clone() nix.CopyIndexes(ix) return nix } -func (ix *Indexed) View() Tensor { - nix := &Indexed{} +func (ix *Rows) View() Tensor { + nix := &Rows{} nix.Tensor = ix.Tensor.View() nix.CopyIndexes(ix) return nix } -// CloneIndexes returns a copy of the current Indexed view with new indexes, +// CloneIndexes returns a copy of the current Rows view with new indexes, // with a pointer to the same underlying Tensor as the source. -func (ix *Indexed) CloneIndexes() *Indexed { - nix := &Indexed{} +func (ix *Rows) CloneIndexes() *Rows { + nix := &Rows{} nix.Tensor = ix.Tensor nix.CopyIndexes(ix) return nix } -// CopyIndexes copies indexes from other Indexed view. -func (ix *Indexed) CopyIndexes(oix *Indexed) { +// CopyIndexes copies indexes from other Rows view. +func (ix *Rows) CopyIndexes(oix *Rows) { if oix.Indexes == nil { ix.Indexes = nil } else { @@ -463,7 +463,7 @@ func (ix *Indexed) CopyIndexes(oix *Indexed) { } // AddRows adds n rows to end of underlying Tensor, and to the indexes in this view -func (ix *Indexed) AddRows(n int) { //types:add +func (ix *Rows) AddRows(n int) { //types:add stidx := ix.Tensor.DimSize(0) ix.Tensor.SetNumRows(stidx + n) if ix.Indexes != nil { @@ -475,7 +475,7 @@ func (ix *Indexed) AddRows(n int) { //types:add // InsertRows adds n rows to end of underlying Tensor, and to the indexes starting at // given index in this view -func (ix *Indexed) InsertRows(at, n int) { +func (ix *Rows) InsertRows(at, n int) { stidx := ix.Tensor.DimSize(0) ix.IndexesNeeded() ix.Tensor.SetNumRows(stidx + n) @@ -487,13 +487,13 @@ func (ix *Indexed) InsertRows(at, n int) { } // DeleteRows deletes n rows of indexes starting at given index in the list of indexes -func (ix *Indexed) DeleteRows(at, n int) { +func (ix *Rows) DeleteRows(at, n int) { ix.IndexesNeeded() ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) } // Swap switches the indexes for i and j -func (ix *Indexed) Swap(i, j int) { +func (ix *Rows) Swap(i, j int) { if ix.Indexes == nil { return } @@ -501,13 +501,13 @@ func (ix *Indexed) Swap(i, j int) { } /////////////////////////////////////////////// -// Indexed access +// Rows access ///////////////////// Floats // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. -func (ix *Indexed) Float(i ...int) float64 { +func (ix *Rows) Float(i ...int) float64 { if ix.Indexes == nil { return ix.Tensor.Float(i...) } @@ -517,8 +517,8 @@ func (ix *Indexed) Float(i ...int) float64 { } // SetFloat sets the value of given index as a float64 -// The first index value is indirected through the [Indexed.Indexes]. -func (ix *Indexed) SetFloat(val float64, i ...int) { +// The first index value is indirected through the [Rows.Indexes]. +func (ix *Rows) SetFloat(val float64, i ...int) { if ix.Indexes == nil { ix.Tensor.SetFloat(val, i...) return @@ -530,23 +530,23 @@ func (ix *Indexed) SetFloat(val float64, i ...int) { // FloatRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexed.Indexes]. -// This is the preferred interface for all Indexed operations. -func (ix *Indexed) FloatRowCell(row, cell int) float64 { +// Row is indirected through the [Rows.Indexes]. +// This is the preferred interface for all Rows operations. +func (ix *Rows) FloatRowCell(row, cell int) float64 { return ix.Tensor.FloatRowCell(ix.RowIndex(row), cell) } // SetFloatRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexed.Indexes]. -// This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetFloatRowCell(val float64, row, cell int) { +// Row is indirected through the [Rows.Indexes]. +// This is the preferred interface for all Rows operations. +func (ix *Rows) SetFloatRowCell(val float64, row, cell int) { ix.Tensor.SetFloatRowCell(val, ix.RowIndex(row), cell) } // Float1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Indexed) Float1D(i int) float64 { +func (ix *Rows) Float1D(i int) float64 { if ix.Indexes == nil { return ix.Tensor.Float1D(i) } @@ -555,18 +555,18 @@ func (ix *Indexed) Float1D(i int) float64 { // SetFloat1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Indexed) SetFloat1D(val float64, i int) { +func (ix *Rows) SetFloat1D(val float64, i int) { if ix.Indexes == nil { ix.Tensor.SetFloat1D(val, i) } ix.SetFloat(val, ix.Tensor.Shape().IndexFrom1D(i)...) } -func (ix *Indexed) FloatRow(row int) float64 { +func (ix *Rows) FloatRow(row int) float64 { return ix.FloatRowCell(row, 0) } -func (ix *Indexed) SetFloatRow(val float64, row int) { +func (ix *Rows) SetFloatRow(val float64, row int) { ix.SetFloatRowCell(val, row, 0) } @@ -574,7 +574,7 @@ func (ix *Indexed) SetFloatRow(val float64, row int) { // StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. -func (ix *Indexed) StringValue(i ...int) string { +func (ix *Rows) StringValue(i ...int) string { if ix.Indexes == nil { return ix.Tensor.StringValue(i...) } @@ -584,8 +584,8 @@ func (ix *Indexed) StringValue(i ...int) string { } // SetString sets the value of given index as a string -// The first index value is indirected through the [Indexed.Indexes]. -func (ix *Indexed) SetString(val string, i ...int) { +// The first index value is indirected through the [Rows.Indexes]. +func (ix *Rows) SetString(val string, i ...int) { if ix.Indexes == nil { ix.Tensor.SetString(val, i...) } @@ -596,23 +596,23 @@ func (ix *Indexed) SetString(val string, i ...int) { // StringRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexed.Indexes]. -// This is the preferred interface for all Indexed operations. -func (ix *Indexed) StringRowCell(row, cell int) string { +// Row is indirected through the [Rows.Indexes]. +// This is the preferred interface for all Rows operations. +func (ix *Rows) StringRowCell(row, cell int) string { return ix.Tensor.StringRowCell(ix.RowIndex(row), cell) } // SetStringRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexed.Indexes]. -// This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetStringRowCell(val string, row, cell int) { +// Row is indirected through the [Rows.Indexes]. +// This is the preferred interface for all Rows operations. +func (ix *Rows) SetStringRowCell(val string, row, cell int) { ix.Tensor.SetStringRowCell(val, ix.RowIndex(row), cell) } // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Indexed) String1D(i int) string { +func (ix *Rows) String1D(i int) string { if ix.Indexes == nil { return ix.Tensor.String1D(i) } @@ -621,18 +621,18 @@ func (ix *Indexed) String1D(i int) string { // SetString1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Indexed) SetString1D(val string, i int) { +func (ix *Rows) SetString1D(val string, i int) { if ix.Indexes == nil { ix.Tensor.SetString1D(val, i) } ix.SetString(val, ix.Tensor.Shape().IndexFrom1D(i)...) } -func (ix *Indexed) StringRow(row int) string { +func (ix *Rows) StringRow(row int) string { return ix.StringRowCell(row, 0) } -func (ix *Indexed) SetStringRow(val string, row int) { +func (ix *Rows) SetStringRow(val string, row int) { ix.SetStringRowCell(val, row, 0) } @@ -640,7 +640,7 @@ func (ix *Indexed) SetStringRow(val string, row int) { // Int returns the value of given index as an int. // The first index value is indirected through the indexes. -func (ix *Indexed) Int(i ...int) int { +func (ix *Rows) Int(i ...int) int { if ix.Indexes == nil { return ix.Tensor.Int(i...) } @@ -650,8 +650,8 @@ func (ix *Indexed) Int(i ...int) int { } // SetInt sets the value of given index as an int -// The first index value is indirected through the [Indexed.Indexes]. -func (ix *Indexed) SetInt(val int, i ...int) { +// The first index value is indirected through the [Rows.Indexes]. +func (ix *Rows) SetInt(val int, i ...int) { if ix.Indexes == nil { ix.Tensor.SetInt(val, i...) return @@ -663,23 +663,23 @@ func (ix *Indexed) SetInt(val int, i ...int) { // IntRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexed.Indexes]. -// This is the preferred interface for all Indexed operations. -func (ix *Indexed) IntRowCell(row, cell int) int { +// Row is indirected through the [Rows.Indexes]. +// This is the preferred interface for all Rows operations. +func (ix *Rows) IntRowCell(row, cell int) int { return ix.Tensor.IntRowCell(ix.RowIndex(row), cell) } // SetIntRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Indexed.Indexes]. -// This is the preferred interface for all Indexed operations. -func (ix *Indexed) SetIntRowCell(val int, row, cell int) { +// Row is indirected through the [Rows.Indexes]. +// This is the preferred interface for all Rows operations. +func (ix *Rows) SetIntRowCell(val int, row, cell int) { ix.Tensor.SetIntRowCell(val, ix.RowIndex(row), cell) } // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Indexed) Int1D(i int) int { +func (ix *Rows) Int1D(i int) int { if ix.Indexes == nil { return ix.Tensor.Int1D(i) } @@ -688,18 +688,18 @@ func (ix *Indexed) Int1D(i int) int { // SetInt1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Indexed) SetInt1D(val int, i int) { +func (ix *Rows) SetInt1D(val int, i int) { if ix.Indexes == nil { ix.Tensor.SetInt1D(val, i) } ix.SetInt(val, ix.Tensor.Shape().IndexFrom1D(i)...) } -func (ix *Indexed) IntRow(row int) int { +func (ix *Rows) IntRow(row int) int { return ix.IntRowCell(row, 0) } -func (ix *Indexed) SetIntRow(val int, row int) { +func (ix *Rows) SetIntRow(val int, row int) { ix.SetIntRowCell(val, row, 0) } @@ -711,9 +711,9 @@ func (ix *Indexed) SetIntRow(val int, row int) { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. -// Indexed version does indexed indirection of the outermost row dimension +// Rows version does indexed indirection of the outermost row dimension // of the offsets. -func (ix *Indexed) SubSpace(offs ...int) Tensor { +func (ix *Rows) SubSpace(offs ...int) Tensor { if len(offs) == 0 { return nil } @@ -721,23 +721,23 @@ func (ix *Indexed) SubSpace(offs ...int) Tensor { return ix.Tensor.SubSpace(offs...) } -// RowTensor is a convenience version of [Indexed.SubSpace] to return the +// RowTensor is a convenience version of [Rows.SubSpace] to return the // SubSpace for the outermost row dimension, indirected through the indexes. -func (ix *Indexed) RowTensor(row int) Tensor { +func (ix *Rows) RowTensor(row int) Tensor { return ix.Tensor.RowTensor(ix.RowIndex(row)) } // SetRowTensor sets the values of the SubSpace at given row to given values, // with row indirected through the indexes. -func (ix *Indexed) SetRowTensor(val Tensor, row int) { +func (ix *Rows) SetRowTensor(val Tensor, row int) { ix.Tensor.SetRowTensor(val, ix.RowIndex(row)) } // CopyFrom copies all values from other tensor into this tensor. -// Checks if source is an Indexed and copies indexes too, +// Checks if source is an Rows and copies indexes too, // otherwise underlying tensor copies from and indexes are reset. -func (ix *Indexed) CopyFrom(from Tensor) { - if fix, ok := from.(*Indexed); ok { +func (ix *Rows) CopyFrom(from Tensor) { + if fix, ok := from.(*Rows); ok { ix.Tensor.CopyFrom(fix.Tensor) ix.CopyIndexes(fix) return @@ -748,39 +748,38 @@ func (ix *Indexed) CopyFrom(from Tensor) { // AppendFrom appends all values from other tensor into this tensor. // This invalidates the indexes which are reset. -func (ix *Indexed) AppendFrom(from Tensor) error { +func (ix *Rows) AppendFrom(from Tensor) error { ix.Sequential() return ix.Tensor.AppendFrom(from) } // CopyCellsFrom copies given range of values from other tensor into this tensor, // This invalidates the indexes which are reset. -func (ix *Indexed) CopyCellsFrom(from Tensor, to, start, n int) { - ix.Sequential() +func (ix *Rows) CopyCellsFrom(from Tensor, to, start, n int) { ix.Tensor.CopyCellsFrom(from, to, start, n) } -func (ix *Indexed) Sizeof() int64 { +func (ix *Rows) Sizeof() int64 { return ix.Tensor.Sizeof() // todo: could be out of sync with shape! } -func (ix *Indexed) Bytes() []byte { +func (ix *Rows) Bytes() []byte { return ix.Tensor.Bytes() // todo: could be out of sync with shape! } -func (ix *Indexed) IsString() bool { +func (ix *Rows) IsString() bool { return ix.Tensor.IsString() } -func (ix *Indexed) DataType() reflect.Kind { +func (ix *Rows) DataType() reflect.Kind { return ix.Tensor.DataType() } -func (ix *Indexed) Range() (min, max float64, minIndex, maxIndex int) { +func (ix *Rows) Range() (min, max float64, minIndex, maxIndex int) { return ix.Tensor.Range() } -func (ix *Indexed) SetZeros() { +func (ix *Rows) SetZeros() { ix.Tensor.SetZeros() } @@ -788,7 +787,7 @@ func (ix *Indexed) SetZeros() { // Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the // 2D Matrix. Assumes Row-major ordering and logs an error if NumDims < 2. -func (ix *Indexed) Dims() (r, c int) { +func (ix *Rows) Dims() (r, c int) { nd := ix.NumDims() if nd < 2 { log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") @@ -799,7 +798,7 @@ func (ix *Indexed) Dims() (r, c int) { // Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric // 2D Matrix. -func (ix *Indexed) Symmetric() (r int) { +func (ix *Rows) Symmetric() (r int) { nd := ix.NumDims() if nd < 2 { log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") @@ -813,7 +812,7 @@ func (ix *Indexed) Symmetric() (r int) { } // SymmetricDim returns the number of rows/columns in the matrix. -func (ix *Indexed) SymmetricDim() int { +func (ix *Rows) SymmetricDim() int { nd := ix.NumDims() if nd < 2 { log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") @@ -828,7 +827,7 @@ func (ix *Indexed) SymmetricDim() int { // At is the gonum/mat.Matrix interface method for returning 2D matrix element at given // row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. -func (ix *Indexed) At(i, j int) float64 { +func (ix *Rows) At(i, j int) float64 { nd := ix.NumDims() if nd < 2 { log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") @@ -845,9 +844,9 @@ func (ix *Indexed) At(i, j int) float64 { // T is the gonum/mat.Matrix transpose method. // It performs an implicit transpose by returning the receiver inside a Transpose. -func (ix *Indexed) T() mat.Matrix { +func (ix *Rows) T() mat.Matrix { return mat.Transpose{ix} } // check for interface impl -var _ Tensor = (*Indexed)(nil) +var _ Tensor = (*Rows)(nil) diff --git a/tensor/sliced.go b/tensor/sliced.go index 47d8252482..5355407d97 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -10,7 +10,6 @@ import ( "reflect" "slices" "sort" - "strings" "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/slicesx" @@ -26,7 +25,7 @@ import ( // Use the [SliceSet] function to copy sliced values back to the original. func Slice(tsr, out *Sliced, ranges ...Range) error { sizes := slices.Clone(tsr.Tensor.ShapeInts()) - sizes[0] = tsr.NumRows() // takes into account indexes + sizes[0] = tsr.DimSize(0) // takes into account indexes nsz, err := SliceSize(sizes, ranges...) if err != nil { return err @@ -159,6 +158,11 @@ func (sl *Sliced) Label() string { return sl.Tensor.Label() } +// String satisfies the fmt.Stringer interface for string of tensor data. +func (sl *Sliced) String() string { + return sprint(sl.Tensor, 0) // todo: no need +} + // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } @@ -204,6 +208,13 @@ func (sl *Sliced) SetShape(sizes Tensor) { sl.SetShapeInts(AsIntSlice(sizes)...) } +// SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. +// This invalidates the indexes. +func (sl *Sliced) SetNumRows(rows int) { + sl.Sequential() + sl.Tensor.SetNumRows(rows) +} + // Len returns the total number of elements in our view of the tensor. func (sl *Sliced) Len() int { return sl.Shape().Len() @@ -233,7 +244,7 @@ func (sl *Sliced) RowCellSize() (rows, cells int) { } else { ln := 1 for d := 1; d < nd; d++ { - ln *= sl.DimSize() + ln *= sl.DimSize(d) } cells = ln } @@ -281,86 +292,35 @@ func (sl *Sliced) SortIndexes(dim int) { sl.Indexes[dim] = ix } -// Sort does default alpha or numeric sort along given dimension of data. -func (sl *Sliced) Sort(dim int, ascending bool) { - if sl.Tensor.IsString() { - sl.SortFunc(dim, func(tsr Tensor, dim, i, j int) int { - return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) - }) - } else { - sl.SortFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) - }) - } -} - -// SortStableFunc stably sorts the row-wise indexes using given compare function. +// SortStableFunc stably sorts along given dimension using given compare function. // The compare function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. // cmp(a, b) should return a negative number when a < b, a positive // number when a > b and zero when a == b. // It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (sl *Sliced) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { - sl.IndexesNeeded() - slices.SortStableFunc(sl.Indexes, func(a, b int) int { - return cmp(sl.Tensor, a, b) // key point: these are already indirected through indexes!! +func (sl *Sliced) SortStableFunc(dim int, cmp func(tsr Tensor, dim, i, j int) int) { + sl.IndexesNeeded(dim) + ix := sl.Indexes[dim] + slices.SortStableFunc(ix, func(a, b int) int { + return cmp(sl.Tensor, dim, a, b) // key point: these are already indirected through indexes!! }) -} - -// SortStable does stable default alpha or numeric sort. -// Uses first cell of higher dimensional data. -func (sl *Sliced) SortStable(ascending bool) { - if sl.Tensor.IsString() { - sl.SortStableFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) - }) - } else { - sl.SortStableFunc(func(tsr Tensor, i, j int) int { - return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) - }) - } + sl.Indexes[dim] = ix } // Filter filters the indexes using given Filter function. // The Filter function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. -func (sl *Sliced) Filter(filterer func(tsr Tensor, row int) bool) { - sl.IndexesNeeded() - sz := len(sl.Indexes) +func (sl *Sliced) Filter(dim int, filterer func(tsr Tensor, dim, idx int) bool) { + sl.IndexesNeeded(dim) + ix := sl.Indexes[dim] + sz := len(ix) for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering - if !filterer(sl.Tensor, sl.Indexes[i]) { // delete - sl.Indexes = append(sl.Indexes[:i], sl.Indexes[i+1:]...) + if !filterer(sl, dim, ix[i]) { // delete + ix = append(ix[:i], ix[i+1:]...) } } -} - -// FilterString filters the indexes using string values compared to given -// string. Includes rows with matching values unless exclude is set. -// If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use the named const args [Include], [Exclude], [Contains], [Equals], -// [IgnoreCase], [UseCase] for greater clarity. -// Uses first cell of higher dimensional data. -func (sl *Sliced) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add - lowstr := strings.ToLower(str) - sl.Filter(func(tsr Tensor, row int) bool { - val := tsr.StringRowCell(row, 0) - has := false - switch { - case contains && ignoreCase: - has = strings.Contains(strings.ToLower(val), lowstr) - case contains: - has = strings.Contains(val, str) - case ignoreCase: - has = strings.EqualFold(val, str) - default: - has = (val == str) - } - if exclude { - return !has - } - return has - }) + sl.Indexes[dim] = ix } // NewTensor returns a new tensor with column data organized according to @@ -421,42 +381,16 @@ func (sl *Sliced) CopyIndexes(oix *Sliced) { } } -// AddRows adds n rows to end of underlying Tensor, and to the indexes in this view -func (sl *Sliced) AddRows(n int) { //types:add - stidx := sl.Tensor.DimSize(0) - sl.Tensor.SetNumRows(stidx + n) - if sl.Indexes != nil { - for i := stidx; i < stidx+n; i++ { - sl.Indexes = append(sl.Indexes, i) - } - } -} - -// InsertRows adds n rows to end of underlying Tensor, and to the indexes starting at -// given index in this view -func (sl *Sliced) InsertRows(at, n int) { - stidx := sl.Tensor.DimSize(0) - sl.IndexesNeeded() - sl.Tensor.SetNumRows(stidx + n) - nw := make([]int, n, n+len(sl.Indexes)-at) - for i := 0; i < n; i++ { - nw[i] = stidx + i - } - sl.Indexes = append(sl.Indexes[:at], append(nw, sl.Indexes[at:]...)...) -} - -// DeleteRows deletes n rows of indexes starting at given index in the list of indexes -func (sl *Sliced) DeleteRows(at, n int) { - sl.IndexesNeeded() - sl.Indexes = append(sl.Indexes[:at], sl.Indexes[at+n:]...) +// AppendFrom appends all values from other tensor into this tensor. +// This invalidates the indexes which are reset. +func (sl *Sliced) AppendFrom(from Tensor) error { + sl.Sequential() + return sl.Tensor.AppendFrom(from) } -// Swap switches the indexes for i and j -func (sl *Sliced) Swap(i, j int) { - if sl.Indexes == nil { - return - } - sl.Indexes[i], sl.Indexes[j] = sl.Indexes[j], sl.Indexes[i] +// CopyCellsFrom copies given range of values from other tensor into this tensor, +func (sl *Sliced) CopyCellsFrom(from Tensor, to, start, n int) { + sl.Tensor.CopyCellsFrom(from, to, start, n) } /////////////////////////////////////////////// @@ -464,27 +398,20 @@ func (sl *Sliced) Swap(i, j int) { ///////////////////// Floats +// todo: these are not the right functions + // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. func (sl *Sliced) Float(i ...int) float64 { - if sl.Indexes == nil { - return sl.Tensor.Float(i...) - } - ic := slices.Clone(i) - ic[0] = sl.Indexes[ic[0]] - return sl.Tensor.Float(ic...) + sh := sl.Shape() + return sl.Tensor.Float1D(sh.IndexTo1D()) } // SetFloat sets the value of given index as a float64 // The first index value is indirected through the [Sliced.Indexes]. func (sl *Sliced) SetFloat(val float64, i ...int) { - if sl.Indexes == nil { - sl.Tensor.SetFloat(val, i...) - return - } - ic := slices.Clone(i) - ic[0] = sl.Indexes[ic[0]] - sl.Tensor.SetFloat(val, ic...) + sh := sl.Shape() + sl.Tensor.SetFloat1D(val, sh.IndexTo1D()) } // FloatRowCell returns the value at given row and cell, @@ -492,6 +419,10 @@ func (sl *Sliced) SetFloat(val float64, i ...int) { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) FloatRowCell(row, cell int) float64 { + sh := sl.Shape() + _, sz := sh.RowCellSize() + i := row*sz + cell + return float64(tsr.Values[i]) return sl.Tensor.FloatRowCell(sl.RowIndex(row), cell) } @@ -705,20 +636,6 @@ func (sl *Sliced) CopyFrom(from Tensor) { sl.Tensor.CopyFrom(from) } -// AppendFrom appends all values from other tensor into this tensor. -// This invalidates the indexes which are reset. -func (sl *Sliced) AppendFrom(from Tensor) error { - sl.Sequential() - return sl.Tensor.AppendFrom(from) -} - -// CopyCellsFrom copies given range of values from other tensor into this tensor, -// This invalidates the indexes which are reset. -func (sl *Sliced) CopyCellsFrom(from Tensor, to, start, n int) { - sl.Sequential() - sl.Tensor.CopyCellsFrom(from, to, start, n) -} - func (sl *Sliced) Sizeof() int64 { return sl.Tensor.Sizeof() // todo: could be out of sync with shape! } diff --git a/tensor/stats/glm/README.md b/tensor/stats/glm/README.md index 1e4b5503c1..cb6b0c323d 100644 --- a/tensor/stats/glm/README.md +++ b/tensor/stats/glm/README.md @@ -2,7 +2,7 @@ GLM contains results and parameters for running a [general linear model](https://en.wikipedia.org/wiki/General_linear_model), which is a general form of multivariate linear regression, supporting multiple independent and dependent variables. -Make a `NewGLM` and then do `Run()` on a tensor [Indexed](../table/Indexed) with the relevant data in columns of the table. +Make a `NewGLM` and then do `Run()` on a [table](../table) with the relevant data in columns of the table. # Fitting Methods diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 783a324954..0877f8b8bf 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -1,6 +1,6 @@ # metric -`metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Indexed` standard data representation. +`metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Tensor` standard data representation. The `Matrix` function returns a distance / similarity matrix computed from the n-dimensional "cells" of row-organized tensor data, and the `LabeledMatrix` type provides labels for displaying such matricies. diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index a408c89c28..e339a043cb 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -111,8 +111,6 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { flatsz := []int{in.DimSize(0), cells} flatvw := in.View() flatvw.SetShapeInts(flatsz...) - // flatix := tensor.NewIndexed(flatvw) - // flatix.Indexes = in.Indexes mout := tensor.NewFloat64Scalar(0.0) out.SetShapeInts(cells, cells) diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index 61350bd229..a6b2ec626d 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -34,9 +34,9 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `VarPop`: population variance (squared diffs from mean, divided by n) * `StdPop`: population standard deviation (sqrt of VarPop) * `SemPop`: population standard error of the mean (StdPop divided by sqrt(n)) -* `Median`: middle value in sorted ordering (only for Indexed) -* `Q1`: Q1 first quartile = 25%ile value = .25 quantile value (only for Indexed) -* `Q3`: Q3 third quartile = 75%ile value = .75 quantile value (only for Indexed) +* `Median`: middle value in sorted ordering (uses a `Rows` view) +* `Q1`: Q1 first quartile = 25%ile value = .25 quantile value (uses `Rows`) +* `Q3`: Q3 third quartile = 75%ile value = .75 quantile value (uses `Rows`) Here is the general info associated with these function calls: diff --git a/tensor/stats/stats/doc.go b/tensor/stats/stats/doc.go index ea364ea36c..bb7b3ed36a 100644 --- a/tensor/stats/stats/doc.go +++ b/tensor/stats/stats/doc.go @@ -3,14 +3,6 @@ // license that can be found in the LICENSE file. /* -Package agg provides aggregation functions operating on Indexed indexed views -of table.Table data, along with standard AggFunc functions that can be used -at any level of aggregation from tensor on up. - -The main functions use names to specify columns, and *Index and *Try versions -are available that operate on column indexes and return errors, respectively. - -See tsragg package for functions that operate directly on a tensor.Tensor -without the indexview indirection. +Package stats provides standard statistic computations operating on the `tensor.Tensor` standard data representation. */ package stats diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 8f8c4d81bb..6aa2446cce 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -124,7 +124,7 @@ func MaxAbsFunc(in, out tensor.Tensor) { // and returns the Float64 output values for subsequent use. func MeanFuncOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor) { sum64 := SumFuncOut64(in, out) - count := tensor.NewIndexed(out.Clone()) + count := out.Clone() count64 = CountFuncOut64(in, count) nsub := out.Len() for i := range nsub { diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index ebd1b416ba..9a70e99ee1 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -15,7 +15,7 @@ import ( ) // Groups generates indexes for each unique value in each of the given tensors. -// One can then use the resulting indexes for the [tensor.Indexed] indexes to +// One can then use the resulting indexes for the [tensor.Rows] indexes to // perform computations restricted to grouped subsets of data, as in the // [GroupStats] function. See [GroupCombined] for function that makes a // "Combined" Group that has a unique group for each _combination_ of @@ -36,7 +36,7 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) { if errors.Log(err) != nil { return } - makeIdxs := func(dir *datafs.Data, srt *tensor.Indexed, val string, start, r int) { + makeIdxs := func(dir *datafs.Data, srt *tensor.Rows, val string, start, r int) { n := r - start it := datafs.NewValue[int](dir, val, n) for j := range n { @@ -54,7 +54,7 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) { nm = strconv.Itoa(i) } td, _ := gd.Mkdir(nm) - srt := tensor.AsIndexed(tsr).CloneIndexes() + srt := tensor.AsRows(tsr).CloneIndexes() srt.SortStable(tensor.Ascending) start := 0 if tsr.IsString() { @@ -104,7 +104,7 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) { if errors.Log(err) != nil { return } - tsr := tensor.AsIndexed(tsrs[0]) + tsr := tensor.AsRows(tsrs[0]) nr := tsr.NumRows() if nr == 0 { return @@ -165,7 +165,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) { sv := datafs.NewValue[float64](vd, stnm, nv) for i, v := range vals { idx := tensor.AsIntSlice(v) - sg := tensor.NewIndexed(tsr, idx...) + sg := tensor.NewRows(tsr, idx...) tensor.Call(stat, sg, stout) sv.SetFloatRow(stout.Float1D(0), i) } diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index c71a85ee3b..2ab67091b4 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -14,7 +14,6 @@ import ( func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - // ix := tensor.NewIndexed(tensor.NewNumberFromSlice(vals...)) ix := tensor.NewNumberFromSlice(vals...) out := tensor.NewFloat64(1) @@ -91,7 +90,7 @@ func TestFuncs64(t *testing.T) { func TestFuncsInt(t *testing.T) { vals := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} tsr := tensor.NewNumberFromSlice(vals...) - ix := tensor.NewIndexed(tsr) + ix := tensor.NewRows(tsr) out := tensor.NewInt(1) results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} @@ -163,7 +162,7 @@ func TestFuncsCell(t *testing.T) { } } - ix := tensor.NewIndexed(tsr) + ix := tensor.NewRows(tsr) out := tensor.NewFloat32(20, 10) CountFunc(ix, out) diff --git a/tensor/string.go b/tensor/string.go index 9b3b129324..dc638a1ef7 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -291,7 +291,7 @@ func (tsr *String) SubSpace(offs ...int) Tensor { } // RowTensor is a convenience version of [Tensor.SubSpace] to return the -// SubSpace for the outermost row dimension. [Indexed] defines a version +// SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. func (tsr *String) RowTensor(row int) Tensor { return tsr.SubSpace(row) diff --git a/tensor/table/README.md b/tensor/table/README.md index 39909e0c4f..d732f94693 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -8,11 +8,11 @@ See [examples/dataproc](examples/dataproc) for a demo of how to use this system Whereas an individual `Tensor` can only hold one data type, the `Table` allows coordinated storage and processing of heterogeneous data types, aligned by the outermost row dimension. The main `tensor` data processing functions are defined on the individual tensors (which are the universal computational element in the `tensor` system), but the coordinated row-wise indexing in the table is important for sorting or filtering a collection of data in the same way, and grouping data by a common set of "splits" for data analysis. Plotting is also driven by the table, with one column providing a shared X axis for the rest of the columns. -The `Table` mainly provides "infrastructure" methods for adding tensor columns and CSV (comma separated values, and related tab separated values, TSV) file reading and writing. Any function that can be performed on an individual column should be done using the `tensor.Indexed` and `Tensor` methods directly. +The `Table` mainly provides "infrastructure" methods for adding tensor columns and CSV (comma separated values, and related tab separated values, TSV) file reading and writing. Any function that can be performed on an individual column should be done using the `tensor.Rows` and `Tensor` methods directly. As a general convention, it is safest, clearest, and quite fast to access columns by name instead of index (there is a `map` from name to index), so the base access method names generally take a column name argument, and those that take a column index have an `Index` suffix. -The table itself stores raw data `tensor.Tensor` values, and the `Column` (by name) and `ColumnByIndex` methods return a `tensor.Indexed` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). +The table itself stores raw data `tensor.Tensor` values, and the `Column` (by name) and `ColumnByIndex` methods return a `tensor.Rows` with the `Indexes` pointing to the shared table-wide `Indexes` (which can be `nil` if standard sequential order is being used). If you call Sort, Filter or other routines on an individual column tensor, then you can grab the updated indexes via the `IndexesFromTensor` method so that they apply to the entire table. The `SortColumn` and `FilterString` methods do this for you. @@ -29,7 +29,7 @@ It is very low-cost to create a new View of an existing Table, via `NewView`, as Column data access: ```Go -// FloatRow is a method on the `tensor.Indexed` returned from the `Column` method. +// FloatRow is a method on the `tensor.Rows` returned from the `Column` method. // This is the best method to use in general for generic 1D data access, // as it works on any data from 1D on up (although it only samples the first value // from higher dimensional data) . @@ -43,7 +43,7 @@ dt.Column("Name").SetStringRow(4) To access higher-dimensional "cell" level data using a simple 1D index into the cell patterns: ```Go -// FloatRowCell is a method on the `tensor.Indexed` returned from the `Column` method. +// FloatRowCell is a method on the `tensor.Rows` returned from the `Column` method. // This is the best method to use in general for generic 1D data access, // as it works on any data from 1D on up (although it only samples the first value // from higher dimensional data) . @@ -71,7 +71,7 @@ gps := byNm.AggsToTable(etable.AddAggName) // etable.AddAggName or etable.ColNam Describe (basic stats) all columns in a table: ```Go -ix := etable.NewIndexed(et) // new view with all rows +ix := etable.NewRows(et) // new view with all rows desc := agg.DescAll(ix) // summary stats of all columns // get value at given column name (from original table), row "Mean" mean := desc.Float("ColNm", desc.RowsByString("Agg", "Mean", etable.Equals, etable.UseCase)[0]) diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 2cd13bfa68..8964cb57a8 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -49,10 +49,10 @@ func (dt *Table) IndexesNeeded() { } } -// IndexesFromTensor copies Indexes from the given [tensor.Indexed] tensor, +// IndexesFromTensor copies Indexes from the given [tensor.Rows] tensor, // including if they are nil. This allows column-specific Sort, Filter and // other such methods to be applied to the entire table. -func (dt *Table) IndexesFromTensor(ix *tensor.Indexed) { +func (dt *Table) IndexesFromTensor(ix *tensor.Rows) { dt.Indexes = ix.Indexes } diff --git a/tensor/table/table.go b/tensor/table/table.go index f424dd8d91..36ff40d025 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -17,7 +17,7 @@ import ( // Table is a table of Tensor columns aligned by a common outermost row dimension. // Use the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a -// [tensor.Indexed] view of the column, using the shared [Table.Indexes] of the Table. +// [tensor.Rows] view of the column, using the shared [Table.Indexes] of the Table. // Thus, a coordinated sorting and filtered view of the column data is automatically // available for any of the tensor package functions that use [tensor.Tensor] as the one // common data representation for all operations. @@ -28,7 +28,7 @@ type Table struct { //types:add // Indexes are the indexes into Tensor rows, with nil = sequential. // Only set if order is different from default sequential order. - // These indexes are shared into the `tensor.Indexed` Column values + // These indexes are shared into the `tensor.Rows` Column values // to provide a coordinated indexed view into the underlying data. Indexes []int @@ -52,7 +52,7 @@ func NewTable(name ...string) *Table { return dt } -// NewView returns a new Table with its own Indexed view into the +// NewView returns a new Table with its own Rows view into the // same underlying set of Column tensor data as the source table. // Indexes are copied from the existing table -- use Sequential // to reset to full sequential view. @@ -76,22 +76,22 @@ func (dt *Table) IsValidRow(row int) error { // NumColumns returns the number of columns. func (dt *Table) NumColumns() int { return dt.Columns.Len() } -// Column returns the tensor with given column name, as a [tensor.Indexed] +// Column returns the tensor with given column name, as a [tensor.Rows] // with the shared [Table.Indexes] from this table. It is best practice to // access columns by name, and direct access through [Table.Columns] does not // provide the shared table-wide Indexes. // Returns nil if not found. -func (dt *Table) Column(name string) *tensor.Indexed { +func (dt *Table) Column(name string) *tensor.Rows { cl := dt.Columns.At(name) if cl == nil { return nil } - return tensor.NewIndexed(cl, dt.Indexes...) + return tensor.NewRows(cl, dt.Indexes...) } // ColumnTry is a version of [Table.Column] that also returns an error // if the column name is not found, for cases when error is needed. -func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { +func (dt *Table) ColumnTry(name string) (*tensor.Rows, error) { cl := dt.Column(name) if cl != nil { return cl, nil @@ -100,17 +100,17 @@ func (dt *Table) ColumnTry(name string) (*tensor.Indexed, error) { } // ColumnIndex returns the tensor at the given column index, -// as a [tensor.Indexed] with the shared [Table.Indexes] from this table. +// as a [tensor.Rows] with the shared [Table.Indexes] from this table. // It is best practice to instead access columns by name using [Table.Column]. // Direct access through [Table.Columns} does not provide the shared table-wide Indexes. // Will panic if out of range. -func (dt *Table) ColumnByIndex(idx int) *tensor.Indexed { +func (dt *Table) ColumnByIndex(idx int) *tensor.Rows { cl := dt.Columns.Values[idx] - return tensor.NewIndexed(cl, dt.Indexes...) + return tensor.NewRows(cl, dt.Indexes...) } // ColumnList returns a list of tensors with given column names, -// as [tensor.Indexed] with the shared [Table.Indexes] from this table. +// as [tensor.Rows] with the shared [Table.Indexes] from this table. func (dt *Table) ColumnList(names ...string) []tensor.Tensor { list := make([]tensor.Tensor, 0, len(names)) for _, nm := range names { diff --git a/tensor/tensor.go b/tensor/tensor.go index 81325c25d2..b837f93191 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -87,7 +87,7 @@ type Tensor interface { // RowCellSize returns the size of the outermost Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Commonly used to organize multiple instances (rows) of higher-dimensional - // patterns (cells), and the [Indexed] type operates on the outer row dimension. + // patterns (cells), and the [Rows] type operates on the outer row dimension. RowCellSize() (rows, cells int) // DataType returns the type of the data elements in the tensor. @@ -115,25 +115,25 @@ type Tensor interface { SetFloat(val float64, i ...int) // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. - // This can be somewhat expensive in wrapper views ([Indexed], [Sliced]), which + // This can be somewhat expensive in wrapper views ([Rows], [Sliced]), which // convert the flat index back into a full n-dimensional index and use that api. // [Tensor.FloatRowCell] is preferred. Float1D(i int) float64 // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. - // This can be somewhat expensive in the commonly-used [Indexed] view; + // This can be somewhat expensive in the commonly-used [Rows] view; // [Tensor.SetFloatRowCell] is preferred. SetFloat1D(val float64, i int) // FloatRowCell returns the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Indexed] tensors index along the row, and use this interface extensively. + // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. FloatRowCell(row, cell int) float64 // SetFloatRowCell sets the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Indexed] tensors index along the row, and use this interface extensively. + // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. SetFloatRowCell(val float64, row, cell int) @@ -166,13 +166,13 @@ type Tensor interface { // StringRowCell returns the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Indexed] tensors index along the row, and use this interface extensively. + // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. StringRowCell(row, cell int) string // SetStringRowCell sets the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Indexed] tensors index along the row, and use this interface extensively. + // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. SetStringRowCell(val string, row, cell int) @@ -204,13 +204,13 @@ type Tensor interface { // IntRowCell returns the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Indexed] tensors index along the row, and use this interface extensively. + // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. IntRowCell(row, cell int) int // SetIntRowCell sets the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Indexed] tensors index along the row, and use this interface extensively. + // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. SetIntRowCell(val int, row, cell int) @@ -238,7 +238,7 @@ type Tensor interface { SubSpace(offs ...int) Tensor // RowTensor is a convenience version of [Tensor.SubSpace] to return the - // SubSpace for the outermost row dimension. [Indexed] defines a version + // SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. RowTensor(row int) Tensor diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 830c092a6b..363cd6669e 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -131,7 +131,7 @@ func (tb *Table) StyleValue(w core.Widget, s *styles.Style, row, col int) { s.SetTextWrap(false) } -// SetTable sets the source table that we are viewing, using a sequential Indexed +// SetTable sets the source table that we are viewing, using a sequential view, // and then configures the display func (tb *Table) SetTable(dt *table.Table) *Table { if dt == nil { @@ -160,28 +160,6 @@ func (tb *Table) AsyncUpdateTable() { tb.AsyncUnlock() } -// SetIndexed sets the source Indexed of a table (using a copy so original is not modified) -// and then configures the display -func (tb *Table) SetIndexed(ix *table.Table) *Table { - if ix == nil { - return tb - } - - tb.Table = ix.Clone() // always copy - - tb.This.(core.Lister).UpdateSliceSize() - tb.StartIndex = 0 - tb.VisibleRows = tb.MinRows - if !tb.IsReadOnly() { - tb.SelectedIndex = -1 - } - tb.ResetSelectedIndexes() - tb.SelectMode = false - tb.MakeIter = 0 - tb.Update() - return tb -} - func (tb *Table) UpdateSliceSize() int { tb.Table.ValidIndexes() // table could have changed if tb.Table.NumRows() == 0 { diff --git a/tensor/tmath/README.md b/tensor/tmath/README.md index 1133efd10d..5748b3610a 100644 --- a/tensor/tmath/README.md +++ b/tensor/tmath/README.md @@ -2,7 +2,7 @@ # math functions -All the standard library [math](https://pkg.go.dev/math) functions are implemented on `*tensor.Indexed`. +All the standard library [math](https://pkg.go.dev/math) functions are implemented on `*tensor.Tensor`. To properly handle the row-wise indexes, all processing is done using row, cell indexes, with the row indirected through the indexes. diff --git a/tensor/tmath/doc.go b/tensor/tmath/doc.go index b5c427f75f..4a66e705f8 100644 --- a/tensor/tmath/doc.go +++ b/tensor/tmath/doc.go @@ -3,6 +3,6 @@ // license that can be found in the LICENSE file. /* -Package tmath provides basic math operations and functions that operate on tensor.Indexed. +Package tmath provides basic math operations and functions that operate on tensor.Tensor. */ package tmath diff --git a/tensor/tmath/norm.go b/tensor/tmath/norm.go index e58e7661ec..f70ea10013 100644 --- a/tensor/tmath/norm.go +++ b/tensor/tmath/norm.go @@ -13,7 +13,7 @@ import ( // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. func ZScore(a, out tensor.Tensor) { - mout := tensor.NewIndexed(tensor.NewFloat64()) + mout := tensor.NewFloat64() std, mean, _ := stats.StdFuncOut64(a, mout) Sub(a, mean, out) Div(out, std, out) @@ -22,7 +22,7 @@ func ZScore(a, out tensor.Tensor) { // UnitNorm computes unit normalized values into given output tensor, // subtracting the Min value and dividing by the Max of the remaining numbers. func UnitNorm(a, out tensor.Tensor) { - mout := tensor.NewIndexed(tensor.NewFloat64()) + mout := tensor.NewFloat64() stats.MinFunc(a, mout) Sub(a, mout, out) stats.MaxFunc(out, mout) diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index bbc90d1959..02bc0db7d3 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -130,7 +130,7 @@ func TestAdd(t *testing.T) { } ZScore(oned, oneout) - mout := tensor.NewIndexed(tensor.NewFloat64()) + mout := tensor.NewFloat64() std, mean, _ := stats.StdFuncOut64(oneout, mout) assert.InDelta(t, 1.0, std.Float1D(0), 1.0e-6) assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) diff --git a/tensor/vectorize.go b/tensor/vectorize.go index 01b0978109..1e5379fe1a 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -27,7 +27,7 @@ var ( // to vectorize over, and initializing any output vectors. // Thus the nfun is often specific to a particular class of functions. // Both functions are called with the same set -// of Indexed Tensors passed as the final argument(s). +// of Tensors passed as the final argument(s). // The role of each tensor is function-dependent: there could be multiple // inputs and outputs, and the output could be effectively scalar, // as in a sum operation. The interpretation of the index is From 40d78b4829441a971b88230f98be6b465b23250a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 00:36:19 -0700 Subject: [PATCH 084/311] first pass impl of Sliced -- needs tests --- tensor/sliced.go | 155 ++++++++++++++++++++--------------------------- tensor/string.go | 2 +- 2 files changed, 68 insertions(+), 89 deletions(-) diff --git a/tensor/sliced.go b/tensor/sliced.go index 5355407d97..1fdfb8fde6 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -16,6 +16,7 @@ import ( "gonum.org/v1/gonum/mat" ) +/* // Slice extracts a subset of values from the given tensor into the // output tensor, according to the provided ranges. // Dimensions beyond the ranges specified are automatically included. @@ -57,6 +58,7 @@ func Slice(tsr, out *Sliced, ranges ...Range) error { } return nil } +*/ // Sliced is a fully indexed wrapper around another [Tensor] that provides a // re-sliced view onto the Tensor defined by the set of [SlicedIndexes], @@ -104,6 +106,44 @@ func (sl *Sliced) SetTensor(tsr Tensor) { sl.Sequential() } +// SliceIndex returns the actual index into underlying tensor dimension +// based on given index value. +func (sl *Sliced) SliceIndex(dim, idx int) int { + ix := sl.Indexes[dim] + if ix == nil { + return idx + } + return ix[idx] +} + +// SliceIndexes returns the actual indexes into underlying tensor +// based on given list of indexes. +func (sl *Sliced) SliceIndexes(i ...int) []int { + ix := slices.Clone(i) + for d, idx := range i { + ix[d] = sl.SliceIndex(d, idx) + } + return ix +} + +// IndexFrom1D returns the full indexes into source tensor based on the +// given 1d index. +func (sl *Sliced) IndexFrom1D(oned int) []int { + sh := sl.Shape() + oix := sh.IndexFrom1D(oned) // full indexes in our coords + return sl.SliceIndexes(oix...) +} + +// RowCellIndex returns the full indexes into source tensor based on the +// row and cell index. Maps through 1D index. +func (sl *Sliced) RowCellIndex(row, cell int) []int { + sh := sl.Shape() + _, csz := sh.RowCellSize() // using our shape + oned := row*csz + cell + oix := sh.IndexFrom1D(oned) // full indexes in our coords + return sl.SliceIndexes(oix...) +} + // ValidIndexes ensures that [Sliced.Indexes] are valid, // removing any out-of-range values and setting the view to nil (full sequential) // for any dimension with no indexes (which is an invalid condition). @@ -398,32 +438,23 @@ func (sl *Sliced) CopyCellsFrom(from Tensor, to, start, n int) { ///////////////////// Floats -// todo: these are not the right functions - // Float returns the value of given index as a float64. -// The first index value is indirected through the indexes. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) Float(i ...int) float64 { - sh := sl.Shape() - return sl.Tensor.Float1D(sh.IndexTo1D()) + return sl.Tensor.Float(sl.SliceIndexes(i...)...) } // SetFloat sets the value of given index as a float64 -// The first index value is indirected through the [Sliced.Indexes]. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) SetFloat(val float64, i ...int) { - sh := sl.Shape() - sl.Tensor.SetFloat1D(val, sh.IndexTo1D()) + sl.Tensor.SetFloat(val, sl.SliceIndexes(i...)...) } // FloatRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Sliced.Indexes]. -// This is the preferred interface for all Sliced operations. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) FloatRowCell(row, cell int) float64 { - sh := sl.Shape() - _, sz := sh.RowCellSize() - i := row*sz + cell - return float64(tsr.Values[i]) - return sl.Tensor.FloatRowCell(sl.RowIndex(row), cell) + return sl.Float(sl.RowCellIndex(row, cell)...) } // SetFloatRowCell sets the value at given row and cell, @@ -431,25 +462,19 @@ func (sl *Sliced) FloatRowCell(row, cell int) float64 { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) SetFloatRowCell(val float64, row, cell int) { - sl.Tensor.SetFloatRowCell(val, sl.RowIndex(row), cell) + sl.SetFloat(val, sl.RowCellIndex(row, cell)...) } // Float1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Float1D(i int) float64 { - if sl.Indexes == nil { - return sl.Tensor.Float1D(i) - } - return sl.Float(sl.Tensor.Shape().IndexFrom1D(i)...) + return sl.Float(sl.IndexFrom1D(i)...) } // SetFloat1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetFloat1D(val float64, i int) { - if sl.Indexes == nil { - sl.Tensor.SetFloat1D(val, i) - } - sl.SetFloat(val, sl.Tensor.Shape().IndexFrom1D(i)...) + sl.SetFloat(val, sl.IndexFrom1D(i)...) } func (sl *Sliced) FloatRow(row int) float64 { @@ -465,23 +490,13 @@ func (sl *Sliced) SetFloatRow(val float64, row int) { // StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. func (sl *Sliced) StringValue(i ...int) string { - if sl.Indexes == nil { - return sl.Tensor.StringValue(i...) - } - ic := slices.Clone(i) - ic[0] = sl.Indexes[ic[0]] - return sl.Tensor.StringValue(ic...) + return sl.Tensor.StringValue(sl.SliceIndexes(i...)...) } // SetString sets the value of given index as a string // The first index value is indirected through the [Sliced.Indexes]. func (sl *Sliced) SetString(val string, i ...int) { - if sl.Indexes == nil { - sl.Tensor.SetString(val, i...) - } - ic := slices.Clone(i) - ic[0] = sl.Indexes[ic[0]] - sl.Tensor.SetString(val, ic...) + sl.Tensor.SetString(val, sl.SliceIndexes(i...)...) } // StringRowCell returns the value at given row and cell, @@ -489,7 +504,7 @@ func (sl *Sliced) SetString(val string, i ...int) { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) StringRowCell(row, cell int) string { - return sl.Tensor.StringRowCell(sl.RowIndex(row), cell) + return sl.StringValue(sl.RowCellIndex(row, cell)...) } // SetStringRowCell sets the value at given row and cell, @@ -497,25 +512,19 @@ func (sl *Sliced) StringRowCell(row, cell int) string { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) SetStringRowCell(val string, row, cell int) { - sl.Tensor.SetStringRowCell(val, sl.RowIndex(row), cell) + sl.SetString(val, sl.RowCellIndex(row, cell)...) } // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) String1D(i int) string { - if sl.Indexes == nil { - return sl.Tensor.String1D(i) - } - return sl.StringValue(sl.Tensor.Shape().IndexFrom1D(i)...) + return sl.StringValue(sl.IndexFrom1D(i)...) } // SetString1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetString1D(val string, i int) { - if sl.Indexes == nil { - sl.Tensor.SetString1D(val, i) - } - sl.SetString(val, sl.Tensor.Shape().IndexFrom1D(i)...) + sl.SetString(val, sl.IndexFrom1D(i)...) } func (sl *Sliced) StringRow(row int) string { @@ -531,24 +540,13 @@ func (sl *Sliced) SetStringRow(val string, row int) { // Int returns the value of given index as an int. // The first index value is indirected through the indexes. func (sl *Sliced) Int(i ...int) int { - if sl.Indexes == nil { - return sl.Tensor.Int(i...) - } - ic := slices.Clone(i) - ic[0] = sl.Indexes[ic[0]] - return sl.Tensor.Int(ic...) + return sl.Tensor.Int(sl.SliceIndexes(i...)...) } // SetInt sets the value of given index as an int // The first index value is indirected through the [Sliced.Indexes]. func (sl *Sliced) SetInt(val int, i ...int) { - if sl.Indexes == nil { - sl.Tensor.SetInt(val, i...) - return - } - ic := slices.Clone(i) - ic[0] = sl.Indexes[ic[0]] - sl.Tensor.SetInt(val, ic...) + sl.Tensor.SetInt(val, sl.SliceIndexes(i...)...) } // IntRowCell returns the value at given row and cell, @@ -556,7 +554,7 @@ func (sl *Sliced) SetInt(val int, i ...int) { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) IntRowCell(row, cell int) int { - return sl.Tensor.IntRowCell(sl.RowIndex(row), cell) + return sl.Int(sl.RowCellIndex(row, cell)...) } // SetIntRowCell sets the value at given row and cell, @@ -564,25 +562,19 @@ func (sl *Sliced) IntRowCell(row, cell int) int { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) SetIntRowCell(val int, row, cell int) { - sl.Tensor.SetIntRowCell(val, sl.RowIndex(row), cell) + sl.SetInt(val, sl.RowCellIndex(row, cell)...) } // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Int1D(i int) int { - if sl.Indexes == nil { - return sl.Tensor.Int1D(i) - } - return sl.Int(sl.Tensor.Shape().IndexFrom1D(i)...) + return sl.Int(sl.IndexFrom1D(i)...) } // SetInt1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetInt1D(val int, i int) { - if sl.Indexes == nil { - sl.Tensor.SetInt1D(val, i) - } - sl.SetInt(val, sl.Tensor.Shape().IndexFrom1D(i)...) + sl.SetInt(val, sl.IndexFrom1D(i)...) } func (sl *Sliced) IntRow(row int) int { @@ -595,38 +587,25 @@ func (sl *Sliced) SetIntRow(val int, row int) { ///////////////////// SubSpaces -// SubSpace returns a new tensor with innermost subspace at given -// offset(s) in outermost dimension(s) (len(offs) < NumDims). -// The new tensor points to the values of the this tensor (i.e., modifications -// will affect both), as its Values slice is a view onto the original (which -// is why only inner-most contiguous supsaces are supported). -// Use Clone() method to separate the two. -// Sliced version does indexed indirection of the outermost row dimension -// of the offsets. +// SubSpace is not supported by Sliced. func (sl *Sliced) SubSpace(offs ...int) Tensor { - if len(offs) == 0 { - return nil - } - offs[0] = sl.RowIndex(offs[0]) - return sl.Tensor.SubSpace(offs...) + return nil } -// RowTensor is a convenience version of [Sliced.SubSpace] to return the -// SubSpace for the outermost row dimension, indirected through the indexes. +// RowTensor is not supported by Sliced. func (sl *Sliced) RowTensor(row int) Tensor { - return sl.Tensor.RowTensor(sl.RowIndex(row)) + return nil } -// SetRowTensor sets the values of the SubSpace at given row to given values, -// with row indirected through the indexes. +// RowTensor is not supported by Sliced. func (sl *Sliced) SetRowTensor(val Tensor, row int) { - sl.Tensor.SetRowTensor(val, sl.RowIndex(row)) } // CopyFrom copies all values from other tensor into this tensor. // Checks if source is an Sliced and copies indexes too, // otherwise underlying tensor copies from and indexes are reset. func (sl *Sliced) CopyFrom(from Tensor) { + // todo: what should this do? if fix, ok := from.(*Sliced); ok { sl.Tensor.CopyFrom(fix.Tensor) sl.CopyIndexes(fix) diff --git a/tensor/string.go b/tensor/string.go index dc638a1ef7..f2e3c5dcb9 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -53,7 +53,7 @@ func Float64ToString(val float64) string { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *String) String() string { - return stringIndexed(tsr, 0, nil) + return sprint(tsr, 0) } func (tsr *String) IsString() bool { From acf823d9626a50691819f923bf653cf0ebfcdf81 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 02:34:53 -0700 Subject: [PATCH 085/311] sliced test working; damn index clone issue again --- tensor/base.go | 2 +- tensor/funcs_test.go | 2 +- tensor/ranges.go | 11 +++++ tensor/shape.go | 2 +- tensor/sliced.go | 54 +++------------------- tensor/tensor_test.go | 102 +++++++++++++++++------------------------- 6 files changed, 60 insertions(+), 113 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index 87f99b6223..476b1d8cb8 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -216,7 +216,7 @@ func sprint(tsr Tensor, maxLen int) string { } var b strings.Builder sh := tsr.Shape() - b.WriteString(sh.String() + " ") + b.WriteString(sh.String() + "\n") oddRow := false rows, cols, _, _ := Projection2DShape(sh, oddRow) ctr := 0 diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 87096057d8..5171d59c59 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -30,7 +30,7 @@ func TestFuncs(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := NewIndexed(NewNumberFromSlice(vals...)) + oned := NewNumberFromSlice(vals...) oneout := oned.Clone() err = Call("Abs", oned, oneout) diff --git a/tensor/ranges.go b/tensor/ranges.go index 7d3a5b9c32..672dcf73ae 100644 --- a/tensor/ranges.go +++ b/tensor/ranges.go @@ -75,3 +75,14 @@ func RangeSizes(sizes []int, ranges ...Range) ([]int, error) { } return nsz, nil } + +func IntRangeTensor(start, end, incr int) *Int { + rn := Range{Start: start, End: end, Incr: incr} + tsr := NewInt(rn.Size(end)) + idx := 0 + for i := start; i < end; i += incr { + tsr.Values[idx] = i + idx++ + } + return tsr +} diff --git a/tensor/shape.go b/tensor/shape.go index 6031df3708..20a6751f96 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -34,7 +34,7 @@ func NewShape(sizes ...int) *Shape { // SetShapeInts sets the shape sizes from list of ints. // RowMajor ordering is used by default. func (sh *Shape) SetShapeInts(sizes ...int) { - sh.Sizes = sizes + sh.Sizes = slices.Clone(sizes) sh.Strides = RowMajorStrides(sizes...) } diff --git a/tensor/sliced.go b/tensor/sliced.go index 1fdfb8fde6..9c1713960f 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -16,50 +16,6 @@ import ( "gonum.org/v1/gonum/mat" ) -/* -// Slice extracts a subset of values from the given tensor into the -// output tensor, according to the provided ranges. -// Dimensions beyond the ranges specified are automatically included. -// Unlike the [Tensor.SubSlice] function, the values extracted here are -// copies of the original, not a slice pointer into them, -// which is necessary to allow discontinuous ranges to be extracted. -// Use the [SliceSet] function to copy sliced values back to the original. -func Slice(tsr, out *Sliced, ranges ...Range) error { - sizes := slices.Clone(tsr.Tensor.ShapeInts()) - sizes[0] = tsr.DimSize(0) // takes into account indexes - nsz, err := SliceSize(sizes, ranges...) - if err != nil { - return err - } - ndim := len(nsz) - out.Tensor.SetShapeInts(nsz...) - out.Sequential() - nl := out.Len() - oc := make([]int, ndim) // orig coords - nr := len(ranges) - for ni := range nl { - nc := out.Tensor.Shape().IndexFrom1D(ni) - for i := range ndim { - c := nc[i] - if i < nr { - r := ranges[i] - oc[i] = r.Start + c*r.IncrActual() - } else { - oc[i] = c - } - } - oc[0] = tsr.RowIndex(oc[0]) - oi := tsr.Tensor.Shape().IndexTo1D(oc...) - if out.Tensor.IsString() { - out.Tensor.SetString1D(tsr.Tensor.String1D(oi), ni) - } else { - out.SetFloat1D(tsr.Float1D(oi), ni) - } - } - return nil -} -*/ - // Sliced is a fully indexed wrapper around another [Tensor] that provides a // re-sliced view onto the Tensor defined by the set of [SlicedIndexes], // for each dimension (must have at least 1 per dimension). @@ -200,7 +156,7 @@ func (sl *Sliced) Label() string { // String satisfies the fmt.Stringer interface for string of tensor data. func (sl *Sliced) String() string { - return sprint(sl.Tensor, 0) // todo: no need + return sprint(sl, 0) } // Metadata returns the metadata for this tensor, which can be used @@ -454,7 +410,7 @@ func (sl *Sliced) SetFloat(val float64, i ...int) { // where row is outermost dim, and cell is 1D index into remaining inner dims. // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) FloatRowCell(row, cell int) float64 { - return sl.Float(sl.RowCellIndex(row, cell)...) + return sl.Tensor.Float(sl.RowCellIndex(row, cell)...) } // SetFloatRowCell sets the value at given row and cell, @@ -462,19 +418,19 @@ func (sl *Sliced) FloatRowCell(row, cell int) float64 { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) SetFloatRowCell(val float64, row, cell int) { - sl.SetFloat(val, sl.RowCellIndex(row, cell)...) + sl.Tensor.SetFloat(val, sl.RowCellIndex(row, cell)...) } // Float1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Float1D(i int) float64 { - return sl.Float(sl.IndexFrom1D(i)...) + return sl.Tensor.Float(sl.IndexFrom1D(i)...) } // SetFloat1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetFloat1D(val float64, i int) { - sl.SetFloat(val, sl.IndexFrom1D(i)...) + sl.Tensor.SetFloat(val, sl.IndexFrom1D(i)...) } func (sl *Sliced) FloatRow(row int) float64 { diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 421f85d245..f8b94ffb97 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -123,76 +123,56 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 0.0, af[1]) } -/* func TestSlice(t *testing.T) { - ft := NewFloat64(3, 4, 5) - for r := range 3 { - for y := range 4 { - for x := range 5 { - v := (r+1)*100 + y*10 + x - ft.SetFloat(float64(v), r, y, x) - } + ft := NewFloat64(3, 4) + for y := range 3 { + for x := range 4 { + v := y*10 + x + ft.SetFloat(float64(v), y, x) } } - res := `Tensor: [20] -[0]: 200 -[1]: 201 -[2]: 202 -[3]: 203 -[4]: 204 -[5]: 210 -[6]: 211 -[7]: 212 -[8]: 213 -[9]: 214 -[10]: 220 -[11]: 221 -[12]: 222 -[13]: 223 -[14]: 224 -[15]: 230 -[16]: 231 -[17]: 232 -[18]: 233 -[19]: 234 -` - // fmt.Println(Cells1D(ft, 1)) - assert.Equal(t, res, Cells1D(ft, 1).String()) - - // fmt.Println(ft.String()) - sf := NewFloat64() - Slice(ft, sf, Range{}, Range{Start: 1, End: 2}) - // fmt.Println(sf.String()) - res = `Tensor: [3, 1, 5] -[0 0]: 110 111 112 113 114 -[1 0]: 210 211 212 213 214 -[2 0]: 310 311 312 313 314 + res := `[3, 4] +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 ` - assert.Equal(t, res, sf.Tensor.String()) - - Slice(ft, sf, Range{}, Range{}, Range{Start: 1, End: 2}) - // fmt.Println(sf.String()) - res = `Tensor: [3, 4, 1] -[0 0]: 101 -[0 1]: 111 -[0 2]: 121 -[0 3]: 131 -[1 0]: 201 -[1 1]: 211 -[1 2]: 221 -[1 3]: 231 -[2 0]: 301 -[2 1]: 311 -[2 2]: 321 -[2 3]: 331 + assert.Equal(t, res, ft.String()) + // fmt.Println(ft) + + res = `[2, 2] +[0]: 23 22 +[1]: 13 12 ` - assert.Equal(t, res, sf.Tensor.String()) + + sl := NewSliced(ft, []int{2, 1}, []int{3, 2}) + // fmt.Println(sl) + assert.Equal(t, res, sl.String()) + + /* + + ft := NewFloat64(2, 3, 4) + for r := range 2 { + for y := range 3 { + for x := range 4 { + v := (r+1)*100 + y*10 + x + ft.SetFloat(float64(v), r, y, x) + } + } + } + + fmt.Println(ft) + + sl := NewSliced(ft, []int{1, 0}, []int{1, 0}, []int{1, 0}) + fmt.Println(sl) + + assert.Equal(t, res, sf.Tensor.String()) + */ + } -*/ func TestSortFilter(t *testing.T) { - tsr := NewIndexed(NewFloat64(5)) + tsr := NewRows(NewFloat64(5)) for i := range 5 { tsr.SetFloatRowCell(float64(i), i, 0) } From 11e90305b972a76d9616e2778195fa8206de657d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 12:12:52 -0700 Subject: [PATCH 086/311] Slice type fully tested and debugged -- surprisingly tricky.. --- tensor/ranges.go | 88 ----------------- tensor/sliced.go | 38 +++++--- tensor/slices.go | 146 +++++++++++++++++++++++++++++ tensor/slices_test.go | 63 +++++++++++++ tensor/stats/metric/matrix.go | 4 +- tensor/stats/metric/metric_test.go | 43 ++++----- tensor/stats/stats/quantiles.go | 2 +- tensor/table/table.go | 1 + tensor/tensor_test.go | 4 +- 9 files changed, 264 insertions(+), 125 deletions(-) delete mode 100644 tensor/ranges.go create mode 100644 tensor/slices.go create mode 100644 tensor/slices_test.go diff --git a/tensor/ranges.go b/tensor/ranges.go deleted file mode 100644 index 672dcf73ae..0000000000 --- a/tensor/ranges.go +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tensor - -import ( - "fmt" - "slices" -) - -// todo: make a version of these functions that takes -// a tensor with n x 3 shape, where the 3 inner values -// specify the Range Start, End, Incr values, across n ranges. -// these would convert to the current Range-based format that does the impl, -// using the Range helper functions, which are also easier and more explicit -// to use in Go code. - -// Range represents a range of values, for extracting slices of data, -// using standard for loop logic with a Start and exclusive End value, -// and an increment: for i := Start; i < End; i += Incr. -// The zero value means all values in the dimension. -type Range struct { - // Starting value. - Start int - - // End value. 0 default = size of relevant dimension. - End int - - // Increment. must be positive, 1 or greater. 0 default = 1. - Incr int -} - -// EndActual is the actual end value given the size of the dimension. -func (rn *Range) EndActual(size int) int { - if rn.End == 0 { - return size - } - return min(rn.End, size) // preserves -1 for no values. -} - -// IncrActual is the actual increment value. -func (rn *Range) IncrActual() int { - return max(1, rn.Incr) -} - -// Size is the number of elements in the actual range given -// size of the dimension. -func (rn *Range) Size(size int) int { - e := rn.EndActual(size) - if e <= rn.Start { - return 0 - } - i := rn.IncrActual() - return (e - rn.Start) / i -} - -// RangeSizes returns a set of sizes applying the ranges, in order, to -// the given dimension sizes. It is important that all dimensions -// are non-zero, otherwise nothing will be included. -// An error is returned if this is the case. -// Dimensions beyond the ranges specified are automatically included. -func RangeSizes(sizes []int, ranges ...Range) ([]int, error) { - nsz := slices.Clone(sizes) - mx := min(len(ranges), len(sizes)) - var zd []int - for i := range mx { - nsz[i] = ranges[i].Size(sizes[i]) - if nsz[i] == 0 { - zd = append(zd, i) - } - } - if len(zd) > 0 { - return nsz, fmt.Errorf("tensor.Shape Slice has zero size for following dimensions: %v", zd) - } - return nsz, nil -} - -func IntRangeTensor(start, end, incr int) *Int { - rn := Range{Start: start, End: end, Incr: incr} - tsr := NewInt(rn.Size(end)) - idx := 0 - for i := start; i < end; i += incr { - tsr.Values[idx] = i - idx++ - } - return tsr -} diff --git a/tensor/sliced.go b/tensor/sliced.go index 9c1713960f..1800eb01c5 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -39,14 +39,30 @@ type Sliced struct { //types:add Indexes [][]int } -// NewSliced returns a new [Sliced] view of given tensor, -// with optional list of indexes (none / nil = sequential). -func NewSliced(tsr Tensor, idxs ...[]int) *Sliced { +// NewSlicedIndexes returns a new [Sliced] view of given tensor, +// with optional list of indexes for each dimension (none / nil = sequential). +func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { sl := &Sliced{Tensor: tsr, Indexes: idxs} sl.ValidIndexes() return sl } +// NewSliced returns a new [Sliced] view of given tensor, +// with given slices for each dimension (none / nil = sequential). +func NewSliced(tsr Tensor, sls ...Slice) *Sliced { + ns := len(sls) + if ns == 0 { + return NewSlicedIndexes(tsr) + } + ns = min(ns, tsr.NumDims()) + ixs := make([][]int, ns) + for d := range ns { + sl := sls[d] + ixs[d] = sl.IntSlice(tsr.DimSize(d)) + } + return NewSlicedIndexes(tsr, ixs...) +} + // AsSliced returns the tensor as a [Sliced] view. // If it already is one, then it is returned, otherwise it is wrapped. func AsSliced(tsr Tensor) *Sliced { @@ -460,7 +476,7 @@ func (sl *Sliced) SetString(val string, i ...int) { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) StringRowCell(row, cell int) string { - return sl.StringValue(sl.RowCellIndex(row, cell)...) + return sl.Tensor.StringValue(sl.RowCellIndex(row, cell)...) } // SetStringRowCell sets the value at given row and cell, @@ -468,19 +484,19 @@ func (sl *Sliced) StringRowCell(row, cell int) string { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) SetStringRowCell(val string, row, cell int) { - sl.SetString(val, sl.RowCellIndex(row, cell)...) + sl.Tensor.SetString(val, sl.RowCellIndex(row, cell)...) } // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) String1D(i int) string { - return sl.StringValue(sl.IndexFrom1D(i)...) + return sl.Tensor.StringValue(sl.IndexFrom1D(i)...) } // SetString1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetString1D(val string, i int) { - sl.SetString(val, sl.IndexFrom1D(i)...) + sl.Tensor.SetString(val, sl.IndexFrom1D(i)...) } func (sl *Sliced) StringRow(row int) string { @@ -510,7 +526,7 @@ func (sl *Sliced) SetInt(val int, i ...int) { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) IntRowCell(row, cell int) int { - return sl.Int(sl.RowCellIndex(row, cell)...) + return sl.Tensor.Int(sl.RowCellIndex(row, cell)...) } // SetIntRowCell sets the value at given row and cell, @@ -518,19 +534,19 @@ func (sl *Sliced) IntRowCell(row, cell int) int { // Row is indirected through the [Sliced.Indexes]. // This is the preferred interface for all Sliced operations. func (sl *Sliced) SetIntRowCell(val int, row, cell int) { - sl.SetInt(val, sl.RowCellIndex(row, cell)...) + sl.Tensor.SetInt(val, sl.RowCellIndex(row, cell)...) } // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Int1D(i int) int { - return sl.Int(sl.IndexFrom1D(i)...) + return sl.Tensor.Int(sl.IndexFrom1D(i)...) } // SetInt1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetInt1D(val int, i int) { - sl.SetInt(val, sl.IndexFrom1D(i)...) + sl.Tensor.SetInt(val, sl.IndexFrom1D(i)...) } func (sl *Sliced) IntRow(row int) int { diff --git a/tensor/slices.go b/tensor/slices.go new file mode 100644 index 0000000000..4e6cce0f0c --- /dev/null +++ b/tensor/slices.go @@ -0,0 +1,146 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +// Slice represents a slice of index values, for extracting slices of data, +// along a dimension of a given size, which is provided separately as an argument. +// using standard 'for' loop logic with a Start and _exclusive_ Stop value, +// and an increment: for i := Start; i < Stop; i += Step. +// The values stored in this struct are the _inputs_ for computing the actual +// slice ranges based on an additional actual size parameter for the dimension. +// Negative numbers count back from the end (i.e., size + val), and +// the zero value results in all values in the dimension, with Step = 1 if 0. +type Slice struct { + // Starting value. If 0 and Step < 0, = size-1; + // If negative, = size+Start. + Start int + + // Stop value. If 0 and Step >= 0, = size; + // If 0 and Step < 0, = -1, to include whole range. + // If negative = size+Stop. + Stop int + + // Step increment. If 0, = 1; if negative then Start must be > Stop + // to produce anything. + Step int +} + +// NewSlice returns a new Slice with given values. +func NewSlice(start, end, incr int) Slice { + return Slice{Start: start, Stop: end, Step: incr} +} + +// StartActual is the actual start value given the size of the dimension. +func (sl Slice) StartActual(size int) int { + if sl.Start == 0 && sl.Step < 0 { + return size - 1 + } + if sl.Start < 0 { + return size + sl.Start + } + return sl.Start +} + +// StopActual is the actual end value given the size of the dimension. +func (sl Slice) StopActual(size int) int { + if sl.Stop == 0 && sl.Step >= 0 { + return size + } + if sl.Stop == 0 && sl.Step < 0 { + return -1 + } + if sl.Stop < 0 { + return size + sl.Stop + } + return min(sl.Stop, size) +} + +// StepActual is the actual increment value. +func (sl Slice) StepActual() int { + if sl.Step == 0 { + return 1 + } + return sl.Step +} + +// Len is the number of elements in the actual slice given +// size of the dimension. +func (sl Slice) Len(size int) int { + s := sl.StartActual(size) + e := sl.StopActual(size) + i := sl.StepActual() + n := max((e-s)/i, 0) + pe := s + n*i + if i < 0 { + if pe > e { + n++ + } + } else { + if pe < e { + n++ + } + } + return n +} + +// ToIntSlice writes values to given []int slice, with given size parameter +// for the dimension being sliced. If slice is wrong size to hold values, +// not all are written: allocate ints using Len(size) to fit. +func (sl Slice) ToIntSlice(size int, ints []int) { + n := len(ints) + if n == 0 { + return + } + s := sl.StartActual(size) + e := sl.StopActual(size) + inc := sl.StepActual() + idx := 0 + if inc < 0 { + for i := s; i > e; i += inc { + ints[idx] = i + idx++ + if idx >= n { + break + } + } + } else { + for i := s; i < e; i += inc { + ints[idx] = i + idx++ + if idx >= n { + break + } + } + } +} + +// IntSlice returns []int slice with slice index values, up to given actual size. +func (sl Slice) IntSlice(size int) []int { + n := sl.Len(size) + if n == 0 { + return nil + } + ints := make([]int, n) + sl.ToIntSlice(size, ints) + return ints +} + +// IntTensor returns an [Int] [Tensor] for slice, using actual size. +func (sl Slice) IntTensor(size int) *Int { + n := sl.Len(size) + if n == 0 { + return nil + } + tsr := NewInt(n) + sl.ToIntSlice(size, tsr.Values) + return tsr +} + +// IntSlice returns a new [Int] [Tensor] with given [Slice] values. +// Stop must be an actual end value and not a size-relative number (such as -1). +func IntSlice(start, end, incr int) *Int { + sl := NewSlice(start, end, incr) + return sl.IntTensor(end) +} diff --git a/tensor/slices_test.go b/tensor/slices_test.go new file mode 100644 index 0000000000..bbfa14b47a --- /dev/null +++ b/tensor/slices_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSlice(t *testing.T) { + assert.Equal(t, 3, Slice{}.Len(3)) + assert.Equal(t, 3, Slice{0, 3, 0}.Len(3)) + assert.Equal(t, 3, Slice{0, 3, 1}.Len(3)) + + assert.Equal(t, 2, Slice{0, 0, 2}.Len(3)) + assert.Equal(t, 2, Slice{0, 0, 2}.Len(4)) + assert.Equal(t, 1, Slice{0, 0, 3}.Len(3)) + assert.Equal(t, 2, Slice{0, 0, 3}.Len(4)) + assert.Equal(t, 2, Slice{0, 0, 3}.Len(6)) + assert.Equal(t, 3, Slice{0, 0, 3}.Len(7)) + + assert.Equal(t, 1, Slice{-1, 0, 0}.Len(3)) + assert.Equal(t, 2, Slice{0, -1, 0}.Len(3)) + assert.Equal(t, 3, Slice{0, 0, -1}.Len(3)) + assert.Equal(t, 3, Slice{-1, 0, -1}.Len(3)) + assert.Equal(t, 1, Slice{-1, -2, -1}.Len(3)) + assert.Equal(t, 2, Slice{-1, -3, -1}.Len(3)) + + assert.Equal(t, 2, Slice{0, 0, -2}.Len(3)) + assert.Equal(t, 2, Slice{0, 0, -2}.Len(4)) + assert.Equal(t, 1, Slice{0, 0, -3}.Len(3)) + assert.Equal(t, 2, Slice{0, 0, -3}.Len(4)) + assert.Equal(t, 2, Slice{0, 0, -3}.Len(6)) + assert.Equal(t, 3, Slice{0, 0, -3}.Len(7)) + + assert.Equal(t, []int{0, 1, 2}, Slice{}.IntSlice(3)) + assert.Equal(t, []int{0, 1, 2}, Slice{0, 3, 0}.IntSlice(3)) + assert.Equal(t, []int{0, 1, 2}, Slice{0, 3, 1}.IntSlice(3)) + + assert.Equal(t, []int{0, 2}, Slice{0, 0, 2}.IntSlice(3)) + assert.Equal(t, []int{0, 2}, Slice{0, 0, 2}.IntSlice(4)) + assert.Equal(t, []int{0}, Slice{0, 0, 3}.IntSlice(3)) + assert.Equal(t, []int{0, 3}, Slice{0, 0, 3}.IntSlice(4)) + assert.Equal(t, []int{0, 3}, Slice{0, 0, 3}.IntSlice(6)) + assert.Equal(t, []int{0, 3, 6}, Slice{0, 0, 3}.IntSlice(7)) + + assert.Equal(t, []int{2}, Slice{-1, 0, 0}.IntSlice(3)) + assert.Equal(t, []int{0, 1}, Slice{0, -1, 0}.IntSlice(3)) + assert.Equal(t, []int{2, 1, 0}, Slice{0, 0, -1}.IntSlice(3)) + assert.Equal(t, []int{2, 1, 0}, Slice{-1, 0, -1}.IntSlice(3)) + assert.Equal(t, []int{2}, Slice{-1, -2, -1}.IntSlice(3)) + assert.Equal(t, []int{2, 1}, Slice{-1, -3, -1}.IntSlice(3)) + + assert.Equal(t, []int{2, 0}, Slice{0, 0, -2}.IntSlice(3)) + assert.Equal(t, []int{3, 1}, Slice{0, 0, -2}.IntSlice(4)) + assert.Equal(t, []int{2}, Slice{0, 0, -3}.IntSlice(3)) + assert.Equal(t, []int{3, 0}, Slice{0, 0, -3}.IntSlice(4)) + assert.Equal(t, []int{5, 2}, Slice{0, 0, -3}.IntSlice(6)) + assert.Equal(t, []int{6, 3, 0}, Slice{0, 0, -3}.IntSlice(7)) +} diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index e339a043cb..b1a41db441 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -125,11 +125,11 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { func(idx int, tsr ...tensor.Tensor) { c := coords[idx] if c.X != curCoords.X { - // tensor.Slice(tsr[0], av, tensor.Range{}, tensor.Range{Start: c.X, End: c.X + 1}) + tensor.Slice(tsr[0], av, tensor.Range{}, tensor.Range{Start: c.X, End: c.X + 1}) curCoords.X = c.X } if c.Y != curCoords.Y { - // tensor.Slice(tsr[0], bv, tensor.Range{}, tensor.Range{Start: c.Y, End: c.Y + 1}) + tensor.Slice(tsr[0], bv, tensor.Range{}, tensor.Range{Start: c.Y, End: c.Y + 1}) curCoords.Y = c.Y } tensor.Call(funcName, av, bv, mout) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 0922b5855a..3c50e8cad5 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -5,6 +5,7 @@ package metric import ( + "fmt" "math" "testing" @@ -21,58 +22,58 @@ func TestFuncs(t *testing.T) { tol := 1.0e-8 - atsr := tensor.NewIndexed(tensor.NewNumberFromSlice(a64...)) - btsr := tensor.NewIndexed(tensor.NewNumberFromSlice(b64...)) + atsr := tensor.NewNumberFromSlice(a64...) + btsr := tensor.NewNumberFromSlice(b64...) out := tensor.NewFloat64(1) - oix := tensor.NewIndexed(out) - EuclideanFunc(atsr, btsr, oix) + EuclideanFunc(atsr, btsr, out) assert.InDelta(t, results[Euclidean], out.Values[0], tol) - SumSquaresFunc(atsr, btsr, oix) + SumSquaresFunc(atsr, btsr, out) assert.InDelta(t, results[SumSquares], out.Values[0], tol) - EuclideanBinTolFunc(atsr, btsr, oix) + EuclideanBinTolFunc(atsr, btsr, out) assert.InDelta(t, results[EuclideanBinTol], out.Values[0], tol) - AbsFunc(atsr, btsr, oix) + AbsFunc(atsr, btsr, out) assert.InDelta(t, results[Abs], out.Values[0], tol) - HammingFunc(atsr, btsr, oix) + HammingFunc(atsr, btsr, out) assert.Equal(t, results[Hamming], out.Values[0]) - SumSquaresBinTolFunc(atsr, btsr, oix) + SumSquaresBinTolFunc(atsr, btsr, out) assert.InDelta(t, results[SumSquaresBinTol], out.Values[0], tol) - CovarianceFunc(atsr, btsr, oix) + CovarianceFunc(atsr, btsr, out) assert.InDelta(t, results[Covariance], out.Values[0], tol) - CorrelationFunc(atsr, btsr, oix) + CorrelationFunc(atsr, btsr, out) assert.InDelta(t, results[Correlation], out.Values[0], tol) - InvCorrelationFunc(atsr, btsr, oix) + InvCorrelationFunc(atsr, btsr, out) assert.InDelta(t, results[InvCorrelation], out.Values[0], tol) - CrossEntropyFunc(atsr, btsr, oix) + CrossEntropyFunc(atsr, btsr, out) assert.InDelta(t, results[CrossEntropy], out.Values[0], tol) - InnerProductFunc(atsr, btsr, oix) + InnerProductFunc(atsr, btsr, out) assert.InDelta(t, results[InnerProduct], out.Values[0], tol) - CosineFunc(atsr, btsr, oix) + CosineFunc(atsr, btsr, out) assert.InDelta(t, results[Cosine], out.Values[0], tol) - InvCosineFunc(atsr, btsr, oix) + InvCosineFunc(atsr, btsr, out) assert.InDelta(t, results[InvCosine], out.Values[0], tol) for met := Euclidean; met < MetricsN; met++ { - Metric(met, atsr, btsr, oix) + Metric(met, atsr, btsr, out) assert.InDelta(t, results[met], out.Values[0], tol) } } func TestMatrix(t *testing.T) { - var simres = `[12, 12] [0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 + var simres = `[12, 12] +[0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 [1]: 3.4641016151377544 0 9.38083151964686 8.717797887081348 9.273618495495704 8.831760866327848 5.830951894845301 4.69041575982343 8.717797887081348 7.937253933193772 6.324555320336759 5.291502622129181 [2]: 8.831760866327848 9.38083151964686 0 3.4641016151377544 4.242640687119285 5.0990195135927845 9.38083151964686 9.899494936611665 4.47213595499958 5.744562646538029 9.38083151964686 9.899494936611665 [3]: 9.273618495495704 8.717797887081348 3.4641016151377544 0 5.477225575051661 3.7416573867739413 9.797958971132712 9.273618495495704 5.656854249492381 4.58257569495584 9.797958971132712 9.273618495495704 @@ -109,14 +110,14 @@ func TestPCAIris(t *testing.T) { data := dt.Column("data") covar := tensor.NewFloat64() CovarianceMatrix(Correlation.FuncName(), data, covar) - // fmt.Printf("correl: %s\n", covar.String()) + fmt.Printf("correl: %s\n", covar.String()) vecs := tensor.NewFloat64() vals := tensor.NewFloat64() PCA(covar, vecs, vals) - // fmt.Printf("correl vec: %v\n", vecs) - // fmt.Printf("correl val: %v\n", vals) + fmt.Printf("correl vec: %v\n", vecs) + fmt.Printf("correl val: %v\n", vals) errtol := 1.0e-9 corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} for i, v := range vals.Values { diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 92a901f315..dbd72999fc 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -26,7 +26,7 @@ func QuantilesFunc(in, qs, out tensor.Tensor) error { if qs.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } - sin := tensor.AsIndexed(in.Clone()) + sin := tensor.AsRows(in.Clone()) sin.ExcludeMissing() sin.Sort(tensor.Ascending) tensor.SetShapeFrom(out, qs) diff --git a/tensor/table/table.go b/tensor/table/table.go index 36ff40d025..30b05c648b 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -21,6 +21,7 @@ import ( // Thus, a coordinated sorting and filtered view of the column data is automatically // available for any of the tensor package functions that use [tensor.Tensor] as the one // common data representation for all operations. +// Tensor Columns are always raw value types and support SubSpace operations on cells. type Table struct { //types:add // Columns has the list of column tensor data for this table. // Different tables can provide different indexed views onto the same Columns. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index f8b94ffb97..0d83848d07 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -123,7 +123,7 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 0.0, af[1]) } -func TestSlice(t *testing.T) { +func TestSliced(t *testing.T) { ft := NewFloat64(3, 4) for y := range 3 { for x := range 4 { @@ -145,7 +145,7 @@ func TestSlice(t *testing.T) { [1]: 13 12 ` - sl := NewSliced(ft, []int{2, 1}, []int{3, 2}) + sl := NewSlicedIndexes(ft, []int{2, 1}, []int{3, 2}) // fmt.Println(sl) assert.Equal(t, res, sl.String()) From a876b8c24a2e10b9a68aa0097765de432294af60 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 13:39:19 -0700 Subject: [PATCH 087/311] tensor: everything building and passing tests; sliced working in pca / covar matrix --- tensor/databrowser/filetree.go | 2 +- tensor/rows.go | 2 +- tensor/stats/metric/matrix.go | 14 +++++--------- tensor/stats/metric/metric_test.go | 7 +++---- 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index 1a52968f4f..a1c0455767 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -223,7 +223,7 @@ func (fn *FileNode) PlotFile() { tsr := d.Data dt = table.NewTable(df) dt.Columns.Rows = tsr.DimSize(0) - if ix, ok := tsr.(*tensor.Indexed); ok { + if ix, ok := tsr.(*tensor.Rows); ok { dt.Indexes = ix.Indexes } rc := dt.AddIntColumn("Row") diff --git a/tensor/rows.go b/tensor/rows.go index 128fabfe82..c68817e25e 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -41,7 +41,7 @@ type Rows struct { //types:add // NewRows returns a new [Rows] view of given tensor, // with optional list of indexes (none / nil = sequential). func NewRows(tsr Tensor, idxs ...int) *Rows { - ix := &Rows{Tensor: tsr, Indexes: idxs} + ix := &Rows{Tensor: tsr, Indexes: slices.Clone(idxs)} return ix } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index b1a41db441..58b35c4769 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -108,14 +108,12 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { return } - flatsz := []int{in.DimSize(0), cells} flatvw := in.View() - flatvw.SetShapeInts(flatsz...) + flatvw.SetShapeInts(in.DimSize(0), cells) mout := tensor.NewFloat64Scalar(0.0) out.SetShapeInts(cells, cells) - av := tensor.NewFloat64(rows) - bv := tensor.NewFloat64(rows) + var av, bv tensor.Tensor curCoords := vecint.Vector2i{-1, -1} coords := TriangularLIndicies(cells) @@ -125,11 +123,11 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { func(idx int, tsr ...tensor.Tensor) { c := coords[idx] if c.X != curCoords.X { - tensor.Slice(tsr[0], av, tensor.Range{}, tensor.Range{Start: c.X, End: c.X + 1}) + av = tensor.NewSliced(tsr[0], tensor.Slice{}, tensor.Slice{Start: c.X, Stop: c.X + 1}) curCoords.X = c.X } if c.Y != curCoords.Y { - tensor.Slice(tsr[0], bv, tensor.Range{}, tensor.Range{Start: c.Y, End: c.Y + 1}) + bv = tensor.NewSliced(tsr[0], tensor.Slice{}, tensor.Slice{Start: c.Y, Stop: c.Y + 1}) curCoords.Y = c.Y } tensor.Call(funcName, av, bv, mout) @@ -208,9 +206,7 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) { // This is typically done with results from SVD or PCA. func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) { ci := int(colindex.Float1D(0)) - _ = ci - col := tensor.NewFloat64() - // tensor.Slice(mtx, col, tensor.Range{}, tensor.Range{Start: ci, End: ci + 1}) + col := tensor.NewSliced(mtx, tensor.Slice{}, tensor.Slice{Start: ci, Stop: ci + 1}) // fmt.Println(mtx.String(), col.String()) rows, cells := vec.RowCellSize() mout := tensor.NewFloat64() diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 3c50e8cad5..c2f5a7310a 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -5,7 +5,6 @@ package metric import ( - "fmt" "math" "testing" @@ -110,14 +109,14 @@ func TestPCAIris(t *testing.T) { data := dt.Column("data") covar := tensor.NewFloat64() CovarianceMatrix(Correlation.FuncName(), data, covar) - fmt.Printf("correl: %s\n", covar.String()) + // fmt.Printf("correl: %s\n", covar.String()) vecs := tensor.NewFloat64() vals := tensor.NewFloat64() PCA(covar, vecs, vals) - fmt.Printf("correl vec: %v\n", vecs) - fmt.Printf("correl val: %v\n", vals) + // fmt.Printf("correl vec: %v\n", vecs) + // fmt.Printf("correl val: %v\n", vals) errtol := 1.0e-9 corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} for i, v := range vals.Values { From 5013132274400546a7015f6e1a84bed154ba6f4c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 17:34:00 -0700 Subject: [PATCH 088/311] tensor: major reorganization into separate interfaces, with basic Tensor providing minimal read-only access, RowCells providing row major memory layout and RowCells interface. need to figure out broadcasting and use for computational routines instead of relying on rowcells for that. rowcells is more for editing and plotting etc. --- base/reflectx/values.go | 12 + tensor/README.md | 4 +- tensor/base.go | 70 +--- tensor/{bits.go => bool.go} | 62 +-- tensor/convert.go | 55 ++- tensor/datafs/copy.go | 6 +- tensor/datafs/data.go | 22 +- tensor/datafs/dir.go | 2 +- tensor/datafs/fs.go | 2 +- tensor/matrix/matrix.go | 76 ++++ tensor/number.go | 124 +++--- tensor/rowcell.go | 116 ++++++ tensor/rows.go | 642 ++++++++++++-------------------- tensor/sliced.go | 476 ++++++----------------- tensor/slices.go | 25 +- tensor/stats/stats/funcs.go | 2 +- tensor/stats/stats/group.go | 2 +- tensor/stats/stats/quantiles.go | 2 +- tensor/string.go | 67 +--- tensor/table/columns.go | 12 +- tensor/table/table.go | 8 +- tensor/tensor.go | 264 ++----------- tensor/values.go | 135 +++++++ 23 files changed, 884 insertions(+), 1302 deletions(-) rename tensor/{bits.go => bool.go} (84%) create mode 100644 tensor/rowcell.go create mode 100644 tensor/values.go diff --git a/base/reflectx/values.go b/base/reflectx/values.go index a3d1e75b34..582980eff4 100644 --- a/base/reflectx/values.go +++ b/base/reflectx/values.go @@ -48,6 +48,18 @@ func KindIsNumber(vk reflect.Kind) bool { return vk >= reflect.Int && vk <= reflect.Complex128 } +// KindIsInt returns whether the given [reflect.Kind] is an int +// type such as int, int32 etc. +func KindIsInt(vk reflect.Kind) bool { + return vk >= reflect.Int && vk <= reflect.Uintptr +} + +// KindIsFloat returns whether the given [reflect.Kind] is a +// float32 or float64. +func KindIsFloat(vk reflect.Kind) bool { + return vk >= reflect.Float32 && vk <= reflect.Float64 +} + // ToBool robustly converts to a bool any basic elemental type // (including pointers to such) using a big type switch organized // for greatest efficiency. It tries the [bools.Booler] diff --git a/tensor/README.md b/tensor/README.md index 8ff75f3b33..2b47f8c047 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -136,11 +136,11 @@ Although powerful, the semantics of all of this is a bit confusing. In the `tens * Basic indexing by individual integer index coordinate values is supported by the basic `Number`, `String`, `Bool` value `Tensor`s. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) view then completes the basic indexing with arbitrary reordering and filtering along entire dimension values. -* The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `CloneValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `CloneIndexes` method returns a 1D list of indexes of the `true` values, which can be used for the `Indexed` view. +* The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `AsValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `CloneIndexes` method returns a 1D list of indexes of the `true` values, which can be used for the `Indexed` view. * The `Indexed` view corresponds to the array-based advanced indexing case in NumPy, but again it is a view, not a copy, so the assignment semantics are as expected from a view (and how NumPy behaves some of the time). Note that the NumPy version uses `n` separate index tensors, where each such tensor specifies the value of a corresponding dimension index, and all such tensors _must have the same shape_; that form can be converted into the single Indexes form with a utility function. Also, NumPy advanced indexing has a somewhat confusing property where it de-duplicates index references during some operations, such that `+=1` only increments +1 even when there are multiple elements in the view. The tensor version does not implement that special case, due to its direct view semantics. -To reiterate, all view tensors have a `CloneValues` function, equivalent to the `copy` function in NumPy, which turns the view into a corresponding basic concrete value Tensor, so the copy semantics of advanced indexing (modulo the direct assignment behavior) can be achieved when assigning to a new variable. +To reiterate, all view tensors have a `AsValues` function, equivalent to the `copy` function in NumPy, which turns the view into a corresponding basic concrete value Tensor, so the copy semantics of advanced indexing (modulo the direct assignment behavior) can be achieved when assigning to a new variable. ## Broadcasting diff --git a/tensor/base.go b/tensor/base.go index 476b1d8cb8..081f36b396 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -6,8 +6,8 @@ package tensor import ( "fmt" - "log" "reflect" + "slices" "strings" "unsafe" @@ -34,7 +34,7 @@ func (tsr *Base[T]) ShapeSizes() Tensor { return tsr.shape.AsTensor() } // ShapeInts returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. -func (tsr *Base[T]) ShapeInts() []int { return tsr.shape.Sizes } +func (tsr *Base[T]) ShapeInts() []int { return slices.Clone(tsr.shape.Sizes) } // SetShape sets the dimension sizes as 1D int values from given tensor. // The backing storage is resized appropriately, retaining all existing data that fits. @@ -121,7 +121,7 @@ func (tsr *Base[T]) SetNumRows(rows int) { // The new tensor points to the values of the this tensor (i.e., modifications // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). -// Use Clone() method to separate the two. +// Use AsValues() method to separate the two. func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { nd := tsr.NumDims() od := len(offs) @@ -153,53 +153,7 @@ func (tsr *Base[T]) StringRowCell(row, cell int) string { // Label satisfies the core.Labeler interface for a summary description of the tensor. func (tsr *Base[T]) Label() string { - nm := tsr.Meta.Name() - if nm != "" { - nm += " " + tsr.shape.String() - } else { - nm = tsr.shape.String() - } - return fmt.Sprintf("Tensor: %s", nm) -} - -// Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the -// 2D Matrix. Assumes Row-major ordering and logs an error if NumDims < 2. -func (tsr *Base[T]) Dims() (r, c int) { - nd := tsr.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0, 0 - } - return tsr.shape.DimSize(nd - 2), tsr.shape.DimSize(nd - 1) -} - -// Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric -// 2D Matrix. -func (tsr *Base[T]) Symmetric() (r int) { - nd := tsr.NumDims() - if nd < 2 { - log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") - return 0 - } - if tsr.shape.DimSize(nd-2) != tsr.shape.DimSize(nd-1) { - log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") - return 0 - } - return tsr.shape.DimSize(nd - 1) -} - -// SymmetricDim returns the number of rows/columns in the matrix. -func (tsr *Base[T]) SymmetricDim() int { - nd := tsr.NumDims() - if nd < 2 { - log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") - return 0 - } - if tsr.shape.DimSize(nd-2) != tsr.shape.DimSize(nd-1) { - log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") - return 0 - } - return tsr.shape.DimSize(nd - 1) + return label(tsr.Meta.Name(), &tsr.shape) } // Sprint returns a string representation of the given tensor, @@ -209,6 +163,15 @@ func Sprint(tsr Tensor, maxLen int) string { return sprint(tsr, maxLen) } +func label(nm string, sh *Shape) string { + if nm != "" { + nm += " " + sh.String() + } else { + nm = sh.String() + } + return nm +} + // sprint is the underlying impl of String func sprint(tsr Tensor, maxLen int) string { if maxLen == 0 { @@ -216,7 +179,12 @@ func sprint(tsr Tensor, maxLen int) string { } var b strings.Builder sh := tsr.Shape() - b.WriteString(sh.String() + "\n") + b.WriteString(tsr.Label()) + if tsr.NumDims() == 1 && tsr.Len() < 8 { + b.WriteString(" ") + } else { + b.WriteString("\n") + } oddRow := false rows, cols, _, _ := Projection2DShape(sh, oddRow) ctr := 0 diff --git a/tensor/bits.go b/tensor/bool.go similarity index 84% rename from tensor/bits.go rename to tensor/bool.go index 58222b3ada..c173c03520 100644 --- a/tensor/bits.go +++ b/tensor/bool.go @@ -6,7 +6,6 @@ package tensor import ( "fmt" - "log/slog" "reflect" "cogentcore.org/core/base/metadata" @@ -14,7 +13,6 @@ import ( "cogentcore.org/core/base/reflectx" "cogentcore.org/core/base/slicesx" "cogentcore.org/core/tensor/bitslice" - "gonum.org/v1/gonum/mat" ) // Bits is a tensor of bits backed by a [bitslice.Slice] for efficient storage @@ -73,6 +71,8 @@ func (tsr *Bits) IsString() bool { return false } +func (tsr *Bits) AsValues() Values { return tsr } + // DataType returns the type of the data elements in the tensor. // Bool is returned for the Bits tensor type. func (tsr *Bits) DataType() reflect.Kind { @@ -116,20 +116,6 @@ func (tsr *Bits) RowCellSize() (rows, cells int) { return tsr.shape.RowCellSize() } -// Value returns value at given tensor index -func (tsr *Bits) Value(i ...int) bool { - return tsr.Values.Index(tsr.shape.IndexTo1D(i...)) -} - -// Value1D returns value at given tensor 1D (flat) index -func (tsr *Bits) Value1D(i int) bool { return tsr.Values.Index(i) } - -func (tsr *Bits) Set(val bool, i ...int) { - tsr.Values.Set(val, tsr.shape.IndexTo1D(i...)) -} - -func (tsr *Bits) Set1D(val bool, i int) { tsr.Values.Set(val, i) } - func (tsr *Bits) SetShape(sizes Tensor) { tsr.shape.SetShape(sizes) nln := tsr.Len() @@ -154,17 +140,17 @@ func (tsr *Bits) SetNumRows(rows int) { } // SubSpace is not possible with Bits. -func (tsr *Bits) SubSpace(offs ...int) Tensor { +func (tsr *Bits) SubSpace(offs ...int) Values { return nil } // RowTensor not possible with Bits. -func (tsr *Bits) RowTensor(row int) Tensor { +func (tsr *Bits) RowTensor(row int) Values { return nil } // SetRowTensor not possible with Bits. -func (tsr *Bits) SetRowTensor(val Tensor, row int) { +func (tsr *Bits) SetRowTensor(val Values, row int) { } @@ -287,13 +273,6 @@ func (tsr *Bits) Label() string { return fmt.Sprintf("tensor.Bits: %s", tsr.shape.String()) } -// Range is not applicable to Bits tensor -func (tsr *Bits) Range() (min, max float64, minIndex, maxIndex int) { - minIndex = -1 - maxIndex = -1 - return -} - // SetZeros is simple convenience function initialize all values to 0 func (tsr *Bits) SetZeros() { ln := tsr.Len() @@ -305,13 +284,13 @@ func (tsr *Bits) SetZeros() { // Clone clones this tensor, creating a duplicate copy of itself with its // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). -func (tsr *Bits) Clone() Tensor { +func (tsr *Bits) Clone() Values { csr := NewBitsShape(&tsr.shape) csr.Values = tsr.Values.Clone() return csr } -func (tsr *Bits) View() Tensor { +func (tsr *Bits) View() Values { nw := &Bits{} nw.shape.CopyShape(&tsr.shape) nw.Values = tsr.Values @@ -322,7 +301,7 @@ func (tsr *Bits) View() Tensor { // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. -func (tsr *Bits) CopyFrom(frm Tensor) { +func (tsr *Bits) CopyFrom(frm Values) { if fsm, ok := frm.(*Bits); ok { copy(tsr.Values, fsm.Values) return @@ -338,7 +317,7 @@ func (tsr *Bits) CopyFrom(frm Tensor) { // It uses and optimized implementation if the other tensor // is of the same type, and otherwise it goes through // appropriate standard type. -func (tsr *Bits) AppendFrom(frm Tensor) error { +func (tsr *Bits) AppendFrom(frm Values) error { rows, cell := tsr.RowCellSize() frows, fcell := frm.RowCellSize() if cell != fcell { @@ -362,7 +341,7 @@ func (tsr *Bits) AppendFrom(frm Tensor) error { // start = starting index on from Tensor to start copying from, and n = number of // values to copy. Uses an optimized implementation if the other tensor is // of the same type, and otherwise it goes through appropriate standard type. -func (tsr *Bits) CopyCellsFrom(frm Tensor, to, start, n int) { +func (tsr *Bits) CopyCellsFrom(frm Values, to, start, n int) { if fsm, ok := frm.(*Bits); ok { for i := 0; i < n; i++ { tsr.Values.Set(fsm.Values.Index(start+i), to+i) @@ -373,24 +352,3 @@ func (tsr *Bits) CopyCellsFrom(frm Tensor, to, start, n int) { tsr.Values.Set(Float64ToBool(frm.Float1D(start+i)), to+i) } } - -// Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the -// 2D Matrix. Not supported for Bits -- do not call! -func (tsr *Bits) Dims() (r, c int) { - slog.Error("tensor Dims gonum Matrix call made on Bits Tensor; not supported") - return 0, 0 -} - -// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given -// row, column index. Not supported for Bits -- do not call! -func (tsr *Bits) At(i, j int) float64 { - slog.Error("tensor At gonum Matrix call made on Bits Tensor; not supported") - return 0 -} - -// T is the gonum/mat.Matrix transpose method. -// Not supported for Bits -- do not call! -func (tsr *Bits) T() mat.Matrix { - slog.Error("tensor T gonum Matrix call made on Bits Tensor; not supported") - return mat.Transpose{tsr} -} diff --git a/tensor/convert.go b/tensor/convert.go index a508e4d905..d397510f15 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -4,20 +4,45 @@ package tensor +import "cogentcore.org/core/base/errors" + +// Clone returns a copy of the given tensor. +// If it is raw [Values] then a [Values.Clone] is returned. +// Otherwise if it is a view, then [Tensor.CloneValues] is returned. +func Clone(tsr Tensor) Values { + if vl, ok := tsr.(Values); ok { + return vl.Clone() + } + return tsr.AsValues() +} + +// SetShapeFrom sets shape of given tensor from a source tensor. +// This will return an error if the destination tensor is not a Values type. +// This is used extensively for output tensors in functions, and all such +// output tensors _must_ be Values tensors. +func SetShapeFrom(tsr, from Tensor) error { + vl, ok := tsr.(Values) + if !ok { + return errors.Log(errors.New("tensor.SetShapeFrom: tensor must be a Values type to have shape modified. All function output tensors must be Values!")) + } + vl.SetShapeInts(from.ShapeInts()...) + return nil +} + // New1DViewOf returns a 1D view into the given tensor, using the same // underlying values, and just changing the shape to a 1D view. // This can be useful e.g., for stats and metric functions that report // on the 1D list of values. -func New1DViewOf(tsr Tensor) Tensor { +func New1DViewOf(tsr Values) Values { vw := tsr.View() vw.SetShapeInts(tsr.Len()) return vw } -// Cells1D returns a flat 1D [Tensor] view of the cells for given row -// index. This is useful for passing to other functions e.g., +// Cells1D returns a flat 1D [Values] view of the cells for given row index. +// This is useful for passing to other functions e.g., // in stats or metrics that process a 1D tensor. -func Cells1D(tsr Tensor, row int) Tensor { +func Cells1D(tsr RowCell, row int) Values { return New1DViewOf(tsr.SubSpace(row)) } @@ -26,7 +51,7 @@ func Cells1D(tsr Tensor, row int) Tensor { // split are collapsed into the row dimension, and from split onward // form the cells dimension. The resulting tensor is a re-shaped view // of the original tensor, sharing the same underlying data. -func RowCellSplit(tsr Tensor, split int) Tensor { +func RowCellSplit(tsr Values, split int) Values { sizes := tsr.ShapeInts() rows := sizes[:split] cells := sizes[split:] @@ -45,40 +70,40 @@ func RowCellSplit(tsr Tensor, split int) Tensor { // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. -func NewFloat64Scalar(val float64) Tensor { +func NewFloat64Scalar(val float64) *Float64 { return NewNumberFromSlice(val) } // NewIntScalar is a convenience method for a Tensor // representation of a single int scalar value. -func NewIntScalar(val int) Tensor { +func NewIntScalar(val int) *Int { return NewNumberFromSlice(val) } // NewStringScalar is a convenience method for a Tensor // representation of a single string scalar value. -func NewStringScalar(val string) Tensor { +func NewStringScalar(val string) *String { return NewStringFromSlice(val) } // NewFloat64FromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewFloat64FromSlice(vals ...float64) Tensor { +func NewFloat64FromSlice(vals ...float64) *Float64 { return NewNumberFromSlice(vals...) } // NewIntFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewIntFromSlice(vals ...int) Tensor { +func NewIntFromSlice(vals ...int) *Int { return NewNumberFromSlice(vals...) } // NewStringFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewStringFromSlice(vals ...string) Tensor { +func NewStringFromSlice(vals ...string) *String { n := len(vals) tsr := &String{} tsr.Values = vals @@ -168,7 +193,7 @@ func AsFloat64Tensor(tsr Tensor) *Float64 { return f } f := NewFloat64(tsr.ShapeInts()...) - f.CopyFrom(tsr) + f.CopyFrom(tsr.AsValues()) return f } @@ -180,7 +205,7 @@ func AsFloat32Tensor(tsr Tensor) *Float32 { return f } f := NewFloat32(AsIntSlice(tsr.ShapeSizes())...) - f.CopyFrom(tsr) + f.CopyFrom(tsr.AsValues()) return f } @@ -194,7 +219,7 @@ func AsStringTensor(tsr Tensor) *String { return f } f := NewString(tsr.ShapeInts()...) - f.CopyFrom(tsr) + f.CopyFrom(tsr.AsValues()) return f } @@ -208,6 +233,6 @@ func AsIntTensor(tsr Tensor) *Int { return f } f := NewInt(tsr.ShapeInts()...) - f.CopyFrom(tsr) + f.CopyFrom(tsr.AsValues()) return f } diff --git a/tensor/datafs/copy.go b/tensor/datafs/copy.go index 9da7e0404c..6f1d7d8acb 100644 --- a/tensor/datafs/copy.go +++ b/tensor/datafs/copy.go @@ -8,6 +8,8 @@ import ( "errors" "io/fs" "time" + + "cogentcore.org/core/tensor" ) const ( @@ -21,7 +23,7 @@ const ( // CopyFromValue copies value from given source data node, cloning it. func (d *Data) CopyFromValue(frd *Data) { d.modTime = time.Now() - d.Data = frd.Data.Clone() + d.Data = tensor.Clone(frd.Data) } // Clone returns a copy of this data item, recursively cloning directory items @@ -29,7 +31,7 @@ func (d *Data) CopyFromValue(frd *Data) { func (d *Data) Clone() *Data { if !d.IsDir() { cp, _ := newData(nil, d.name) - cp.Data = d.Data.Clone() + cp.Data = tensor.Clone(d.Data) return cp } items := d.ItemsFunc(nil) diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 7d1c48bf24..87a2d52538 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -81,7 +81,7 @@ func NewScalar[T tensor.DataTypes](dir *Data, names ...string) tensor.Tensor { // NewValue returns a new Data value as a [tensor.Tensor] // of given data type and shape sizes, in given directory Data item. // The name must be unique in the directory. -func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Tensor { +func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Values { tsr := tensor.New[T](sizes...) tsr.Metadata().SetName(name) d, err := newData(dir, name) @@ -89,14 +89,14 @@ func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.T return nil } d.Data = tsr - return d.Data + return tsr } // NewOfType returns a new Data value as a [tensor.Tensor] // of given reflect.Kind type and shape sizes per dimension, in given directory Data item. // Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. // The name must be unique in the directory. -func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Tensor { +func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Values { tsr := tensor.NewOfType(typ, sizes...) tsr.Metadata().SetName(name) nd, err := newData(d, name) @@ -104,7 +104,7 @@ func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Ten return nil } nd.Data = tsr - return nd.Data + return tsr } func (d *Data) KnownFileInfo() fileinfo.Known { @@ -129,7 +129,7 @@ func (d *Data) Bytes() []byte { if d.Data == nil { return nil } - return d.Data.Bytes() + return d.Data.AsValues().Bytes() } // AsString returns data as scalar string. @@ -137,7 +137,7 @@ func (d *Data) AsString() string { if d.Data == nil { return "" } - return d.Data.StringRow(0) + return d.Data.String1D(0) } // SetString sets scalar data value from given string. @@ -145,7 +145,7 @@ func (d *Data) SetString(v string) { if d.Data == nil { return } - d.Data.SetStringRow(v, 0) + d.Data.SetString1D(v, 0) } // AsFloat64 returns data as a scalar float64 (first element of tensor). @@ -153,7 +153,7 @@ func (d *Data) AsFloat64() float64 { if d.Data == nil { return 0 } - return d.Data.FloatRow(0) + return d.Data.Float1D(0) } // SetFloat64 sets scalar data value from given float64. @@ -161,7 +161,7 @@ func (d *Data) SetFloat64(v float64) { if d.Data == nil { return } - d.Data.SetFloatRow(v, 0) + d.Data.SetFloat1D(v, 0) } // AsFloat32 returns data as a scalar float32 (first element of tensor). @@ -179,7 +179,7 @@ func (d *Data) AsInt() int { if d.Data == nil { return 0 } - return d.Data.IntRow(0) + return d.Data.Int1D(0) } // SetInt sets scalar data value from given int. @@ -187,5 +187,5 @@ func (d *Data) SetInt(v int) { if d.Data == nil { return } - d.Data.SetIntRow(v, 0) + d.Data.SetInt1D(v, 0) } diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 73eae7b8cd..6bcac3fc58 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -382,7 +382,7 @@ func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { if it.Parent != d { nm = fsx.DirAndFile(string(it.Path())) } - dt.AddColumn(nm, tsr) + dt.AddColumn(nm, tsr.AsValues()) } d.DirTable = dt return dt diff --git a/tensor/datafs/fs.go b/tensor/datafs/fs.go index 75ffff2744..d58dd11758 100644 --- a/tensor/datafs/fs.go +++ b/tensor/datafs/fs.go @@ -115,7 +115,7 @@ func (d *Data) Size() int64 { if d.Data == nil { return 0 } - return d.Data.Sizeof() + return d.Data.AsValues().Sizeof() } func (d *Data) IsDir() bool { diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go index 0937112696..799b4c5caf 100644 --- a/tensor/matrix/matrix.go +++ b/tensor/matrix/matrix.go @@ -5,10 +5,86 @@ package matrix import ( + "errors" + "cogentcore.org/core/tensor" "gonum.org/v1/gonum/mat" ) +// Matrix provides a view of the given [tensor.Tensor] as a [gonum] +// [mat.Matrix] interface type. +type Matrix struct { + Tensor tensor.Tensor +} + +// NewMatrix returns given [tensor.Tensor] as a [gonum] [mat.Matrix]. +// It returns an error if the tensor is not 2D. +func NewMatrix(tsr tensor.Tensor) (*Matrix, error) { + if tsr.IsString() { + err := errors.New("matrix.NewMatrix: tensor has string values; must be numeric") + return nil, err + } + nd := tsr.NumDims() + if nd != 2 { + err := errors.New("matrix.NewMatrix: tensor is not 2D") + return nil, err + } + return &Matrix{Tensor: tsr} +} + +// Dims is the gonum/mat.Matrix interface method for returning the +// dimension sizes of the 2D Matrix. Assumes Row-major ordering. +func (mx *Matrix) Dims() (r, c int) { + return tsr.shape.DimSize(0), tsr.shape.DimSize(1) +} + +// At is the gonum/mat.Matrix interface method for returning 2D +// matrix element at given row, column index. Assumes Row-major ordering. +func (mx *Matrix) At(i, j int) float64 { + return mx.Tensor.Float(i, j) +} + +// T is the gonum/mat.Matrix transpose method. +// It performs an implicit transpose by returning the receiver inside a Transpose. +func (mx *Matrix) T() mat.Matrix { + return mat.Transpose{tsr} +} + +///////////////////////// Symmetric + +// Symmetric provides a view of the given [tensor.Tensor] as a [gonum] +// [mat.Symmetric] matrix interface type. +type Symmetric struct { + Matrix +} + +// NewSymmetric returns given [tensor.Tensor] as a [gonum] [mat.Symmetric] matrix. +// It returns an error if the tensor is not 2D or not symmetric. +func NewSymmetric(tsr tensor.Tensor) (*Symmetric, error) { + if tsr.IsString() { + err := errors.New("matrix.NewSymmetric: tensor has string values; must be numeric") + return nil, err + } + nd := tsr.NumDims() + if nd != 2 { + err := errors.New("matrix.NewSymmetric: tensor is not 2D") + return nil, err + } + if tsr.DimSize(0) != tsr.DimSize(1) { + err := errors.New("matrix.NewSymmetric: tensor is not symmetric") + return nil, err + } + sy := &Symmetric{} + sy.Tensor = tsr + return sy +} + +// SymmetricDim is the gonum/mat.Matrix interface method for returning the +// dimensionality of a symmetric 2D Matrix. +func (sy *Symmetric) SymmetricDim() (r int) { + return sy.Tensor.DimSize(0) +} + // CopyDense copies a gonum mat.Dense matrix into given Tensor // using standard Float64 interface func CopyDense(to tensor.Tensor, dm *mat.Dense) { diff --git a/tensor/number.go b/tensor/number.go index f5f7a41103..12b9892afa 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -6,12 +6,11 @@ package tensor import ( "fmt" - "log" "math" "strconv" "cogentcore.org/core/base/num" - "gonum.org/v1/gonum/mat" + "cogentcore.org/core/base/reflectx" ) // Number is a tensor of numerical values @@ -88,7 +87,7 @@ func NewNumberShape[T num.Number](shape *Shape) *Number[T] { // NewNumberFromSlice returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewNumberFromSlice[T num.Number](vals ...T) Tensor { +func NewNumberFromSlice[T num.Number](vals ...T) *Number[T] { n := len(vals) tsr := &Number[T]{} tsr.Values = vals @@ -97,13 +96,11 @@ func NewNumberFromSlice[T num.Number](vals ...T) Tensor { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Number[T]) String() string { - return sprint(tsr, 0) -} +func (tsr *Number[T]) String() string { return sprint(tsr, 0) } -func (tsr *Number[T]) IsString() bool { - return false -} +func (tsr *Number[T]) IsString() bool { return false } + +func (tsr *Number[T]) AsValues() Values { return tsr } ///////////////////// Strings @@ -233,52 +230,6 @@ func (tsr *Number[T]) SetIntRow(val int, row int) { tsr.SetIntRowCell(val, row, 0) } -// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given -// row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. -func (tsr *Number[T]) At(i, j int) float64 { - nd := tsr.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0 - } else if nd == 2 { - return tsr.Float(i, j) - } else { - ix := make([]int, nd) - ix[nd-2] = i - ix[nd-1] = j - return tsr.Float(ix...) - } -} - -// T is the gonum/mat.Matrix transpose method. -// It performs an implicit transpose by returning the receiver inside a Transpose. -func (tsr *Number[T]) T() mat.Matrix { - return mat.Transpose{tsr} -} - -// Range returns the min, max (and associated indexes, -1 = no values) for the tensor. -// This is needed for display and is thus in the core api in optimized form -// Other math operations can be done using gonum/floats package. -func (tsr *Number[T]) Range() (min, max float64, minIndex, maxIndex int) { - minIndex = -1 - maxIndex = -1 - for j, vl := range tsr.Values { - fv := float64(vl) - if math.IsNaN(fv) { - continue - } - if fv < min || minIndex < 0 { - min = fv - minIndex = j - } - if fv > max || maxIndex < 0 { - max = fv - maxIndex = j - } - } - return -} - // SetZeros is simple convenience function initialize all values to 0 func (tsr *Number[T]) SetZeros() { for j := range tsr.Values { @@ -286,30 +237,35 @@ func (tsr *Number[T]) SetZeros() { } } +func (tsr *Number[T]) View() Values { + return &Number[T]{*tsr.view()} +} + // Clone clones this tensor, creating a duplicate copy of itself with its -// own separate memory representation of all the values, and returns -// that as a Tensor (which can be converted into the known type as needed). -func (tsr *Number[T]) Clone() Tensor { +// own separate memory representation of all the values. +func (tsr *Number[T]) Clone() Values { csr := NewNumberShape[T](&tsr.shape) copy(csr.Values, tsr.Values) return csr } -func (tsr *Number[T]) View() Tensor { - return &Number[T]{*tsr.view()} -} - // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. -func (tsr *Number[T]) CopyFrom(frm Tensor) { +func (tsr *Number[T]) CopyFrom(frm Values) { if fsm, ok := frm.(*Number[T]); ok { copy(tsr.Values, fsm.Values) return } sz := min(len(tsr.Values), frm.Len()) - for i := 0; i < sz; i++ { - tsr.Values[i] = T(frm.Float1D(i)) + if reflectx.KindIsInt(tsr.DataType()) { + for i := range sz { + tsr.Values[i] = T(frm.Int1D(i)) + } + } else { + for i := range sz { + tsr.Values[i] = T(frm.Float1D(i)) + } } } @@ -318,7 +274,7 @@ func (tsr *Number[T]) CopyFrom(frm Tensor) { // It uses and optimized implementation if the other tensor // is of the same type, and otherwise it goes through // appropriate standard type. -func (tsr *Number[T]) AppendFrom(frm Tensor) error { +func (tsr *Number[T]) AppendFrom(frm Values) error { rows, cell := tsr.RowCellSize() frows, fcell := frm.RowCellSize() if cell != fcell { @@ -342,11 +298,9 @@ func (tsr *Number[T]) AppendFrom(frm Tensor) error { // start = starting index on from Tensor to start copying from, and n = number of // values to copy. Uses an optimized implementation if the other tensor is // of the same type, and otherwise it goes through appropriate standard type. -func (tsr *Number[T]) CopyCellsFrom(frm Tensor, to, start, n int) { +func (tsr *Number[T]) CopyCellsFrom(frm Values, to, start, n int) { if fsm, ok := frm.(*Number[T]); ok { - for i := range n { - tsr.Values[to+i] = fsm.Values[start+i] - } + copy(tsr.Values[to:to+n], fsm.Values[start:start+n]) return } for i := range n { @@ -359,8 +313,8 @@ func (tsr *Number[T]) CopyCellsFrom(frm Tensor, to, start, n int) { // The new tensor points to the values of the this tensor (i.e., modifications // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). -// Use Clone() method to separate the two. -func (tsr *Number[T]) SubSpace(offs ...int) Tensor { +// Use AsValues() method to separate the two. +func (tsr *Number[T]) SubSpace(offs ...int) Values { b := tsr.subSpaceImpl(offs...) rt := &Number[T]{Base: *b} return rt @@ -369,14 +323,36 @@ func (tsr *Number[T]) SubSpace(offs ...int) Tensor { // RowTensor is a convenience version of [Tensor.SubSpace] to return the // SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. -func (tsr *Number[T]) RowTensor(row int) Tensor { +func (tsr *Number[T]) RowTensor(row int) Values { return tsr.SubSpace(row) } // SetRowTensor sets the values of the SubSpace at given row to given values. -func (tsr *Number[T]) SetRowTensor(val Tensor, row int) { +func (tsr *Number[T]) SetRowTensor(val Values, row int) { _, cells := tsr.RowCellSize() st := row * cells mx := min(val.Len(), cells) tsr.CopyCellsFrom(val, st, 0, mx) } + +// Range returns the min, max (and associated indexes, -1 = no values) for the tensor. +// This is needed for display and is thus in the core api in optimized form +func (tsr *Number[T]) Range() (min, max float64, minIndex, maxIndex int) { + minIndex = -1 + maxIndex = -1 + for j, vl := range tsr.Values { + fv := float64(vl) + if math.IsNaN(fv) { + continue + } + if fv < min || minIndex < 0 { + min = fv + minIndex = j + } + if fv > max || maxIndex < 0 { + max = fv + maxIndex = j + } + } + return +} diff --git a/tensor/rowcell.go b/tensor/rowcell.go new file mode 100644 index 0000000000..7ce89c7c48 --- /dev/null +++ b/tensor/rowcell.go @@ -0,0 +1,116 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +// RowCell is subclass of [Tensor] that supports efficient access via the +// outermost 'row' dimension, with all remaining inner dimensions comprising the +// 'cells' of data per row (1 scalar value in the case of a 1D tensor). +// It is implemented by raw [Values] tensors, and the [Rows] indexed view of +// raw Values tensors. Other views however do not retain the underlying +// outer to inner memory structure and thus do not benefit from this interface. +type RowCell interface { + Tensor + + // RowCellSize returns the size of the outermost Row shape dimension, + // and the size of all the remaining inner dimensions (the "cell" size). + // Commonly used to organize multiple instances (rows) of higher-dimensional + // patterns (cells), and the [Rows] type operates on the outer row dimension. + RowCellSize() (rows, cells int) + + // SubSpace returns a new tensor with innermost subspace at given + // offset(s) in outermost dimension(s) (len(offs) < [NumDims]). + // The new tensor points to the values of the this tensor (i.e., modifications + // will affect both), as its Values slice is a view onto the original (which + // is why only inner-most contiguous supsaces are supported). + // Use AsValues() method to separate the two. See [Slice] function to + // extract arbitrary subspaces along ranges of each dimension. + SubSpace(offs ...int) Values + + // RowTensor is a convenience version of [Tensor.SubSpace] to return the + // SubSpace for the outermost row dimension. [Rows] defines a version + // of this that indirects through the row indexes. + RowTensor(row int) Values + + // SetRowTensor sets the values of the [Tensor.SubSpace] at given row to given values. + SetRowTensor(val Values, row int) + + ///////////////////// Floats + + // FloatRowCell returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Rows] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. + FloatRowCell(row, cell int) float64 + + // SetFloatRowCell sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Rows] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. + SetFloatRowCell(val float64, row, cell int) + + // FloatRow returns the value at given row (outermost dimension). + // It is a convenience wrapper for FloatRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + FloatRow(row int) float64 + + // SetFloatRow sets the value at given row (outermost dimension). + // It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + SetFloatRow(val float64, row int) + + ///////////////////// Ints + + // IntRowCell returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Rows] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. + IntRowCell(row, cell int) int + + // SetIntRowCell sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Rows] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. + SetIntRowCell(val int, row, cell int) + + // IntRow returns the value at given row (outermost dimension). + // It is a convenience wrapper for IntRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + IntRow(row int) int + + // SetIntRow sets the value at given row (outermost dimension). + // It is a convenience wrapper for SetIntRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + SetIntRow(val int, row int) + + ///////////////////// Strings + + // StringRowCell returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Rows] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. + StringRowCell(row, cell int) string + + // SetStringRowCell sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + // [Rows] tensors index along the row, and use this interface extensively. + // This is useful for lists of patterns, and the [table.Table] container. + SetStringRowCell(val string, row, cell int) + + // StringRow returns the value at given row (outermost dimension). + // It is a convenience wrapper for StringRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + StringRow(row int) string + + // SetStringRow sets the value at given row (outermost dimension). + // It is a convenience wrapper for SetStringRowCell(row, 0), providing robust + // operations on 1D and higher-dimensional data (which nevertheless should + // generally be processed separately in ways that treat it properly). + SetStringRow(val string, row int) +} diff --git a/tensor/rows.go b/tensor/rows.go index c68817e25e..c7f67d8adb 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -6,7 +6,6 @@ package tensor import ( "cmp" - "log" "math" "math/rand" "reflect" @@ -15,22 +14,20 @@ import ( "strings" "cogentcore.org/core/base/metadata" - "gonum.org/v1/gonum/mat" ) -// Rows is an indexed wrapper around another [Tensor] that provides a +// Rows is a row-indexed wrapper around a [Values] [Tensor] that provides a // specific view onto the Tensor defined by the set of [Rows.Indexes], // which apply to the outermost row dimension (with default row-major indexing). // Sorting and filtering a tensor only requires updating the indexes while // leaving the underlying Tensor alone. -// To produce a new [Tensor] that has its raw data actually organized according -// to the indexed order (i.e., the copy function of numpy), call [Rows.NewTensor]. -// Use the [Set]FloatRow[Cell] methods wherever possible, for the most efficient -// and natural indirection through the indexes. +// Use [CloneValues] to obtain a concrete [Values] representation with the current row +// sorting. Use the [Set]FloatRow[Cell] methods wherever possible, +// for the most efficient and natural indirection through the indexes. type Rows struct { //types:add // Tensor that we are an indexed view onto. - Tensor Tensor + Tensor Values // Indexes are the indexes into Tensor rows, with nil = sequential. // Only set if order is different from default sequential order. @@ -40,202 +37,174 @@ type Rows struct { //types:add // NewRows returns a new [Rows] view of given tensor, // with optional list of indexes (none / nil = sequential). -func NewRows(tsr Tensor, idxs ...int) *Rows { - ix := &Rows{Tensor: tsr, Indexes: slices.Clone(idxs)} - return ix +func NewRows(tsr Values, idxs ...int) *Rows { + rw := &Rows{Tensor: tsr, Indexes: slices.Clone(idxs)} + return rw } -// AsRows returns the tensor as an [Rows[] view. -// If it already is one, then it is returned, otherwise it is wrapped. +// AsRows returns the tensor as a [Rows] view. +// If it already is one, then it is returned, otherwise +// a new Rows is created to wrap around the given tensor, which is +// enforced to be a [Values] tensor either because it already is one, +// or by calling [Tensor.AsValues] on it. func AsRows(tsr Tensor) *Rows { - if ix, ok := tsr.(*Rows); ok { - return ix + if rw, ok := tsr.(*Rows); ok { + return rw } - return NewRows(tsr) + return NewRows(tsr.AsValues()) } -// SetTensor sets as indexes into given tensor with sequential initial indexes. -func (ix *Rows) SetTensor(tsr Tensor) { - ix.Tensor = tsr - ix.Sequential() +// SetTensor sets as indexes into given [Values] tensor with sequential initial indexes. +func (rw *Rows) SetTensor(tsr Values) { + rw.Tensor = tsr + rw.Sequential() } +func (rw *Rows) IsString() bool { return rw.Tensor.IsString() } + +func (rw *Rows) DataType() reflect.Kind { return rw.Tensor.DataType() } + // RowIndex returns the actual index into underlying tensor row based on given // index value. If Indexes == nil, index is passed through. -func (ix *Rows) RowIndex(idx int) int { - if ix.Indexes == nil { +func (rw *Rows) RowIndex(idx int) int { + if rw.Indexes == nil { return idx } - return ix.Indexes[idx] + return rw.Indexes[idx] } // NumRows returns the effective number of rows in this Rows view, // which is the length of the index list or number of outer // rows dimension of tensor if no indexes (full sequential view). -func (ix *Rows) NumRows() int { - if ix.Indexes == nil { - return ix.Tensor.DimSize(0) +func (rw *Rows) NumRows() int { + if rw.Indexes == nil { + return rw.Tensor.DimSize(0) } - return len(ix.Indexes) + return len(rw.Indexes) } // String satisfies the fmt.Stringer interface for string of tensor data. -func (ix *Rows) String() string { - return sprint(ix.Tensor, 0) // todo: no need +func (rw *Rows) String() string { + return sprint(rw.Tensor, 0) // todo: no need } // Label satisfies the core.Labeler interface for a summary description of the tensor. -func (ix *Rows) Label() string { - return ix.Tensor.Label() +func (rw *Rows) Label() string { + return rw.Tensor.Label() } // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. -func (ix *Rows) Metadata() *metadata.Data { return ix.Tensor.Metadata() } +func (rw *Rows) Metadata() *metadata.Data { return rw.Tensor.Metadata() } // If we have Indexes, this is the effective shape sizes using // the current number of indexes as the outermost row dimension size. -func (ix *Rows) ShapeInts() []int { - if ix.Indexes == nil || ix.Tensor.NumDims() == 0 { - return ix.Tensor.ShapeInts() +func (rw *Rows) ShapeInts() []int { + if rw.Indexes == nil || rw.Tensor.NumDims() == 0 { + return rw.Tensor.ShapeInts() } - sh := slices.Clone(ix.Tensor.ShapeInts()) - sh[0] = len(ix.Indexes) + sh := slices.Clone(rw.Tensor.ShapeInts()) + sh[0] = len(rw.Indexes) return sh } -func (ix *Rows) ShapeSizes() Tensor { - if ix.Indexes == nil { - return ix.Tensor.ShapeSizes() +func (rw *Rows) ShapeSizes() Tensor { + if rw.Indexes == nil { + return rw.Tensor.ShapeSizes() } - return NewIntFromSlice(ix.ShapeInts()...) + return NewIntFromSlice(rw.ShapeInts()...) } // Shape() returns a [Shape] representation of the tensor shape // (dimension sizes). If we have Indexes, this is the effective // shape using the current number of indexes as the outermost row dimension size. -func (ix *Rows) Shape() *Shape { - if ix.Indexes == nil { - return ix.Tensor.Shape() - } - return NewShape(ix.ShapeInts()...) -} - -// SetShapeInts sets our shape to given sizes. -// If we do not have indexes, or the row-wise shape dimension -// in the new shape is the same as current, then we set the shape -// of the wrapped Tensor accordingly. -// This allows reshaping of inner dimensions while preserving indexes, -// e.g., for computational routines that use a 1D cell view. -// Otherwise, we reset the indexes and then set the wrapped shape, -// because our current indexes are now invalidated. -func (ix *Rows) SetShapeInts(sizes ...int) { - if ix.Indexes == nil || ix.Tensor.NumDims() == 0 { - ix.Tensor.SetShapeInts(sizes...) - return +func (rw *Rows) Shape() *Shape { + if rw.Indexes == nil { + return rw.Tensor.Shape() } - sh := ix.Tensor.ShapeInts() - if sizes[0] == sh[0] { // keep our indexes - ix.Tensor.SetShapeInts(sizes...) - return - } - ix.Indexes = nil // now invalid - ix.Tensor.SetShapeInts(sizes...) -} - -// SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. -// This invalidates the indexes. -func (ix *Rows) SetNumRows(rows int) { - ix.Sequential() - ix.Tensor.SetNumRows(rows) -} - -// SetShape sets our shape to given sizes. -// See [Rows.SetShapeInts] for details. -func (ix *Rows) SetShape(sizes Tensor) { - ix.SetShapeInts(AsIntSlice(sizes)...) + return NewShape(rw.ShapeInts()...) } // Len returns the total number of elements in the tensor, // taking into account the Indexes via [Rows], // as NumRows() * cell size. -func (ix *Rows) Len() int { - rows := ix.NumRows() - _, cells := ix.Tensor.RowCellSize() +func (rw *Rows) Len() int { + rows := rw.NumRows() + _, cells := rw.Tensor.RowCellSize() return cells * rows } // NumDims returns the total number of dimensions. -func (ix *Rows) NumDims() int { return ix.Tensor.NumDims() } +func (rw *Rows) NumDims() int { return rw.Tensor.NumDims() } // DimSize returns size of given dimension, returning NumRows() // for first dimension. -func (ix *Rows) DimSize(dim int) int { +func (rw *Rows) DimSize(dim int) int { if dim == 0 { - return ix.NumRows() + return rw.NumRows() } - return ix.Tensor.DimSize(dim) + return rw.Tensor.DimSize(dim) } // RowCellSize returns the size of the outermost Row shape dimension // (via [Rows.NumRows] method), and the size of all the remaining // inner dimensions (the "cell" size). -func (ix *Rows) RowCellSize() (rows, cells int) { - _, cells = ix.Tensor.RowCellSize() - rows = ix.NumRows() +func (rw *Rows) RowCellSize() (rows, cells int) { + _, cells = rw.Tensor.RowCellSize() + rows = rw.NumRows() return } // ValidIndexes deletes all invalid indexes from the list. // Call this if rows (could) have been deleted from tensor. -func (ix *Rows) ValidIndexes() { - if ix.Tensor.DimSize(0) <= 0 || ix.Indexes == nil { - ix.Indexes = nil +func (rw *Rows) ValidIndexes() { + if rw.Tensor.DimSize(0) <= 0 || rw.Indexes == nil { + rw.Indexes = nil return } - ni := ix.NumRows() + ni := rw.NumRows() for i := ni - 1; i >= 0; i-- { - if ix.Indexes[i] >= ix.Tensor.DimSize(0) { - ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + if rw.Indexes[i] >= rw.Tensor.DimSize(0) { + rw.Indexes = append(rw.Indexes[:i], rw.Indexes[i+1:]...) } } } // Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor. -func (ix *Rows) Sequential() { //types:add - ix.Indexes = nil +func (rw *Rows) Sequential() { //types:add + rw.Indexes = nil } // IndexesNeeded is called prior to an operation that needs actual indexes, // e.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise // current indexes are left as is. Use Sequential, then IndexesNeeded to ensure // all rows are represented. -func (ix *Rows) IndexesNeeded() { - if ix.Tensor.DimSize(0) <= 0 { - ix.Indexes = nil +func (rw *Rows) IndexesNeeded() { + if rw.Tensor.DimSize(0) <= 0 { + rw.Indexes = nil return } - if ix.Indexes != nil { + if rw.Indexes != nil { return } - ix.Indexes = make([]int, ix.Tensor.DimSize(0)) - for i := range ix.Indexes { - ix.Indexes[i] = i + rw.Indexes = make([]int, rw.Tensor.DimSize(0)) + for i := range rw.Indexes { + rw.Indexes[i] = i } } // ExcludeMissing deletes indexes where the values are missing, as indicated by NaN. // Uses first cell of higher dimensional data. -func (ix *Rows) ExcludeMissing() { //types:add - if ix.Tensor.DimSize(0) <= 0 { - ix.Indexes = nil +func (rw *Rows) ExcludeMissing() { //types:add + if rw.Tensor.DimSize(0) <= 0 { + rw.Indexes = nil return } - ix.IndexesNeeded() - ni := ix.NumRows() + rw.IndexesNeeded() + ni := rw.NumRows() for i := ni - 1; i >= 0; i-- { - if math.IsNaN(ix.Tensor.FloatRowCell(ix.Indexes[i], 0)) { - ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + if math.IsNaN(rw.Tensor.FloatRowCell(rw.Indexes[i], 0)) { + rw.Indexes = append(rw.Indexes[:i], rw.Indexes[i+1:]...) } } } @@ -243,16 +212,16 @@ func (ix *Rows) ExcludeMissing() { //types:add // Permuted sets indexes to a permuted order. If indexes already exist // then existing list of indexes is permuted, otherwise a new set of // permuted indexes are generated -func (ix *Rows) Permuted() { - if ix.Tensor.DimSize(0) <= 0 { - ix.Indexes = nil +func (rw *Rows) Permuted() { + if rw.Tensor.DimSize(0) <= 0 { + rw.Indexes = nil return } - if ix.Indexes == nil { - ix.Indexes = rand.Perm(ix.Tensor.DimSize(0)) + if rw.Indexes == nil { + rw.Indexes = rand.Perm(rw.Tensor.DimSize(0)) } else { - rand.Shuffle(len(ix.Indexes), func(i, j int) { - ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] + rand.Shuffle(len(rw.Indexes), func(i, j int) { + rw.Indexes[i], rw.Indexes[j] = rw.Indexes[j], rw.Indexes[i] }) } } @@ -276,21 +245,21 @@ const ( // as these row numbers have already been projected through the indexes. // cmp(a, b) should return a negative number when a < b, a positive // number when a > b and zero when a == b. -func (ix *Rows) SortFunc(cmp func(tsr Tensor, i, j int) int) { - ix.IndexesNeeded() - slices.SortFunc(ix.Indexes, func(a, b int) int { - return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! +func (rw *Rows) SortFunc(cmp func(tsr Values, i, j int) int) { + rw.IndexesNeeded() + slices.SortFunc(rw.Indexes, func(a, b int) int { + return cmp(rw.Tensor, a, b) // key point: these are already indirected through indexes!! }) } // SortIndexes sorts the indexes into our Tensor directly in // numerical order, producing the native ordering, while preserving // any filtering that might have occurred. -func (ix *Rows) SortIndexes() { - if ix.Indexes == nil { +func (rw *Rows) SortIndexes() { + if rw.Indexes == nil { return } - sort.Ints(ix.Indexes) + sort.Ints(rw.Indexes) } func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { @@ -302,13 +271,13 @@ func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { // Sort does default alpha or numeric sort of row-wise data. // Uses first cell of higher dimensional data. -func (ix *Rows) Sort(ascending bool) { - if ix.Tensor.IsString() { - ix.SortFunc(func(tsr Tensor, i, j int) int { +func (rw *Rows) Sort(ascending bool) { + if rw.Tensor.IsString() { + rw.SortFunc(func(tsr Values, i, j int) int { return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) }) } else { - ix.SortFunc(func(tsr Tensor, i, j int) int { + rw.SortFunc(func(tsr Values, i, j int) int { return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) }) } @@ -321,22 +290,22 @@ func (ix *Rows) Sort(ascending bool) { // number when a > b and zero when a == b. // It is *essential* that it always returns 0 when the two are equal // for the stable function to actually work. -func (ix *Rows) SortStableFunc(cmp func(tsr Tensor, i, j int) int) { - ix.IndexesNeeded() - slices.SortStableFunc(ix.Indexes, func(a, b int) int { - return cmp(ix.Tensor, a, b) // key point: these are already indirected through indexes!! +func (rw *Rows) SortStableFunc(cmp func(tsr Values, i, j int) int) { + rw.IndexesNeeded() + slices.SortStableFunc(rw.Indexes, func(a, b int) int { + return cmp(rw.Tensor, a, b) // key point: these are already indirected through indexes!! }) } // SortStable does stable default alpha or numeric sort. // Uses first cell of higher dimensional data. -func (ix *Rows) SortStable(ascending bool) { - if ix.Tensor.IsString() { - ix.SortStableFunc(func(tsr Tensor, i, j int) int { +func (rw *Rows) SortStable(ascending bool) { + if rw.Tensor.IsString() { + rw.SortStableFunc(func(tsr Values, i, j int) int { return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) }) } else { - ix.SortStableFunc(func(tsr Tensor, i, j int) int { + rw.SortStableFunc(func(tsr Values, i, j int) int { return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) }) } @@ -345,17 +314,17 @@ func (ix *Rows) SortStable(ascending bool) { // FilterFunc is a function used for filtering that returns // true if Tensor row should be included in the current filtered // view of the tensor, and false if it should be removed. -type FilterFunc func(tsr Tensor, row int) bool +type FilterFunc func(tsr Values, row int) bool // Filter filters the indexes using given Filter function. // The Filter function operates directly on row numbers into the Tensor // as these row numbers have already been projected through the indexes. -func (ix *Rows) Filter(filterer func(tsr Tensor, row int) bool) { - ix.IndexesNeeded() - sz := len(ix.Indexes) +func (rw *Rows) Filter(filterer func(tsr Values, row int) bool) { + rw.IndexesNeeded() + sz := len(rw.Indexes) for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering - if !filterer(ix.Tensor, ix.Indexes[i]) { // delete - ix.Indexes = append(ix.Indexes[:i], ix.Indexes[i+1:]...) + if !filterer(rw.Tensor, rw.Indexes[i]) { // delete + rw.Indexes = append(rw.Indexes[:i], rw.Indexes[i+1:]...) } } } @@ -382,9 +351,9 @@ const ( // Use the named const args [Include], [Exclude], [Contains], [Equals], // [IgnoreCase], [UseCase] for greater clarity. // Uses first cell of higher dimensional data. -func (ix *Rows) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add +func (rw *Rows) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add lowstr := strings.ToLower(str) - ix.Filter(func(tsr Tensor, row int) bool { + rw.Filter(func(tsr Values, row int) bool { val := tsr.StringRowCell(row, 0) has := false switch { @@ -404,100 +373,78 @@ func (ix *Rows) FilterString(str string, exclude, contains, ignoreCase bool) { / }) } -// NewTensor returns a new tensor with column data organized according to -// the Indexes. If Indexes are nil, a clone of the current tensor is returned -// but this function is only sensible if there is an indexed view in place. -func (ix *Rows) NewTensor() Tensor { - nt := ix.Tensor.Clone() - if ix.Indexes == nil { - return nt +// AsValues returns this tensor as raw [Values]. +// If the row [Rows.Indexes] are nil, then the wrapped Values tensor +// is returned. Otherwise, it "renders" the Rows view into a fully contiguous +// and optimized memory representation of that view, which will be faster +// to access for further processing, and enables all the additional +// functionality provided by the [Values] interface. +func (rw *Rows) AsValues() Values { + if rw.Indexes == nil { + return rw.Tensor } - rows := len(ix.Indexes) - nt.SetNumRows(rows) - _, cells := ix.Tensor.RowCellSize() - str := ix.Tensor.IsString() + vt := NewOfType(rw.Tensor.DataType(), rw.ShapeInts()...) + rows := rw.NumRows() for r := range rows { - for c := range cells { - if str { - nt.SetStringRowCell(ix.StringRowCell(r, c), r, c) - } else { - nt.SetFloatRowCell(ix.FloatRowCell(r, c), r, c) - } - } + vt.SetRowTensor(rw.RowTensor(r), r) } - return nt -} - -// Clone returns a copy of the current Rows view with a cloned copy of -// the underlying Tensor and copy of the indexes. -func (ix *Rows) Clone() Tensor { - nix := &Rows{} - nix.Tensor = ix.Tensor.Clone() - nix.CopyIndexes(ix) - return nix -} - -func (ix *Rows) View() Tensor { - nix := &Rows{} - nix.Tensor = ix.Tensor.View() - nix.CopyIndexes(ix) - return nix + return vt } // CloneIndexes returns a copy of the current Rows view with new indexes, // with a pointer to the same underlying Tensor as the source. -func (ix *Rows) CloneIndexes() *Rows { +func (rw *Rows) CloneIndexes() *Rows { nix := &Rows{} - nix.Tensor = ix.Tensor - nix.CopyIndexes(ix) + nix.Tensor = rw.Tensor + nix.CopyIndexes(rw) return nix } // CopyIndexes copies indexes from other Rows view. -func (ix *Rows) CopyIndexes(oix *Rows) { +func (rw *Rows) CopyIndexes(oix *Rows) { if oix.Indexes == nil { - ix.Indexes = nil + rw.Indexes = nil } else { - ix.Indexes = slices.Clone(oix.Indexes) + rw.Indexes = slices.Clone(oix.Indexes) } } // AddRows adds n rows to end of underlying Tensor, and to the indexes in this view -func (ix *Rows) AddRows(n int) { //types:add - stidx := ix.Tensor.DimSize(0) - ix.Tensor.SetNumRows(stidx + n) - if ix.Indexes != nil { +func (rw *Rows) AddRows(n int) { //types:add + stidx := rw.Tensor.DimSize(0) + rw.Tensor.SetNumRows(stidx + n) + if rw.Indexes != nil { for i := stidx; i < stidx+n; i++ { - ix.Indexes = append(ix.Indexes, i) + rw.Indexes = append(rw.Indexes, i) } } } // InsertRows adds n rows to end of underlying Tensor, and to the indexes starting at // given index in this view -func (ix *Rows) InsertRows(at, n int) { - stidx := ix.Tensor.DimSize(0) - ix.IndexesNeeded() - ix.Tensor.SetNumRows(stidx + n) - nw := make([]int, n, n+len(ix.Indexes)-at) +func (rw *Rows) InsertRows(at, n int) { + stidx := rw.Tensor.DimSize(0) + rw.IndexesNeeded() + rw.Tensor.SetNumRows(stidx + n) + nw := make([]int, n, n+len(rw.Indexes)-at) for i := 0; i < n; i++ { nw[i] = stidx + i } - ix.Indexes = append(ix.Indexes[:at], append(nw, ix.Indexes[at:]...)...) + rw.Indexes = append(rw.Indexes[:at], append(nw, rw.Indexes[at:]...)...) } // DeleteRows deletes n rows of indexes starting at given index in the list of indexes -func (ix *Rows) DeleteRows(at, n int) { - ix.IndexesNeeded() - ix.Indexes = append(ix.Indexes[:at], ix.Indexes[at+n:]...) +func (rw *Rows) DeleteRows(at, n int) { + rw.IndexesNeeded() + rw.Indexes = append(rw.Indexes[:at], rw.Indexes[at+n:]...) } // Swap switches the indexes for i and j -func (ix *Rows) Swap(i, j int) { - if ix.Indexes == nil { +func (rw *Rows) Swap(i, j int) { + if rw.Indexes == nil { return } - ix.Indexes[i], ix.Indexes[j] = ix.Indexes[j], ix.Indexes[i] + rw.Indexes[i], rw.Indexes[j] = rw.Indexes[j], rw.Indexes[i] } /////////////////////////////////////////////// @@ -507,200 +454,200 @@ func (ix *Rows) Swap(i, j int) { // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. -func (ix *Rows) Float(i ...int) float64 { - if ix.Indexes == nil { - return ix.Tensor.Float(i...) +func (rw *Rows) Float(i ...int) float64 { + if rw.Indexes == nil { + return rw.Tensor.Float(i...) } ic := slices.Clone(i) - ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.Float(ic...) + ic[0] = rw.Indexes[ic[0]] + return rw.Tensor.Float(ic...) } // SetFloat sets the value of given index as a float64 // The first index value is indirected through the [Rows.Indexes]. -func (ix *Rows) SetFloat(val float64, i ...int) { - if ix.Indexes == nil { - ix.Tensor.SetFloat(val, i...) +func (rw *Rows) SetFloat(val float64, i ...int) { + if rw.Indexes == nil { + rw.Tensor.SetFloat(val, i...) return } ic := slices.Clone(i) - ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetFloat(val, ic...) + ic[0] = rw.Indexes[ic[0]] + rw.Tensor.SetFloat(val, ic...) } // FloatRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (ix *Rows) FloatRowCell(row, cell int) float64 { - return ix.Tensor.FloatRowCell(ix.RowIndex(row), cell) +func (rw *Rows) FloatRowCell(row, cell int) float64 { + return rw.Tensor.FloatRowCell(rw.RowIndex(row), cell) } // SetFloatRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (ix *Rows) SetFloatRowCell(val float64, row, cell int) { - ix.Tensor.SetFloatRowCell(val, ix.RowIndex(row), cell) +func (rw *Rows) SetFloatRowCell(val float64, row, cell int) { + rw.Tensor.SetFloatRowCell(val, rw.RowIndex(row), cell) } // Float1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Rows) Float1D(i int) float64 { - if ix.Indexes == nil { - return ix.Tensor.Float1D(i) +func (rw *Rows) Float1D(i int) float64 { + if rw.Indexes == nil { + return rw.Tensor.Float1D(i) } - return ix.Float(ix.Tensor.Shape().IndexFrom1D(i)...) + return rw.Float(rw.Tensor.Shape().IndexFrom1D(i)...) } // SetFloat1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Rows) SetFloat1D(val float64, i int) { - if ix.Indexes == nil { - ix.Tensor.SetFloat1D(val, i) +func (rw *Rows) SetFloat1D(val float64, i int) { + if rw.Indexes == nil { + rw.Tensor.SetFloat1D(val, i) } - ix.SetFloat(val, ix.Tensor.Shape().IndexFrom1D(i)...) + rw.SetFloat(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -func (ix *Rows) FloatRow(row int) float64 { - return ix.FloatRowCell(row, 0) +func (rw *Rows) FloatRow(row int) float64 { + return rw.FloatRowCell(row, 0) } -func (ix *Rows) SetFloatRow(val float64, row int) { - ix.SetFloatRowCell(val, row, 0) +func (rw *Rows) SetFloatRow(val float64, row int) { + rw.SetFloatRowCell(val, row, 0) } ///////////////////// Strings // StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. -func (ix *Rows) StringValue(i ...int) string { - if ix.Indexes == nil { - return ix.Tensor.StringValue(i...) +func (rw *Rows) StringValue(i ...int) string { + if rw.Indexes == nil { + return rw.Tensor.StringValue(i...) } ic := slices.Clone(i) - ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.StringValue(ic...) + ic[0] = rw.Indexes[ic[0]] + return rw.Tensor.StringValue(ic...) } // SetString sets the value of given index as a string // The first index value is indirected through the [Rows.Indexes]. -func (ix *Rows) SetString(val string, i ...int) { - if ix.Indexes == nil { - ix.Tensor.SetString(val, i...) +func (rw *Rows) SetString(val string, i ...int) { + if rw.Indexes == nil { + rw.Tensor.SetString(val, i...) } ic := slices.Clone(i) - ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetString(val, ic...) + ic[0] = rw.Indexes[ic[0]] + rw.Tensor.SetString(val, ic...) } // StringRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (ix *Rows) StringRowCell(row, cell int) string { - return ix.Tensor.StringRowCell(ix.RowIndex(row), cell) +func (rw *Rows) StringRowCell(row, cell int) string { + return rw.Tensor.StringRowCell(rw.RowIndex(row), cell) } // SetStringRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (ix *Rows) SetStringRowCell(val string, row, cell int) { - ix.Tensor.SetStringRowCell(val, ix.RowIndex(row), cell) +func (rw *Rows) SetStringRowCell(val string, row, cell int) { + rw.Tensor.SetStringRowCell(val, rw.RowIndex(row), cell) } // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Rows) String1D(i int) string { - if ix.Indexes == nil { - return ix.Tensor.String1D(i) +func (rw *Rows) String1D(i int) string { + if rw.Indexes == nil { + return rw.Tensor.String1D(i) } - return ix.StringValue(ix.Tensor.Shape().IndexFrom1D(i)...) + return rw.StringValue(rw.Tensor.Shape().IndexFrom1D(i)...) } // SetString1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Rows) SetString1D(val string, i int) { - if ix.Indexes == nil { - ix.Tensor.SetString1D(val, i) +func (rw *Rows) SetString1D(val string, i int) { + if rw.Indexes == nil { + rw.Tensor.SetString1D(val, i) } - ix.SetString(val, ix.Tensor.Shape().IndexFrom1D(i)...) + rw.SetString(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -func (ix *Rows) StringRow(row int) string { - return ix.StringRowCell(row, 0) +func (rw *Rows) StringRow(row int) string { + return rw.StringRowCell(row, 0) } -func (ix *Rows) SetStringRow(val string, row int) { - ix.SetStringRowCell(val, row, 0) +func (rw *Rows) SetStringRow(val string, row int) { + rw.SetStringRowCell(val, row, 0) } ///////////////////// Ints // Int returns the value of given index as an int. // The first index value is indirected through the indexes. -func (ix *Rows) Int(i ...int) int { - if ix.Indexes == nil { - return ix.Tensor.Int(i...) +func (rw *Rows) Int(i ...int) int { + if rw.Indexes == nil { + return rw.Tensor.Int(i...) } ic := slices.Clone(i) - ic[0] = ix.Indexes[ic[0]] - return ix.Tensor.Int(ic...) + ic[0] = rw.Indexes[ic[0]] + return rw.Tensor.Int(ic...) } // SetInt sets the value of given index as an int // The first index value is indirected through the [Rows.Indexes]. -func (ix *Rows) SetInt(val int, i ...int) { - if ix.Indexes == nil { - ix.Tensor.SetInt(val, i...) +func (rw *Rows) SetInt(val int, i ...int) { + if rw.Indexes == nil { + rw.Tensor.SetInt(val, i...) return } ic := slices.Clone(i) - ic[0] = ix.Indexes[ic[0]] - ix.Tensor.SetInt(val, ic...) + ic[0] = rw.Indexes[ic[0]] + rw.Tensor.SetInt(val, ic...) } // IntRowCell returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (ix *Rows) IntRowCell(row, cell int) int { - return ix.Tensor.IntRowCell(ix.RowIndex(row), cell) +func (rw *Rows) IntRowCell(row, cell int) int { + return rw.Tensor.IntRowCell(rw.RowIndex(row), cell) } // SetIntRowCell sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (ix *Rows) SetIntRowCell(val int, row, cell int) { - ix.Tensor.SetIntRowCell(val, ix.RowIndex(row), cell) +func (rw *Rows) SetIntRowCell(val int, row, cell int) { + rw.Tensor.SetIntRowCell(val, rw.RowIndex(row), cell) } // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Rows) Int1D(i int) int { - if ix.Indexes == nil { - return ix.Tensor.Int1D(i) +func (rw *Rows) Int1D(i int) int { + if rw.Indexes == nil { + return rw.Tensor.Int1D(i) } - return ix.Int(ix.Tensor.Shape().IndexFrom1D(i)...) + return rw.Int(rw.Tensor.Shape().IndexFrom1D(i)...) } // SetInt1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. -func (ix *Rows) SetInt1D(val int, i int) { - if ix.Indexes == nil { - ix.Tensor.SetInt1D(val, i) +func (rw *Rows) SetInt1D(val int, i int) { + if rw.Indexes == nil { + rw.Tensor.SetInt1D(val, i) } - ix.SetInt(val, ix.Tensor.Shape().IndexFrom1D(i)...) + rw.SetInt(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -func (ix *Rows) IntRow(row int) int { - return ix.IntRowCell(row, 0) +func (rw *Rows) IntRow(row int) int { + return rw.IntRowCell(row, 0) } -func (ix *Rows) SetIntRow(val int, row int) { - ix.SetIntRowCell(val, row, 0) +func (rw *Rows) SetIntRow(val int, row int) { + rw.SetIntRowCell(val, row, 0) } ///////////////////// SubSpaces @@ -713,140 +660,25 @@ func (ix *Rows) SetIntRow(val int, row int) { // Use Clone() method to separate the two. // Rows version does indexed indirection of the outermost row dimension // of the offsets. -func (ix *Rows) SubSpace(offs ...int) Tensor { +func (rw *Rows) SubSpace(offs ...int) Values { if len(offs) == 0 { return nil } - offs[0] = ix.RowIndex(offs[0]) - return ix.Tensor.SubSpace(offs...) + offs[0] = rw.RowIndex(offs[0]) + return rw.Tensor.SubSpace(offs...) } // RowTensor is a convenience version of [Rows.SubSpace] to return the // SubSpace for the outermost row dimension, indirected through the indexes. -func (ix *Rows) RowTensor(row int) Tensor { - return ix.Tensor.RowTensor(ix.RowIndex(row)) +func (rw *Rows) RowTensor(row int) Values { + return rw.Tensor.RowTensor(rw.RowIndex(row)) } // SetRowTensor sets the values of the SubSpace at given row to given values, // with row indirected through the indexes. -func (ix *Rows) SetRowTensor(val Tensor, row int) { - ix.Tensor.SetRowTensor(val, ix.RowIndex(row)) -} - -// CopyFrom copies all values from other tensor into this tensor. -// Checks if source is an Rows and copies indexes too, -// otherwise underlying tensor copies from and indexes are reset. -func (ix *Rows) CopyFrom(from Tensor) { - if fix, ok := from.(*Rows); ok { - ix.Tensor.CopyFrom(fix.Tensor) - ix.CopyIndexes(fix) - return - } - ix.Sequential() - ix.Tensor.CopyFrom(from) -} - -// AppendFrom appends all values from other tensor into this tensor. -// This invalidates the indexes which are reset. -func (ix *Rows) AppendFrom(from Tensor) error { - ix.Sequential() - return ix.Tensor.AppendFrom(from) -} - -// CopyCellsFrom copies given range of values from other tensor into this tensor, -// This invalidates the indexes which are reset. -func (ix *Rows) CopyCellsFrom(from Tensor, to, start, n int) { - ix.Tensor.CopyCellsFrom(from, to, start, n) -} - -func (ix *Rows) Sizeof() int64 { - return ix.Tensor.Sizeof() // todo: could be out of sync with shape! -} - -func (ix *Rows) Bytes() []byte { - return ix.Tensor.Bytes() // todo: could be out of sync with shape! -} - -func (ix *Rows) IsString() bool { - return ix.Tensor.IsString() -} - -func (ix *Rows) DataType() reflect.Kind { - return ix.Tensor.DataType() -} - -func (ix *Rows) Range() (min, max float64, minIndex, maxIndex int) { - return ix.Tensor.Range() -} - -func (ix *Rows) SetZeros() { - ix.Tensor.SetZeros() -} - -////////////////////////// gonum matrix api - -// Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the -// 2D Matrix. Assumes Row-major ordering and logs an error if NumDims < 2. -func (ix *Rows) Dims() (r, c int) { - nd := ix.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0, 0 - } - return ix.DimSize(nd - 2), ix.DimSize(nd - 1) -} - -// Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric -// 2D Matrix. -func (ix *Rows) Symmetric() (r int) { - nd := ix.NumDims() - if nd < 2 { - log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") - return 0 - } - if ix.DimSize(nd-2) != ix.DimSize(nd-1) { - log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") - return 0 - } - return ix.DimSize(nd - 1) -} - -// SymmetricDim returns the number of rows/columns in the matrix. -func (ix *Rows) SymmetricDim() int { - nd := ix.NumDims() - if nd < 2 { - log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") - return 0 - } - if ix.DimSize(nd-2) != ix.DimSize(nd-1) { - log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") - return 0 - } - return ix.DimSize(nd - 1) -} - -// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given -// row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. -func (ix *Rows) At(i, j int) float64 { - nd := ix.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0 - } else if nd == 2 { - return ix.Float(i, j) - } else { - nix := make([]int, nd) - nix[nd-2] = i - nix[nd-1] = j - return ix.Float(nix...) - } -} - -// T is the gonum/mat.Matrix transpose method. -// It performs an implicit transpose by returning the receiver inside a Transpose. -func (ix *Rows) T() mat.Matrix { - return mat.Transpose{ix} +func (rw *Rows) SetRowTensor(val Values, row int) { + rw.Tensor.SetRowTensor(val, rw.RowIndex(row)) } // check for interface impl -var _ Tensor = (*Rows)(nil) +var _ RowCell = (*Rows)(nil) diff --git a/tensor/sliced.go b/tensor/sliced.go index 1800eb01c5..ac21c88656 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -5,15 +5,14 @@ package tensor import ( - "log" "math/rand" "reflect" "slices" "sort" "cogentcore.org/core/base/metadata" + "cogentcore.org/core/base/reflectx" "cogentcore.org/core/base/slicesx" - "gonum.org/v1/gonum/mat" ) // Sliced is a fully indexed wrapper around another [Tensor] that provides a @@ -106,16 +105,6 @@ func (sl *Sliced) IndexFrom1D(oned int) []int { return sl.SliceIndexes(oix...) } -// RowCellIndex returns the full indexes into source tensor based on the -// row and cell index. Maps through 1D index. -func (sl *Sliced) RowCellIndex(row, cell int) []int { - sh := sl.Shape() - _, csz := sh.RowCellSize() // using our shape - oned := row*csz + cell - oix := sh.IndexFrom1D(oned) // full indexes in our coords - return sl.SliceIndexes(oix...) -} - // ValidIndexes ensures that [Sliced.Indexes] are valid, // removing any out-of-range values and setting the view to nil (full sequential) // for any dimension with no indexes (which is an invalid condition). @@ -167,7 +156,7 @@ func (sl *Sliced) IndexesNeeded(d int) { // Label satisfies the core.Labeler interface for a summary description of the tensor. func (sl *Sliced) Label() string { - return sl.Tensor.Label() + return label(sl.Metadata().Name(), sl.Shape()) } // String satisfies the fmt.Stringer interface for string of tensor data. @@ -179,6 +168,14 @@ func (sl *Sliced) String() string { // to encode plotting options, etc. func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } +func (sl *Sliced) IsString() bool { + return sl.Tensor.IsString() +} + +func (sl *Sliced) DataType() reflect.Kind { + return sl.Tensor.DataType() +} + // For each dimension, we return the effective shape sizes using // the current number of indexes per dimension. func (sl *Sliced) ShapeInts() []int { @@ -206,27 +203,6 @@ func (sl *Sliced) Shape() *Shape { return NewShape(sl.ShapeInts()...) } -// SetShapeInts sets the shape of the underlying wrapped tensor -// to the given sizes per dimension, and resets our indexes -// which are now invalid. -func (sl *Sliced) SetShapeInts(sizes ...int) { - sl.Tensor.SetShapeInts(sizes...) - sl.Sequential() -} - -// SetShape sets our shape to given sizes. -// See [Sliced.SetShapeInts] for details. -func (sl *Sliced) SetShape(sizes Tensor) { - sl.SetShapeInts(AsIntSlice(sizes)...) -} - -// SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. -// This invalidates the indexes. -func (sl *Sliced) SetNumRows(rows int) { - sl.Sequential() - sl.Tensor.SetNumRows(rows) -} - // Len returns the total number of elements in our view of the tensor. func (sl *Sliced) Len() int { return sl.Shape().Len() @@ -243,167 +219,49 @@ func (sl *Sliced) DimSize(dim int) int { return sl.Tensor.DimSize(dim) } -// RowCellSize returns the size of the outermost Row shape dimension -// (via [Sliced.Rows] method), and the size of all the remaining -// inner dimensions (the "cell" size). -func (sl *Sliced) RowCellSize() (rows, cells int) { - rows = sl.DimSize(0) - nd := sl.Tensor.NumDims() - if nd == 1 { - cells = 1 - } else if rows > 0 { - cells = sl.Len() / rows - } else { - ln := 1 - for d := 1; d < nd; d++ { - ln *= sl.DimSize(d) +// AsValues returns a copy of this tensor as raw [Values]. +// This "renders" the Sliced view into a fully contiguous +// and optimized memory representation of that view, which will be faster +// to access for further processing, and enables all the additional +// functionality provided by the [Values] interface. +func (sl *Sliced) AsValues() Values { + dt := sl.Tensor.DataType() + vt := NewOfType(dt, sl.ShapeInts()...) + n := sl.Len() + switch { + case sl.Tensor.IsString(): + for i := range n { + vt.SetString1D(sl.String1D(i), i) } - cells = ln - } - return -} - -// Permuted sets indexes in given dimension to a permuted order. -// If indexes already exist then existing list of indexes is permuted, -// otherwise a new set of permuted indexes are generated -func (sl *Sliced) Permuted(dim int) { - ix := sl.Indexes[dim] - if ix == nil { - ix = rand.Perm(sl.Tensor.DimSize(dim)) - } else { - rand.Shuffle(len(ix), func(i, j int) { - ix[i], ix[j] = ix[j], ix[i] - }) - } - sl.Indexes[dim] = ix -} - -// SortFunc sorts the indexes along given dimension using given compare function. -// The compare function operates directly on indexes into the Tensor -// as these row numbers have already been projected through the indexes. -// cmp(a, b) should return a negative number when a < b, a positive -// number when a > b and zero when a == b. -func (sl *Sliced) SortFunc(dim int, cmp func(tsr Tensor, dim, i, j int) int) { - sl.IndexesNeeded(dim) - ix := sl.Indexes[dim] - slices.SortFunc(ix, func(a, b int) int { - return cmp(sl.Tensor, dim, a, b) // key point: these are already indirected through indexes!! - }) - sl.Indexes[dim] = ix -} - -// SortIndexes sorts the indexes along given dimension directly in -// numerical order, producing the native ordering, while preserving -// any filtering that might have occurred. -func (sl *Sliced) SortIndexes(dim int) { - ix := sl.Indexes[dim] - if ix == nil { - return - } - sort.Ints(ix) - sl.Indexes[dim] = ix -} - -// SortStableFunc stably sorts along given dimension using given compare function. -// The compare function operates directly on row numbers into the Tensor -// as these row numbers have already been projected through the indexes. -// cmp(a, b) should return a negative number when a < b, a positive -// number when a > b and zero when a == b. -// It is *essential* that it always returns 0 when the two are equal -// for the stable function to actually work. -func (sl *Sliced) SortStableFunc(dim int, cmp func(tsr Tensor, dim, i, j int) int) { - sl.IndexesNeeded(dim) - ix := sl.Indexes[dim] - slices.SortStableFunc(ix, func(a, b int) int { - return cmp(sl.Tensor, dim, a, b) // key point: these are already indirected through indexes!! - }) - sl.Indexes[dim] = ix -} - -// Filter filters the indexes using given Filter function. -// The Filter function operates directly on row numbers into the Tensor -// as these row numbers have already been projected through the indexes. -func (sl *Sliced) Filter(dim int, filterer func(tsr Tensor, dim, idx int) bool) { - sl.IndexesNeeded(dim) - ix := sl.Indexes[dim] - sz := len(ix) - for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering - if !filterer(sl, dim, ix[i]) { // delete - ix = append(ix[:i], ix[i+1:]...) + case reflectx.KindIsFloat(dt): + for i := range n { + vt.SetFloat1D(sl.Float1D(i), i) } - } - sl.Indexes[dim] = ix -} - -// NewTensor returns a new tensor with column data organized according to -// the Indexes. If Indexes are nil, a clone of the current tensor is returned -// but this function is only sensible if there is an indexed view in place. -func (sl *Sliced) NewTensor() Tensor { - nt := sl.Tensor.Clone() - if sl.Indexes == nil { - return nt - } - rows := len(sl.Indexes) - nt.SetNumRows(rows) - _, cells := sl.Tensor.RowCellSize() - str := sl.Tensor.IsString() - for r := range rows { - for c := range cells { - if str { - nt.SetStringRowCell(sl.StringRowCell(r, c), r, c) - } else { - nt.SetFloatRowCell(sl.FloatRowCell(r, c), r, c) - } + default: + for i := range n { + vt.SetInt1D(sl.Int1D(i), i) } } - return nt -} - -// Clone returns a copy of the current Sliced view with a cloned copy of -// the underlying Tensor and copy of the indexes. -func (sl *Sliced) Clone() Tensor { - nix := &Sliced{} - nix.Tensor = sl.Tensor.Clone() - nix.CopyIndexes(sl) - return nix -} - -func (sl *Sliced) View() Tensor { - nix := &Sliced{} - nix.Tensor = sl.Tensor.View() - nix.CopyIndexes(sl) - return nix -} - -// CloneIndexes returns a copy of the current Sliced view with new indexes, -// with a pointer to the same underlying Tensor as the source. -func (sl *Sliced) CloneIndexes() *Sliced { - nix := &Sliced{} - nix.Tensor = sl.Tensor - nix.CopyIndexes(sl) - return nix -} - -// CopyIndexes copies indexes from other Sliced view. -func (sl *Sliced) CopyIndexes(oix *Sliced) { - if oix.Indexes == nil { - sl.Indexes = nil - } else { - sl.Indexes = slices.Clone(oix.Indexes) - } -} - -// AppendFrom appends all values from other tensor into this tensor. -// This invalidates the indexes which are reset. -func (sl *Sliced) AppendFrom(from Tensor) error { - sl.Sequential() - return sl.Tensor.AppendFrom(from) -} - -// CopyCellsFrom copies given range of values from other tensor into this tensor, -func (sl *Sliced) CopyCellsFrom(from Tensor, to, start, n int) { - sl.Tensor.CopyCellsFrom(from, to, start, n) -} + return vt +} + +// // CloneIndexes returns a copy of the current Sliced view with new indexes, +// // with a pointer to the same underlying Tensor as the source. +// func (sl *Sliced) CloneIndexes() *Sliced { +// nix := &Sliced{} +// nix.Tensor = sl.Tensor +// nix.CopyIndexes(sl) +// return nix +// } +// +// // CopyIndexes copies indexes from other Sliced view. +// func (sl *Sliced) CopyIndexes(oix *Sliced) { +// if oix.Indexes == nil { +// sl.Indexes = nil +// } else { +// sl.Indexes = slices.Clone(oix.Indexes) +// } +// } /////////////////////////////////////////////// // Sliced access @@ -422,21 +280,6 @@ func (sl *Sliced) SetFloat(val float64, i ...int) { sl.Tensor.SetFloat(val, sl.SliceIndexes(i...)...) } -// FloatRowCell returns the value at given row and cell, -// where row is outermost dim, and cell is 1D index into remaining inner dims. -// The indexes are indirected through the [Sliced.Indexes]. -func (sl *Sliced) FloatRowCell(row, cell int) float64 { - return sl.Tensor.Float(sl.RowCellIndex(row, cell)...) -} - -// SetFloatRowCell sets the value at given row and cell, -// where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Sliced.Indexes]. -// This is the preferred interface for all Sliced operations. -func (sl *Sliced) SetFloatRowCell(val float64, row, cell int) { - sl.Tensor.SetFloat(val, sl.RowCellIndex(row, cell)...) -} - // Float1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Float1D(i int) float64 { @@ -449,14 +292,6 @@ func (sl *Sliced) SetFloat1D(val float64, i int) { sl.Tensor.SetFloat(val, sl.IndexFrom1D(i)...) } -func (sl *Sliced) FloatRow(row int) float64 { - return sl.FloatRowCell(row, 0) -} - -func (sl *Sliced) SetFloatRow(val float64, row int) { - sl.SetFloatRowCell(val, row, 0) -} - ///////////////////// Strings // StringValue returns the value of given index as a string. @@ -471,22 +306,6 @@ func (sl *Sliced) SetString(val string, i ...int) { sl.Tensor.SetString(val, sl.SliceIndexes(i...)...) } -// StringRowCell returns the value at given row and cell, -// where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Sliced.Indexes]. -// This is the preferred interface for all Sliced operations. -func (sl *Sliced) StringRowCell(row, cell int) string { - return sl.Tensor.StringValue(sl.RowCellIndex(row, cell)...) -} - -// SetStringRowCell sets the value at given row and cell, -// where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Sliced.Indexes]. -// This is the preferred interface for all Sliced operations. -func (sl *Sliced) SetStringRowCell(val string, row, cell int) { - sl.Tensor.SetString(val, sl.RowCellIndex(row, cell)...) -} - // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) String1D(i int) string { @@ -499,14 +318,6 @@ func (sl *Sliced) SetString1D(val string, i int) { sl.Tensor.SetString(val, sl.IndexFrom1D(i)...) } -func (sl *Sliced) StringRow(row int) string { - return sl.StringRowCell(row, 0) -} - -func (sl *Sliced) SetStringRow(val string, row int) { - sl.SetStringRowCell(val, row, 0) -} - ///////////////////// Ints // Int returns the value of given index as an int. @@ -521,22 +332,6 @@ func (sl *Sliced) SetInt(val int, i ...int) { sl.Tensor.SetInt(val, sl.SliceIndexes(i...)...) } -// IntRowCell returns the value at given row and cell, -// where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Sliced.Indexes]. -// This is the preferred interface for all Sliced operations. -func (sl *Sliced) IntRowCell(row, cell int) int { - return sl.Tensor.Int(sl.RowCellIndex(row, cell)...) -} - -// SetIntRowCell sets the value at given row and cell, -// where row is outermost dim, and cell is 1D index into remaining inner dims. -// Row is indirected through the [Sliced.Indexes]. -// This is the preferred interface for all Sliced operations. -func (sl *Sliced) SetIntRowCell(val int, row, cell int) { - sl.Tensor.SetInt(val, sl.RowCellIndex(row, cell)...) -} - // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Int1D(i int) int { @@ -549,131 +344,76 @@ func (sl *Sliced) SetInt1D(val int, i int) { sl.Tensor.SetInt(val, sl.IndexFrom1D(i)...) } -func (sl *Sliced) IntRow(row int) int { - return sl.IntRowCell(row, 0) -} - -func (sl *Sliced) SetIntRow(val int, row int) { - sl.SetIntRowCell(val, row, 0) -} - -///////////////////// SubSpaces - -// SubSpace is not supported by Sliced. -func (sl *Sliced) SubSpace(offs ...int) Tensor { - return nil -} - -// RowTensor is not supported by Sliced. -func (sl *Sliced) RowTensor(row int) Tensor { - return nil -} - -// RowTensor is not supported by Sliced. -func (sl *Sliced) SetRowTensor(val Tensor, row int) { -} - -// CopyFrom copies all values from other tensor into this tensor. -// Checks if source is an Sliced and copies indexes too, -// otherwise underlying tensor copies from and indexes are reset. -func (sl *Sliced) CopyFrom(from Tensor) { - // todo: what should this do? - if fix, ok := from.(*Sliced); ok { - sl.Tensor.CopyFrom(fix.Tensor) - sl.CopyIndexes(fix) - return +// Permuted sets indexes in given dimension to a permuted order. +// If indexes already exist then existing list of indexes is permuted, +// otherwise a new set of permuted indexes are generated +func (sl *Sliced) Permuted(dim int) { + ix := sl.Indexes[dim] + if ix == nil { + ix = rand.Perm(sl.Tensor.DimSize(dim)) + } else { + rand.Shuffle(len(ix), func(i, j int) { + ix[i], ix[j] = ix[j], ix[i] + }) } - sl.Sequential() - sl.Tensor.CopyFrom(from) -} - -func (sl *Sliced) Sizeof() int64 { - return sl.Tensor.Sizeof() // todo: could be out of sync with shape! -} - -func (sl *Sliced) Bytes() []byte { - return sl.Tensor.Bytes() // todo: could be out of sync with shape! -} - -func (sl *Sliced) IsString() bool { - return sl.Tensor.IsString() -} - -func (sl *Sliced) DataType() reflect.Kind { - return sl.Tensor.DataType() -} - -func (sl *Sliced) Range() (min, max float64, minIndex, maxIndex int) { - return sl.Tensor.Range() + sl.Indexes[dim] = ix } -func (sl *Sliced) SetZeros() { - sl.Tensor.SetZeros() +// SortFunc sorts the indexes along given dimension using given compare function. +// The compare function operates directly on indexes into the Tensor +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +func (sl *Sliced) SortFunc(dim int, cmp func(tsr Tensor, dim, i, j int) int) { + sl.IndexesNeeded(dim) + ix := sl.Indexes[dim] + slices.SortFunc(ix, func(a, b int) int { + return cmp(sl.Tensor, dim, a, b) // key point: these are already indirected through indexes!! + }) + sl.Indexes[dim] = ix } -////////////////////////// gonum matrix api - -// Dims is the gonum/mat.Matrix interface method for returning the dimensionality of the -// 2D Matrix. Assumes Row-major ordering and logs an error if NumDims < 2. -func (sl *Sliced) Dims() (r, c int) { - nd := sl.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0, 0 +// SortIndexes sorts the indexes along given dimension directly in +// numerical order, producing the native ordering, while preserving +// any filtering that might have occurred. +func (sl *Sliced) SortIndexes(dim int) { + ix := sl.Indexes[dim] + if ix == nil { + return } - return sl.DimSize(nd - 2), sl.DimSize(nd - 1) + sort.Ints(ix) + sl.Indexes[dim] = ix } -// Symmetric is the gonum/mat.Matrix interface method for returning the dimensionality of a symmetric -// 2D Matrix. -func (sl *Sliced) Symmetric() (r int) { - nd := sl.NumDims() - if nd < 2 { - log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") - return 0 - } - if sl.DimSize(nd-2) != sl.DimSize(nd-1) { - log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") - return 0 - } - return sl.DimSize(nd - 1) +// SortStableFunc stably sorts along given dimension using given compare function. +// The compare function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +// cmp(a, b) should return a negative number when a < b, a positive +// number when a > b and zero when a == b. +// It is *essential* that it always returns 0 when the two are equal +// for the stable function to actually work. +func (sl *Sliced) SortStableFunc(dim int, cmp func(tsr Tensor, dim, i, j int) int) { + sl.IndexesNeeded(dim) + ix := sl.Indexes[dim] + slices.SortStableFunc(ix, func(a, b int) int { + return cmp(sl.Tensor, dim, a, b) // key point: these are already indirected through indexes!! + }) + sl.Indexes[dim] = ix } -// SymmetricDim returns the number of rows/columns in the matrix. -func (sl *Sliced) SymmetricDim() int { - nd := sl.NumDims() - if nd < 2 { - log.Println("tensor Symmetric gonum Matrix call made on Tensor with dims < 2") - return 0 - } - if sl.DimSize(nd-2) != sl.DimSize(nd-1) { - log.Println("tensor Symmetric gonum Matrix call made on Tensor that is not symmetric") - return 0 - } - return sl.DimSize(nd - 1) -} - -// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given -// row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. -func (sl *Sliced) At(i, j int) float64 { - nd := sl.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0 - } else if nd == 2 { - return sl.Float(i, j) - } else { - nix := make([]int, nd) - nix[nd-2] = i - nix[nd-1] = j - return sl.Float(nix...) +// Filter filters the indexes using given Filter function. +// The Filter function operates directly on row numbers into the Tensor +// as these row numbers have already been projected through the indexes. +func (sl *Sliced) Filter(dim int, filterer func(tsr Tensor, dim, idx int) bool) { + sl.IndexesNeeded(dim) + ix := sl.Indexes[dim] + sz := len(ix) + for i := sz - 1; i >= 0; i-- { // always go in reverse for filtering + if !filterer(sl, dim, ix[i]) { // delete + ix = append(ix[:i], ix[i+1:]...) + } } -} - -// T is the gonum/mat.Matrix transpose method. -// It performs an implicit transpose by returning the receiver inside a Transpose. -func (sl *Sliced) T() mat.Matrix { - return mat.Transpose{sl} + sl.Indexes[dim] = ix } // check for interface impl diff --git a/tensor/slices.go b/tensor/slices.go index 4e6cce0f0c..e5b67915b6 100644 --- a/tensor/slices.go +++ b/tensor/slices.go @@ -6,12 +6,13 @@ package tensor // Slice represents a slice of index values, for extracting slices of data, // along a dimension of a given size, which is provided separately as an argument. -// using standard 'for' loop logic with a Start and _exclusive_ Stop value, -// and an increment: for i := Start; i < Stop; i += Step. +// Uses standard 'for' loop logic with a Start and _exclusive_ Stop value, +// and a Step increment: for i := Start; i < Stop; i += Step. // The values stored in this struct are the _inputs_ for computing the actual -// slice ranges based on an additional actual size parameter for the dimension. +// slice values based on the actual size parameter for the dimension. // Negative numbers count back from the end (i.e., size + val), and -// the zero value results in all values in the dimension, with Step = 1 if 0. +// the zero value results in a list of all values in the dimension, with Step = 1 if 0. +// The behavior is identical to the NumPy slice. type Slice struct { // Starting value. If 0 and Step < 0, = size-1; // If negative, = size+Start. @@ -27,9 +28,9 @@ type Slice struct { Step int } -// NewSlice returns a new Slice with given values. -func NewSlice(start, end, incr int) Slice { - return Slice{Start: start, Stop: end, Step: incr} +// NewSlice returns a new Slice with given srat, stop, step values. +func NewSlice(start, stop, step int) Slice { + return Slice{Start: start, Stop: stop, Step: step} } // StartActual is the actual start value given the size of the dimension. @@ -138,9 +139,9 @@ func (sl Slice) IntTensor(size int) *Int { return tsr } -// IntSlice returns a new [Int] [Tensor] with given [Slice] values. -// Stop must be an actual end value and not a size-relative number (such as -1). -func IntSlice(start, end, incr int) *Int { - sl := NewSlice(start, end, incr) - return sl.IntTensor(end) +// NewSliceInts returns a new [Int] [Tensor] with given [Slice] start, stop, step values, +// for given dimension size. +func NewSliceInts(size, start, stop, step int) *Int { + sl := NewSlice(start, stop, step) + return sl.IntTensor(size) } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 6aa2446cce..9fa28ff20d 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -124,7 +124,7 @@ func MaxAbsFunc(in, out tensor.Tensor) { // and returns the Float64 output values for subsequent use. func MeanFuncOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor) { sum64 := SumFuncOut64(in, out) - count := out.Clone() + count := out.AsValues().Clone() count64 = CountFuncOut64(in, count) nsub := out.Len() for i := range nsub { diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 9a70e99ee1..9607c4dc28 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -165,7 +165,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) { sv := datafs.NewValue[float64](vd, stnm, nv) for i, v := range vals { idx := tensor.AsIntSlice(v) - sg := tensor.NewRows(tsr, idx...) + sg := tensor.NewRows(tsr.AsValues(), idx...) tensor.Call(stat, sg, stout) sv.SetFloatRow(stout.Float1D(0), i) } diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index dbd72999fc..5ac66f9c27 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -26,7 +26,7 @@ func QuantilesFunc(in, qs, out tensor.Tensor) error { if qs.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } - sin := tensor.AsRows(in.Clone()) + sin := tensor.AsRows(in.AsValues()) sin.ExcludeMissing() sin.Sort(tensor.Ascending) tensor.SetShapeFrom(out, qs) diff --git a/tensor/string.go b/tensor/string.go index f2e3c5dcb9..ce35fe048d 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -6,12 +6,9 @@ package tensor import ( "fmt" - "log" - "math" "strconv" "cogentcore.org/core/base/errors" - "gonum.org/v1/gonum/mat" ) // String is a tensor of string values @@ -60,6 +57,8 @@ func (tsr *String) IsString() bool { return true } +func (tsr *String) AsValues() Values { return tsr } + ///////////////////// Strings func (tsr *String) SetString(val string, i ...int) { @@ -156,52 +155,6 @@ func (tsr *String) SetIntRow(val int, row int) { tsr.SetIntRowCell(val, row, 0) } -// At is the gonum/mat.Matrix interface method for returning 2D matrix element at given -// row, column index. Assumes Row-major ordering and logs an error if NumDims < 2. -func (tsr *String) At(i, j int) float64 { - nd := tsr.NumDims() - if nd < 2 { - log.Println("tensor Dims gonum Matrix call made on Tensor with dims < 2") - return 0 - } else if nd == 2 { - return tsr.Float(i, j) - } else { - ix := make([]int, nd) - ix[nd-2] = i - ix[nd-1] = j - return tsr.Float(ix...) - } -} - -// T is the gonum/mat.Matrix transpose method. -// It performs an implicit transpose by returning the receiver inside a Transpose. -func (tsr *String) T() mat.Matrix { - return mat.Transpose{tsr} -} - -// Range returns the min, max (and associated indexes, -1 = no values) for the tensor. -// This is needed for display and is thus in the core api in optimized form -// Other math operations can be done using gonum/floats package. -func (tsr *String) Range() (min, max float64, minIndex, maxIndex int) { - minIndex = -1 - maxIndex = -1 - for j, vl := range tsr.Values { - fv := StringToFloat64(vl) - if math.IsNaN(fv) { - continue - } - if fv < min || minIndex < 0 { - min = fv - minIndex = j - } - if fv > max || maxIndex < 0 { - max = fv - maxIndex = j - } - } - return -} - // SetZeros is a simple convenience function initialize all values to the // zero value of the type (empty strings for string type). func (tsr *String) SetZeros() { @@ -213,20 +166,20 @@ func (tsr *String) SetZeros() { // Clone clones this tensor, creating a duplicate copy of itself with its // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). -func (tsr *String) Clone() Tensor { +func (tsr *String) Clone() Values { csr := NewStringShape(&tsr.shape) copy(csr.Values, tsr.Values) return csr } -func (tsr *String) View() Tensor { +func (tsr *String) View() Values { return &String{*tsr.view()} } // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. -func (tsr *String) CopyFrom(frm Tensor) { +func (tsr *String) CopyFrom(frm Values) { if fsm, ok := frm.(*String); ok { copy(tsr.Values, fsm.Values) return @@ -242,7 +195,7 @@ func (tsr *String) CopyFrom(frm Tensor) { // It uses and optimized implementation if the other tensor // is of the same type, and otherwise it goes through // appropriate standard type. -func (tsr *String) AppendFrom(frm Tensor) error { +func (tsr *String) AppendFrom(frm Values) error { rows, cell := tsr.RowCellSize() frows, fcell := frm.RowCellSize() if cell != fcell { @@ -266,7 +219,7 @@ func (tsr *String) AppendFrom(frm Tensor) error { // start = starting index on from Tensor to start copying from, and n = number of // values to copy. Uses an optimized implementation if the other tensor is // of the same type, and otherwise it goes through appropriate standard type. -func (tsr *String) CopyCellsFrom(frm Tensor, to, start, n int) { +func (tsr *String) CopyCellsFrom(frm Values, to, start, n int) { if fsm, ok := frm.(*String); ok { for i := 0; i < n; i++ { tsr.Values[to+i] = fsm.Values[start+i] @@ -284,7 +237,7 @@ func (tsr *String) CopyCellsFrom(frm Tensor, to, start, n int) { // will affect both), as its Values slice is a view onto the original (which // is why only inner-most contiguous supsaces are supported). // Use Clone() method to separate the two. -func (tsr *String) SubSpace(offs ...int) Tensor { +func (tsr *String) SubSpace(offs ...int) Values { b := tsr.subSpaceImpl(offs...) rt := &String{Base: *b} return rt @@ -293,12 +246,12 @@ func (tsr *String) SubSpace(offs ...int) Tensor { // RowTensor is a convenience version of [Tensor.SubSpace] to return the // SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. -func (tsr *String) RowTensor(row int) Tensor { +func (tsr *String) RowTensor(row int) Values { return tsr.SubSpace(row) } // SetRowTensor sets the values of the SubSpace at given row to given values. -func (tsr *String) SetRowTensor(val Tensor, row int) { +func (tsr *String) SetRowTensor(val Values, row int) { _, cells := tsr.RowCellSize() st := row * cells mx := min(val.Len(), cells) diff --git a/tensor/table/columns.go b/tensor/table/columns.go index 6ea345c60a..75a14ca5d2 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -5,13 +5,15 @@ package table import ( + "cogentcore.org/core/base/keylist" "cogentcore.org/core/tensor" ) // Columns is the underlying column list and number of rows for Table. -// [Table] is an Indexed view onto the Columns. +// Each column is a raw [tensor.Values] tensor, and [Table] +// provides a [tensor.Rows] indexed view onto the Columns. type Columns struct { - tensor.List + keylist.List[string, tensor.Values] // number of rows, which is enforced to be the size of the // outermost row dimension of the column tensors. @@ -35,11 +37,11 @@ func (cl *Columns) SetNumRows(rows int) *Columns { //types:add return cl } -// AddColumn adds the given tensor as a column, +// AddColumn adds the given tensor (as a [tensor.Values]) as a column, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows, // and calls the metadata SetName with column name. -func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { +func (cl *Columns) AddColumn(name string, tsr tensor.Values) error { err := cl.Add(name, tsr) if err != nil { return err @@ -52,7 +54,7 @@ func (cl *Columns) AddColumn(name string, tsr tensor.Tensor) error { // InsertColumn inserts the given tensor as a column at given index, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows. -func (cl *Columns) InsertColumn(idx int, name string, tsr tensor.Tensor) error { +func (cl *Columns) InsertColumn(idx int, name string, tsr tensor.Values) error { cl.Insert(idx, name, tsr) tsr.SetNumRows(cl.Rows) return nil diff --git a/tensor/table/table.go b/tensor/table/table.go index 30b05c648b..1afb83fbec 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -170,17 +170,17 @@ func InsertColumn[T tensor.DataTypes](dt *Table, name string, idx int, cellSizes return tsr } -// AddColumn adds the given tensor as a column to the table, +// AddColumn adds the given [tensor.Values] as a column to the table, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows. -func (dt *Table) AddColumn(name string, tsr tensor.Tensor) error { +func (dt *Table) AddColumn(name string, tsr tensor.Values) error { return dt.Columns.AddColumn(name, tsr) } -// InsertColumn inserts the given tensor as a column to the table at given index, +// InsertColumn inserts the given [tensor.Values] as a column to the table at given index, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows. -func (dt *Table) InsertColumn(idx int, name string, tsr tensor.Tensor) error { +func (dt *Table) InsertColumn(idx int, name string, tsr tensor.Values) error { return dt.Columns.InsertColumn(idx, name, tsr) } diff --git a/tensor/tensor.go b/tensor/tensor.go index b837f93191..425e1132fc 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -10,10 +10,7 @@ import ( "fmt" "reflect" - "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/keylist" "cogentcore.org/core/base/metadata" - "gonum.org/v1/gonum/mat" ) // DataTypes are the primary tensor data types with specific support. @@ -31,19 +28,26 @@ var MaxSprintLength = 1000 // It is also possible to use Column-Major order, which is used in R, Julia, and MATLAB // where the inner-most index is first and outermost last. -// Tensor is the interface for n-dimensional tensors. +// Tensor is the most general interface for n-dimensional tensors. // Per C / Go / Python conventions, indexes are Row-Major, ordered from // outer to inner left-to-right, so the inner-most is right-most. -// It is implemented by the Base and Number generic types specialized -// by different concrete types: float64, float32, int, int32, byte, -// string, bits (bools). -// For float32 and float64 values, use NaN to indicate missing values. -// All of the data analysis and plot packages skip NaNs. +// It is implemented for raw [Values] with direct integer indexing +// by the [Number], [String], and [Bool] types, covering the different +// concrete types specified by [DataTypes] (see [Values] for +// additional interface methods for raw value types). +// For float32 and float64 values, use NaN to indicate missing values, +// as all of the data analysis and plot packages skip NaNs. +// View Tensor types provide different ways of viewing a source tensor, +// including [Sliced] for arbitrary slices of dimension indexes, +// [Masked] for boolean masked access and setting of individual indexes, +// and [Indexed] for arbitrary indexes of values, organized into the +// shape of the indexes, not the original source data. +// The [Rows] view provides an optimized row-indexed view for [table.Table] data. type Tensor interface { fmt.Stringer - mat.Matrix - // Label satisfies the core.Labeler interface for a summary description of the tensor. + // Label satisfies the core.Labeler interface for a summary + // description of the tensor, including metadata Name if set. Label() string // Metadata returns the metadata for this tensor, which can be used @@ -52,9 +56,9 @@ type Tensor interface { // Shape() returns a [Shape] representation of the tensor shape // (dimension sizes). For tensors that present a view onto another - // tensor, this typically must be constructed on the fly. - // In general, it is much better to access [Tensor.ShapeSizes], - // [Tensor.ShapeInts], [Tensor.DimSize] etc information as neeed. + // tensor, this typically must be constructed. + // In general, it is better to use the specific [Tensor.ShapeSizes], + // [Tensor.ShapeInts], [Tensor.DimSize] etc as neeed. Shape() *Shape // ShapeSizes returns the sizes of each dimension as an int tensor. @@ -64,14 +68,6 @@ type Tensor interface { // This is the preferred access for Go code. ShapeInts() []int - // SetShape sets the dimension sizes as 1D int values from given tensor. - // The backing storage is resized appropriately, retaining all existing data that fits. - SetShape(sizes Tensor) - - // SetShapeInts sets the dimension sizes of the tensor, and resizes - // backing storage appropriately, retaining all existing data that fits. - SetShapeInts(sizes ...int) - // Len returns the total number of elements in the tensor, // i.e., the product of all shape dimensions. // Len must always be such that the 1D() accessors return @@ -84,28 +80,20 @@ type Tensor interface { // DimSize returns size of given dimension. DimSize(dim int) int - // RowCellSize returns the size of the outermost Row shape dimension, - // and the size of all the remaining inner dimensions (the "cell" size). - // Commonly used to organize multiple instances (rows) of higher-dimensional - // patterns (cells), and the [Rows] type operates on the outer row dimension. - RowCellSize() (rows, cells int) - // DataType returns the type of the data elements in the tensor. // Bool is returned for the Bits tensor type. DataType() reflect.Kind - // Sizeof returns the number of bytes contained in the Values of this tensor. - // For String types, this is just the string pointers, not the string content. - Sizeof() int64 - - // Bytes returns the underlying byte representation of the tensor values. - // This is the actual underlying data, so make a copy if it can be - // unintentionally modified or retained more than for immediate use. - Bytes() []byte - // IsString returns true if the data type is a String; otherwise it is numeric. IsString() bool + // AsValues returns this tensor as raw [Values]. If it already is, + // it is returned directly. If it is a View tensor, the view is + // "rendered" into a fully contiguous and optimized [Values] representation + // of that view, which will be faster to access for further processing, + // and enables all the additional functionality provided by the [Values] interface. + AsValues() Values + ///////////////////// Floats // Float returns the value of given n-dimensional index (matching Shape) as a float64. @@ -125,30 +113,6 @@ type Tensor interface { // [Tensor.SetFloatRowCell] is preferred. SetFloat1D(val float64, i int) - // FloatRowCell returns the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - FloatRowCell(row, cell int) float64 - - // SetFloatRowCell sets the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - SetFloatRowCell(val float64, row, cell int) - - // FloatRow returns the value at given row (outermost dimension). - // It is a convenience wrapper for FloatRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - FloatRow(row int) float64 - - // SetFloatRow sets the value at given row (outermost dimension). - // It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - SetFloatRow(val float64, row int) - ///////////////////// Strings // StringValue returns the value of given n-dimensional index (matching Shape) as a string. @@ -164,30 +128,6 @@ type Tensor interface { // SetString1D sets the value of given 1-dimensional index (0-Len()-1) as a string. SetString1D(val string, i int) - // StringRowCell returns the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - StringRowCell(row, cell int) string - - // SetStringRowCell sets the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - SetStringRowCell(val string, row, cell int) - - // StringRow returns the value at given row (outermost dimension). - // It is a convenience wrapper for StringRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - StringRow(row int) string - - // SetStringRow sets the value at given row (outermost dimension). - // It is a convenience wrapper for SetStringRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - SetStringRow(val string, row int) - ///////////////////// Ints // Int returns the value of given n-dimensional index (matching Shape) as a int. @@ -201,158 +141,4 @@ type Tensor interface { // SetInt1D sets the value of given 1-dimensional index (0-Len()-1) as a int. SetInt1D(val int, i int) - - // IntRowCell returns the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - IntRowCell(row, cell int) int - - // SetIntRowCell sets the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - SetIntRowCell(val int, row, cell int) - - // IntRow returns the value at given row (outermost dimension). - // It is a convenience wrapper for IntRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - IntRow(row int) int - - // SetIntRow sets the value at given row (outermost dimension). - // It is a convenience wrapper for SetIntRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - SetIntRow(val int, row int) - - ///////////////////// SubSpaces - - // SubSpace returns a new tensor with innermost subspace at given - // offset(s) in outermost dimension(s) (len(offs) < [NumDims]). - // The new tensor points to the values of the this tensor (i.e., modifications - // will affect both), as its Values slice is a view onto the original (which - // is why only inner-most contiguous supsaces are supported). - // Use Clone() method to separate the two. See [Slice] function to - // extract arbitrary subspaces along ranges of each dimension. - SubSpace(offs ...int) Tensor - - // RowTensor is a convenience version of [Tensor.SubSpace] to return the - // SubSpace for the outermost row dimension. [Rows] defines a version - // of this that indirects through the row indexes. - RowTensor(row int) Tensor - - // SetRowTensor sets the values of the [Tensor.SubSpace] at given row to given values. - SetRowTensor(val Tensor, row int) - - // Range returns the min, max (and associated indexes, -1 = no values) for the tensor. - // This is needed for display and is thus in the core api in optimized form. - // See the [stats] package for many more statistics functions. - Range() (min, max float64, minIndex, maxIndex int) - - // SetZeros is a simple convenience function initialize all values to the - // zero value of the type (empty strings for string type). - SetZeros() - - // Clone clones this tensor, creating a duplicate copy of itself with its - // own separate memory representation of all the values, and returns - // that as a Tensor (which can be converted into the known type as needed). - Clone() Tensor - - // View clones this tensor, *keeping the same underlying Values slice*, - // instead of making a copy like [Tensor.Clone] does. A major use of this - // is to then change the shape of the view to provide a different way - // of accessing the same data. See [New1DViewOf] for example. - View() Tensor - - // CopyFrom copies all values from other tensor into this tensor, with an - // optimized implementation if the other tensor is of the same type, and - // otherwise it goes through the appropriate standard type (Float, Int, String). - CopyFrom(from Tensor) - - // AppendFrom appends all values from other tensor into this tensor, with an - // optimized implementation if the other tensor is of the same type, and - // otherwise it goes through the appropriate standard type (Float, Int, String). - AppendFrom(from Tensor) error - - // CopyCellsFrom copies given range of values from other tensor into this tensor, - // using flat 1D indexes: to = starting index in this Tensor to start copying into, - // start = starting index on from Tensor to start copying from, and n = number of - // values to copy. Uses an optimized implementation if the other tensor is - // of the same type, and otherwise it goes through appropriate standard type. - CopyCellsFrom(from Tensor, to, start, n int) - - // SetNumRows sets the number of rows (outermost dimension). - // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) - // it is best to first set the anticipated full size, which allocates the - // full amount of memory, and then set to 0 and grow incrementally. - SetNumRows(rows int) -} - -// New returns a new n-dimensional tensor of given value type -// with the given sizes per dimension (shape). -func New[T DataTypes](sizes ...int) Tensor { - var v T - switch any(v).(type) { - case string: - return NewString(sizes...) - case bool: - return NewBits(sizes...) - case float64: - return NewNumber[float64](sizes...) - case float32: - return NewNumber[float32](sizes...) - case int: - return NewNumber[int](sizes...) - case int32: - return NewNumber[int32](sizes...) - case byte: - return NewNumber[byte](sizes...) - default: - panic("tensor.New: unexpected error: type not supported") - } } - -// NewOfType returns a new n-dimensional tensor of given reflect.Kind type -// with the given sizes per dimension (shape). -// Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. -func NewOfType(typ reflect.Kind, sizes ...int) Tensor { - switch typ { - case reflect.String: - return NewString(sizes...) - case reflect.Bool: - return NewBits(sizes...) - case reflect.Float64: - return NewNumber[float64](sizes...) - case reflect.Float32: - return NewNumber[float32](sizes...) - case reflect.Int: - return NewNumber[int](sizes...) - case reflect.Int32: - return NewNumber[int32](sizes...) - case reflect.Uint8: - return NewNumber[byte](sizes...) - default: - panic(fmt.Sprintf("tensor.NewOfType: type not supported: %v", typ)) - } -} - -// SetShapeFrom sets shape of given tensor from a source tensor. -func SetShapeFrom(tsr, from Tensor) { - tsr.SetShapeInts(from.ShapeInts()...) -} - -// metadata helpers - -// SetShapeNames sets the tensor shape dimension names into given metadata. -func SetShapeNames(md *metadata.Data, names ...string) { - md.Set("ShapeNames", names) -} - -// ShapeNames gets the tensor shape dimension names from given metadata. -func ShapeNames(md *metadata.Data) []string { - return errors.Log1(metadata.Get[[]string](*md, "ShapeNames")) -} - -// List is an ordered list of Tensors with name lookup. -type List = keylist.List[string, Tensor] diff --git a/tensor/values.go b/tensor/values.go new file mode 100644 index 0000000000..a0b22ec7f8 --- /dev/null +++ b/tensor/values.go @@ -0,0 +1,135 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "fmt" + "reflect" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/metadata" +) + +// Values is an extended [Tensor] interface for raw value tensors. +// This supports direct setting of the shape of the underlying values, +// sub-space access to inner-dimensional subspaces of values, etc. +type Values interface { + RowCell + + // SetShape sets the dimension sizes as 1D int values from given tensor. + // The backing storage is resized appropriately, retaining all existing data that fits. + SetShape(sizes Tensor) + + // SetShapeInts sets the dimension sizes of the tensor, and resizes + // backing storage appropriately, retaining all existing data that fits. + SetShapeInts(sizes ...int) + + // SetNumRows sets the number of rows (outermost dimension). + // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) + // it is best to first set the anticipated full size, which allocates the + // full amount of memory, and then set to 0 and grow incrementally. + SetNumRows(rows int) + + // Sizeof returns the number of bytes contained in the Values of this tensor. + // For String types, this is just the string pointers, not the string content. + Sizeof() int64 + + // Bytes returns the underlying byte representation of the tensor values. + // This is the actual underlying data, so make a copy if it can be + // unintentionally modified or retained more than for immediate use. + Bytes() []byte + + // SetZeros is a convenience function initialize all values to the + // zero value of the type (empty strings for string type). + // New tensors always start out with zeros. + SetZeros() + + // View clones this tensor, *keeping the same underlying Values slice*, + // instead of making a copy like [Values.Clone] does. A major use of this + // is to then change the shape of the view to provide a different way + // of accessing the same data. See [New1DViewOf] for example. + View() Values + + // Clone clones this tensor, creating a duplicate copy of itself with its + // own separate memory representation of all the values. + Clone() Values + + // CopyFrom copies all values from other tensor into this tensor, with an + // optimized implementation if the other tensor is of the same type, and + // otherwise it goes through the appropriate standard type (Float, Int, String). + CopyFrom(from Values) + + // CopyCellsFrom copies given range of values from other tensor into this tensor, + // using flat 1D indexes: to = starting index in this Tensor to start copying into, + // start = starting index on from Tensor to start copying from, and n = number of + // values to copy. Uses an optimized implementation if the other tensor is + // of the same type, and otherwise it goes through appropriate standard type. + CopyCellsFrom(from Values, to, start, n int) + + // AppendFrom appends all values from other tensor into this tensor, with an + // optimized implementation if the other tensor is of the same type, and + // otherwise it goes through the appropriate standard type (Float, Int, String). + AppendFrom(from Values) error +} + +// New returns a new n-dimensional tensor of given value type +// with the given sizes per dimension (shape). +func New[T DataTypes](sizes ...int) Values { + var v T + switch any(v).(type) { + case string: + return NewString(sizes...) + case bool: + return NewBits(sizes...) + case float64: + return NewNumber[float64](sizes...) + case float32: + return NewNumber[float32](sizes...) + case int: + return NewNumber[int](sizes...) + case int32: + return NewNumber[int32](sizes...) + case byte: + return NewNumber[byte](sizes...) + default: + panic("tensor.New: unexpected error: type not supported") + } +} + +// NewOfType returns a new n-dimensional tensor of given reflect.Kind type +// with the given sizes per dimension (shape). +// Supported types are in [DataTypes]. +func NewOfType(typ reflect.Kind, sizes ...int) Values { + switch typ { + case reflect.String: + return NewString(sizes...) + case reflect.Bool: + return NewBits(sizes...) + case reflect.Float64: + return NewNumber[float64](sizes...) + case reflect.Float32: + return NewNumber[float32](sizes...) + case reflect.Int: + return NewNumber[int](sizes...) + case reflect.Int32: + return NewNumber[int32](sizes...) + case reflect.Uint8: + return NewNumber[byte](sizes...) + default: + panic(fmt.Sprintf("tensor.NewOfType: type not supported: %v", typ)) + } +} + +// metadata helpers + +// SetShapeNames sets the tensor shape dimension names into given metadata. +func SetShapeNames(md *metadata.Data, names ...string) { + md.Set("ShapeNames", names) +} + +// ShapeNames gets the tensor shape dimension names from given metadata. +func ShapeNames(md *metadata.Data) []string { + return errors.Log1(metadata.Get[[]string](*md, "ShapeNames")) +} From bcb1a65ff7c3839cb618b1717af39722573f53cc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 17:37:14 -0700 Subject: [PATCH 089/311] renamed RowCell -> RowMajor to better reflect its constraint. --- tensor/convert.go | 2 +- tensor/{rowcell.go => rowmajor.go} | 11 ++++++----- tensor/rows.go | 2 +- tensor/values.go | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) rename tensor/{rowcell.go => rowmajor.go} (93%) diff --git a/tensor/convert.go b/tensor/convert.go index d397510f15..afb0567d89 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -42,7 +42,7 @@ func New1DViewOf(tsr Values) Values { // Cells1D returns a flat 1D [Values] view of the cells for given row index. // This is useful for passing to other functions e.g., // in stats or metrics that process a 1D tensor. -func Cells1D(tsr RowCell, row int) Values { +func Cells1D(tsr RowMajor, row int) Values { return New1DViewOf(tsr.SubSpace(row)) } diff --git a/tensor/rowcell.go b/tensor/rowmajor.go similarity index 93% rename from tensor/rowcell.go rename to tensor/rowmajor.go index 7ce89c7c48..e254685c58 100644 --- a/tensor/rowcell.go +++ b/tensor/rowmajor.go @@ -4,13 +4,14 @@ package tensor -// RowCell is subclass of [Tensor] that supports efficient access via the -// outermost 'row' dimension, with all remaining inner dimensions comprising the -// 'cells' of data per row (1 scalar value in the case of a 1D tensor). +// RowMajor is subclass of [Tensor] that maintains a row major memory organization +// that thereby supports efficient access via the outermost 'row' dimension, +// with all remaining inner dimensions comprising the 'cells' of data per row +// (1 scalar value in the case of a 1D tensor). // It is implemented by raw [Values] tensors, and the [Rows] indexed view of // raw Values tensors. Other views however do not retain the underlying -// outer to inner memory structure and thus do not benefit from this interface. -type RowCell interface { +// outer to inner row major memory structure and thus do not benefit from this interface. +type RowMajor interface { Tensor // RowCellSize returns the size of the outermost Row shape dimension, diff --git a/tensor/rows.go b/tensor/rows.go index c7f67d8adb..edf57fb9ef 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -681,4 +681,4 @@ func (rw *Rows) SetRowTensor(val Values, row int) { } // check for interface impl -var _ RowCell = (*Rows)(nil) +var _ RowMajor = (*Rows)(nil) diff --git a/tensor/values.go b/tensor/values.go index a0b22ec7f8..5ca7b76aaa 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -16,7 +16,7 @@ import ( // This supports direct setting of the shape of the underlying values, // sub-space access to inner-dimensional subspaces of values, etc. type Values interface { - RowCell + RowMajor // SetShape sets the dimension sizes as 1D int values from given tensor. // The backing storage is resized appropriately, retaining all existing data that fits. From e7f3ed5aee5b8852627700c7ea423eff10882dd2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 21 Sep 2024 21:46:09 -0700 Subject: [PATCH 090/311] Bits -> Bool --- tensor/base.go | 2 +- tensor/bool.go | 140 ++++++++++++++++++------------------ tensor/datafs/data.go | 2 +- tensor/table/table.go | 2 +- tensor/tensor.go | 2 +- tensor/tensor_test.go | 2 +- tensor/tensorcore/values.go | 2 +- tensor/tensormpi/tensor.go | 4 +- tensor/values.go | 4 +- 9 files changed, 80 insertions(+), 80 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index 081f36b396..ab4e373700 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -69,7 +69,7 @@ func (tsr *Base[T]) RowCellSize() (rows, cells int) { } // DataType returns the type of the data elements in the tensor. -// Bool is returned for the Bits tensor type. +// Bool is returned for the Bool tensor type. func (tsr *Base[T]) DataType() reflect.Kind { var v T return reflect.TypeOf(v).Kind() diff --git a/tensor/bool.go b/tensor/bool.go index c173c03520..238de76a3d 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -15,28 +15,28 @@ import ( "cogentcore.org/core/tensor/bitslice" ) -// Bits is a tensor of bits backed by a [bitslice.Slice] for efficient storage -// of binary data. Bits does not support [Tensor.SubSpace] access and related +// Bool is a tensor of bits backed by a [bitslice.Slice] for efficient storage +// of binary data. Bool does not support [Tensor.SubSpace] access and related // methods due to the nature of the underlying data representation. -type Bits struct { +type Bool struct { shape Shape Values bitslice.Slice Meta metadata.Data } -// NewBits returns a new n-dimensional tensor of bit values +// NewBool returns a new n-dimensional tensor of bit values // with the given sizes per dimension (shape), and optional dimension names. -func NewBits(sizes ...int) *Bits { - tsr := &Bits{} +func NewBool(sizes ...int) *Bool { + tsr := &Bool{} tsr.SetShapeInts(sizes...) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr } -// NewBitsShape returns a new n-dimensional tensor of bit values +// NewBoolShape returns a new n-dimensional tensor of bit values // using given shape. -func NewBitsShape(shape *Shape) *Bits { - tsr := &Bits{} +func NewBoolShape(shape *Shape) *Bool { + tsr := &Bool{} tsr.shape.CopyShape(shape) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr @@ -63,66 +63,66 @@ func BoolToInt(bv bool) int { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Bits) String() string { +func (tsr *Bool) String() string { return sprint(tsr, 0) } -func (tsr *Bits) IsString() bool { +func (tsr *Bool) IsString() bool { return false } -func (tsr *Bits) AsValues() Values { return tsr } +func (tsr *Bool) AsValues() Values { return tsr } // DataType returns the type of the data elements in the tensor. -// Bool is returned for the Bits tensor type. -func (tsr *Bits) DataType() reflect.Kind { +// Bool is returned for the Bool tensor type. +func (tsr *Bool) DataType() reflect.Kind { return reflect.Bool } -func (tsr *Bits) Sizeof() int64 { +func (tsr *Bool) Sizeof() int64 { return int64(len(tsr.Values)) } -func (tsr *Bits) Bytes() []byte { +func (tsr *Bool) Bytes() []byte { return slicesx.ToBytes(tsr.Values) } -func (tsr *Bits) Shape() *Shape { return &tsr.shape } +func (tsr *Bool) Shape() *Shape { return &tsr.shape } // ShapeSizes returns the sizes of each dimension as an int tensor. -func (tsr *Bits) ShapeSizes() Tensor { return tsr.shape.AsTensor() } +func (tsr *Bool) ShapeSizes() Tensor { return tsr.shape.AsTensor() } // ShapeInts returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. -func (tsr *Bits) ShapeInts() []int { return tsr.shape.Sizes } +func (tsr *Bool) ShapeInts() []int { return tsr.shape.Sizes } // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. -func (tsr *Bits) Metadata() *metadata.Data { return &tsr.Meta } +func (tsr *Bool) Metadata() *metadata.Data { return &tsr.Meta } // Len returns the number of elements in the tensor (product of shape dimensions). -func (tsr *Bits) Len() int { return tsr.shape.Len() } +func (tsr *Bool) Len() int { return tsr.shape.Len() } // NumDims returns the total number of dimensions. -func (tsr *Bits) NumDims() int { return tsr.shape.NumDims() } +func (tsr *Bool) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension -func (tsr *Bits) DimSize(dim int) int { return tsr.shape.DimSize(dim) } +func (tsr *Bool) DimSize(dim int) int { return tsr.shape.DimSize(dim) } // RowCellSize returns the size of the outermost Row shape dimension, // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. -func (tsr *Bits) RowCellSize() (rows, cells int) { +func (tsr *Bool) RowCellSize() (rows, cells int) { return tsr.shape.RowCellSize() } -func (tsr *Bits) SetShape(sizes Tensor) { +func (tsr *Bool) SetShape(sizes Tensor) { tsr.shape.SetShape(sizes) nln := tsr.Len() tsr.Values.SetLen(nln) } -func (tsr *Bits) SetShapeInts(sizes ...int) { +func (tsr *Bool) SetShapeInts(sizes ...int) { tsr.shape.SetShapeInts(sizes...) nln := tsr.Len() tsr.Values.SetLen(nln) @@ -132,149 +132,149 @@ func (tsr *Bits) SetShapeInts(sizes ...int) { // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) // it is best to first set the anticipated full size, which allocates the // full amount of memory, and then set to 0 and grow incrementally. -func (tsr *Bits) SetNumRows(rows int) { +func (tsr *Bool) SetNumRows(rows int) { _, cells := tsr.shape.RowCellSize() nln := rows * cells tsr.shape.Sizes[0] = rows tsr.Values.SetLen(nln) } -// SubSpace is not possible with Bits. -func (tsr *Bits) SubSpace(offs ...int) Values { +// SubSpace is not possible with Bool. +func (tsr *Bool) SubSpace(offs ...int) Values { return nil } -// RowTensor not possible with Bits. -func (tsr *Bits) RowTensor(row int) Values { +// RowTensor not possible with Bool. +func (tsr *Bool) RowTensor(row int) Values { return nil } -// SetRowTensor not possible with Bits. -func (tsr *Bits) SetRowTensor(val Values, row int) { +// SetRowTensor not possible with Bool. +func (tsr *Bool) SetRowTensor(val Values, row int) { } ///////////////////// Strings -func (tsr *Bits) String1D(off int) string { +func (tsr *Bool) String1D(off int) string { return reflectx.ToString(tsr.Values.Index(off)) } -func (tsr *Bits) SetString1D(val string, off int) { +func (tsr *Bool) SetString1D(val string, off int) { if bv, err := reflectx.ToBool(val); err == nil { tsr.Values.Set(bv, off) } } -func (tsr *Bits) StringValue(i ...int) string { +func (tsr *Bool) StringValue(i ...int) string { return reflectx.ToString(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) } -func (tsr *Bits) SetString(val string, i ...int) { +func (tsr *Bool) SetString(val string, i ...int) { if bv, err := reflectx.ToBool(val); err == nil { tsr.Values.Set(bv, tsr.shape.IndexTo1D(i...)) } } -func (tsr *Bits) StringRowCell(row, cell int) string { +func (tsr *Bool) StringRowCell(row, cell int) string { _, sz := tsr.RowCellSize() return reflectx.ToString(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bits) SetStringRowCell(val string, row, cell int) { +func (tsr *Bool) SetStringRowCell(val string, row, cell int) { if bv, err := reflectx.ToBool(val); err == nil { _, sz := tsr.RowCellSize() tsr.Values.Set(bv, row*sz+cell) } } -func (tsr *Bits) StringRow(row int) string { +func (tsr *Bool) StringRow(row int) string { return tsr.StringRowCell(row, 0) } -func (tsr *Bits) SetStringRow(val string, row int) { +func (tsr *Bool) SetStringRow(val string, row int) { tsr.SetStringRowCell(val, row, 0) } ///////////////////// Floats -func (tsr *Bits) Float(i ...int) float64 { +func (tsr *Bool) Float(i ...int) float64 { return BoolToFloat64(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) } -func (tsr *Bits) SetFloat(val float64, i ...int) { +func (tsr *Bool) SetFloat(val float64, i ...int) { tsr.Values.Set(Float64ToBool(val), tsr.shape.IndexTo1D(i...)) } -func (tsr *Bits) Float1D(off int) float64 { +func (tsr *Bool) Float1D(off int) float64 { return BoolToFloat64(tsr.Values.Index(off)) } -func (tsr *Bits) SetFloat1D(val float64, off int) { +func (tsr *Bool) SetFloat1D(val float64, off int) { tsr.Values.Set(Float64ToBool(val), off) } -func (tsr *Bits) FloatRowCell(row, cell int) float64 { +func (tsr *Bool) FloatRowCell(row, cell int) float64 { _, sz := tsr.RowCellSize() return BoolToFloat64(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bits) SetFloatRowCell(val float64, row, cell int) { +func (tsr *Bool) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.RowCellSize() tsr.Values.Set(Float64ToBool(val), row*sz+cell) } -func (tsr *Bits) FloatRow(row int) float64 { +func (tsr *Bool) FloatRow(row int) float64 { return tsr.FloatRowCell(row, 0) } -func (tsr *Bits) SetFloatRow(val float64, row int) { +func (tsr *Bool) SetFloatRow(val float64, row int) { tsr.SetFloatRowCell(val, row, 0) } ///////////////////// Ints -func (tsr *Bits) Int(i ...int) int { +func (tsr *Bool) Int(i ...int) int { return BoolToInt(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) } -func (tsr *Bits) SetInt(val int, i ...int) { +func (tsr *Bool) SetInt(val int, i ...int) { tsr.Values.Set(IntToBool(val), tsr.shape.IndexTo1D(i...)) } -func (tsr *Bits) Int1D(off int) int { +func (tsr *Bool) Int1D(off int) int { return BoolToInt(tsr.Values.Index(off)) } -func (tsr *Bits) SetInt1D(val int, off int) { +func (tsr *Bool) SetInt1D(val int, off int) { tsr.Values.Set(IntToBool(val), off) } -func (tsr *Bits) IntRowCell(row, cell int) int { +func (tsr *Bool) IntRowCell(row, cell int) int { _, sz := tsr.RowCellSize() return BoolToInt(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bits) SetIntRowCell(val int, row, cell int) { +func (tsr *Bool) SetIntRowCell(val int, row, cell int) { _, sz := tsr.RowCellSize() tsr.Values.Set(IntToBool(val), row*sz+cell) } -func (tsr *Bits) IntRow(row int) int { +func (tsr *Bool) IntRow(row int) int { return tsr.IntRowCell(row, 0) } -func (tsr *Bits) SetIntRow(val int, row int) { +func (tsr *Bool) SetIntRow(val int, row int) { tsr.SetIntRowCell(val, row, 0) } // Label satisfies the core.Labeler interface for a summary description of the tensor -func (tsr *Bits) Label() string { - return fmt.Sprintf("tensor.Bits: %s", tsr.shape.String()) +func (tsr *Bool) Label() string { + return fmt.Sprintf("tensor.Bool: %s", tsr.shape.String()) } // SetZeros is simple convenience function initialize all values to 0 -func (tsr *Bits) SetZeros() { +func (tsr *Bool) SetZeros() { ln := tsr.Len() for j := 0; j < ln; j++ { tsr.Values.Set(false, j) @@ -284,14 +284,14 @@ func (tsr *Bits) SetZeros() { // Clone clones this tensor, creating a duplicate copy of itself with its // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). -func (tsr *Bits) Clone() Values { - csr := NewBitsShape(&tsr.shape) +func (tsr *Bool) Clone() Values { + csr := NewBoolShape(&tsr.shape) csr.Values = tsr.Values.Clone() return csr } -func (tsr *Bits) View() Values { - nw := &Bits{} +func (tsr *Bool) View() Values { + nw := &Bool{} nw.shape.CopyShape(&tsr.shape) nw.Values = tsr.Values nw.Meta = tsr.Meta @@ -301,8 +301,8 @@ func (tsr *Bits) View() Values { // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. -func (tsr *Bits) CopyFrom(frm Values) { - if fsm, ok := frm.(*Bits); ok { +func (tsr *Bool) CopyFrom(frm Values) { + if fsm, ok := frm.(*Bool); ok { copy(tsr.Values, fsm.Values) return } @@ -317,7 +317,7 @@ func (tsr *Bits) CopyFrom(frm Values) { // It uses and optimized implementation if the other tensor // is of the same type, and otherwise it goes through // appropriate standard type. -func (tsr *Bits) AppendFrom(frm Values) error { +func (tsr *Bool) AppendFrom(frm Values) error { rows, cell := tsr.RowCellSize() frows, fcell := frm.RowCellSize() if cell != fcell { @@ -326,7 +326,7 @@ func (tsr *Bits) AppendFrom(frm Values) error { tsr.SetNumRows(rows + frows) st := rows * cell fsz := frows * fcell - if fsm, ok := frm.(*Bits); ok { + if fsm, ok := frm.(*Bool); ok { copy(tsr.Values[st:st+fsz], fsm.Values) return nil } @@ -341,8 +341,8 @@ func (tsr *Bits) AppendFrom(frm Values) error { // start = starting index on from Tensor to start copying from, and n = number of // values to copy. Uses an optimized implementation if the other tensor is // of the same type, and otherwise it goes through appropriate standard type. -func (tsr *Bits) CopyCellsFrom(frm Values, to, start, n int) { - if fsm, ok := frm.(*Bits); ok { +func (tsr *Bool) CopyCellsFrom(frm Values, to, start, n int) { + if fsm, ok := frm.(*Bool); ok { for i := 0; i < n; i++ { tsr.Values.Set(fsm.Values.Index(start+i), to+i) } diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 87a2d52538..5eb18c67eb 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -94,7 +94,7 @@ func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.V // NewOfType returns a new Data value as a [tensor.Tensor] // of given reflect.Kind type and shape sizes per dimension, in given directory Data item. -// Supported types are string, bool (for [Bits]), float32, float64, int, int32, and byte. +// Supported types are string, bool (for [Bool]), float32, float64, int, int32, and byte. // The name must be unique in the directory. func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Values { tsr := tensor.NewOfType(typ, sizes...) diff --git a/tensor/table/table.go b/tensor/table/table.go index 1afb83fbec..f7ec5c6262 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -188,7 +188,7 @@ func (dt *Table) InsertColumn(idx int, name string, tsr tensor.Values) error { // column name (which must be unique), // If no cellSizes are specified, it holds scalar values, // otherwise the cells are n-dimensional tensors of given size. -// Supported types include string, bool (for [tensor.Bits]), float32, float64, int, int32, and byte. +// Supported types include string, bool (for [tensor.Bool]), float32, float64, int, int32, and byte. func (dt *Table) AddColumnOfType(name string, typ reflect.Kind, cellSizes ...int) tensor.Tensor { rows := dt.Columns.Rows sz := append([]int{rows}, cellSizes...) diff --git a/tensor/tensor.go b/tensor/tensor.go index 425e1132fc..56e89f4195 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -81,7 +81,7 @@ type Tensor interface { DimSize(dim int) int // DataType returns the type of the data elements in the tensor. - // Bool is returned for the Bits tensor type. + // Bool is returned for the Bool tensor type. DataType() reflect.Kind // IsString returns true if the data type is a String; otherwise it is numeric. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 0d83848d07..fbe64bdd6b 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -211,7 +211,7 @@ func TestGrowRow(t *testing.T) { assert.Equal(t, 100000, cap(tsr2.Values)) assert.Equal(t, 100, tsr2.Len()) - bits := NewBits(1000) + bits := NewBool(1000) assert.Equal(t, 1000, bits.Len()) bits.SetNumRows(0) assert.Equal(t, 0, bits.Len()) diff --git a/tensor/tensorcore/values.go b/tensor/tensorcore/values.go index de27c12fdd..7f77470b97 100644 --- a/tensor/tensorcore/values.go +++ b/tensor/tensorcore/values.go @@ -20,7 +20,7 @@ func init() { core.AddValueType[tensor.Int32, TensorButton]() core.AddValueType[tensor.Byte, TensorButton]() core.AddValueType[tensor.String, TensorButton]() - core.AddValueType[tensor.Bits, TensorButton]() + core.AddValueType[tensor.Bool, TensorButton]() // core.AddValueType[simat.SimMat, SimMatButton]() } diff --git a/tensor/tensormpi/tensor.go b/tensor/tensormpi/tensor.go index 96f573b59d..f4d95a3d16 100644 --- a/tensor/tensormpi/tensor.go +++ b/tensor/tensormpi/tensor.go @@ -124,8 +124,8 @@ func ReduceTensor(dest, src tensor.Tensor, comm *mpi.Comm, op mpi.Op) error { var err error switch dt { case reflect.Bool: - dt := dest.(*tensor.Bits) - st := src.(*tensor.Bits) + dt := dest.(*tensor.Bool) + st := src.(*tensor.Bool) err = comm.AllReduceU8(op, dt.Values, st.Values) case reflect.Uint8: dt := dest.(*tensor.Byte) diff --git a/tensor/values.go b/tensor/values.go index 5ca7b76aaa..5632d306d8 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -82,7 +82,7 @@ func New[T DataTypes](sizes ...int) Values { case string: return NewString(sizes...) case bool: - return NewBits(sizes...) + return NewBool(sizes...) case float64: return NewNumber[float64](sizes...) case float32: @@ -106,7 +106,7 @@ func NewOfType(typ reflect.Kind, sizes ...int) Values { case reflect.String: return NewString(sizes...) case reflect.Bool: - return NewBits(sizes...) + return NewBool(sizes...) case reflect.Float64: return NewNumber[float64](sizes...) case reflect.Float32: From e4e781f7edee7f16c7f9b9dd2392ac6ad32e53c7 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 00:10:38 -0700 Subject: [PATCH 091/311] masked working --- tensor/base.go | 8 +- tensor/bool.go | 70 ++++++------ tensor/masked.go | 244 ++++++++++++++++++++++++++++++++++++++++++ tensor/rows.go | 2 +- tensor/shape.go | 2 +- tensor/sliced.go | 22 ++-- tensor/tensor.go | 2 +- tensor/tensor_test.go | 37 +++++++ tensor/typegen.go | 6 +- 9 files changed, 346 insertions(+), 47 deletions(-) create mode 100644 tensor/masked.go diff --git a/tensor/base.go b/tensor/base.go index ab4e373700..7230fc7d7f 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -30,7 +30,7 @@ func (tsr *Base[T]) Metadata() *metadata.Data { return &tsr.Meta } func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } // ShapeSizes returns the sizes of each dimension as an int tensor. -func (tsr *Base[T]) ShapeSizes() Tensor { return tsr.shape.AsTensor() } +func (tsr *Base[T]) ShapeSizes() *Int { return tsr.shape.AsTensor() } // ShapeInts returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. @@ -180,8 +180,10 @@ func sprint(tsr Tensor, maxLen int) string { var b strings.Builder sh := tsr.Shape() b.WriteString(tsr.Label()) + noidx := false if tsr.NumDims() == 1 && tsr.Len() < 8 { b.WriteString(" ") + noidx = true } else { b.WriteString("\n") } @@ -190,7 +192,9 @@ func sprint(tsr Tensor, maxLen int) string { ctr := 0 for r := range rows { rc, _ := Projection2DCoords(sh, oddRow, r, 0) - b.WriteString(fmt.Sprintf("%v: ", rc)) + if !noidx { + b.WriteString(fmt.Sprintf("%v: ", rc)) + } ri := r for c := 0; c < cols; c++ { if tsr.IsString() { diff --git a/tensor/bool.go b/tensor/bool.go index 238de76a3d..8109b586b5 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -16,8 +16,8 @@ import ( ) // Bool is a tensor of bits backed by a [bitslice.Slice] for efficient storage -// of binary data. Bool does not support [Tensor.SubSpace] access and related -// methods due to the nature of the underlying data representation. +// of binary, boolean data. Bool does not support [Tensor.SubSpace] access +// and related methods due to the nature of the underlying data representation. type Bool struct { shape Shape Values bitslice.Slice @@ -63,34 +63,29 @@ func BoolToInt(bv bool) int { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Bool) String() string { - return sprint(tsr, 0) -} +func (tsr *Bool) String() string { return sprint(tsr, 0) } -func (tsr *Bool) IsString() bool { - return false +// Label satisfies the core.Labeler interface for a summary description of the tensor +func (tsr *Bool) Label() string { + return label(tsr.Metadata().Name(), tsr.Shape()) } +func (tsr *Bool) IsString() bool { return false } + func (tsr *Bool) AsValues() Values { return tsr } // DataType returns the type of the data elements in the tensor. // Bool is returned for the Bool tensor type. -func (tsr *Bool) DataType() reflect.Kind { - return reflect.Bool -} +func (tsr *Bool) DataType() reflect.Kind { return reflect.Bool } -func (tsr *Bool) Sizeof() int64 { - return int64(len(tsr.Values)) -} +func (tsr *Bool) Sizeof() int64 { return int64(len(tsr.Values)) } -func (tsr *Bool) Bytes() []byte { - return slicesx.ToBytes(tsr.Values) -} +func (tsr *Bool) Bytes() []byte { return slicesx.ToBytes(tsr.Values) } func (tsr *Bool) Shape() *Shape { return &tsr.shape } // ShapeSizes returns the sizes of each dimension as an int tensor. -func (tsr *Bool) ShapeSizes() Tensor { return tsr.shape.AsTensor() } +func (tsr *Bool) ShapeSizes() *Int { return tsr.shape.AsTensor() } // ShapeInts returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. @@ -140,19 +135,13 @@ func (tsr *Bool) SetNumRows(rows int) { } // SubSpace is not possible with Bool. -func (tsr *Bool) SubSpace(offs ...int) Values { - return nil -} +func (tsr *Bool) SubSpace(offs ...int) Values { return nil } // RowTensor not possible with Bool. -func (tsr *Bool) RowTensor(row int) Values { - return nil -} +func (tsr *Bool) RowTensor(row int) Values { return nil } // SetRowTensor not possible with Bool. -func (tsr *Bool) SetRowTensor(val Values, row int) { - -} +func (tsr *Bool) SetRowTensor(val Values, row int) {} ///////////////////// Strings @@ -268,12 +257,25 @@ func (tsr *Bool) SetIntRow(val int, row int) { tsr.SetIntRowCell(val, row, 0) } -// Label satisfies the core.Labeler interface for a summary description of the tensor -func (tsr *Bool) Label() string { - return fmt.Sprintf("tensor.Bool: %s", tsr.shape.String()) +///////////////////// Bools + +func (tsr *Bool) Bool(i ...int) bool { + return tsr.Values.Index(tsr.shape.IndexTo1D(i...)) +} + +func (tsr *Bool) SetBool(val bool, i ...int) { + tsr.Values.Set(val, tsr.shape.IndexTo1D(i...)) } -// SetZeros is simple convenience function initialize all values to 0 +func (tsr *Bool) Bool1D(off int) bool { + return tsr.Values.Index(off) +} + +func (tsr *Bool) SetBool1D(val bool, off int) { + tsr.Values.Set(val, off) +} + +// SetZeros is a convenience function initialize all values to 0 (false). func (tsr *Bool) SetZeros() { ln := tsr.Len() for j := 0; j < ln; j++ { @@ -281,6 +283,14 @@ func (tsr *Bool) SetZeros() { } } +// SetTrue is simple convenience function initialize all values to 0 +func (tsr *Bool) SetTrue() { + ln := tsr.Len() + for j := 0; j < ln; j++ { + tsr.Values.Set(true, j) + } +} + // Clone clones this tensor, creating a duplicate copy of itself with its // own separate memory representation of all the values, and returns // that as a Tensor (which can be converted into the known type as needed). diff --git a/tensor/masked.go b/tensor/masked.go new file mode 100644 index 0000000000..4b5f3c7b01 --- /dev/null +++ b/tensor/masked.go @@ -0,0 +1,244 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "math" + "reflect" + + "cogentcore.org/core/base/metadata" + "cogentcore.org/core/base/reflectx" +) + +// Masked is a wrapper around another [Tensor] that provides a +// bit-masked view onto the Tensor defined by a [Bool] [Values] +// tensor with a matching shape. If the bool mask has a 'false' +// then the corresponding value cannot be set and Float access returns +// NaN indicating missing data. +// To produce a new [Values] tensor with only the 'true' cases, +// (i.e., the copy function of numpy), call [Masked.CloneValues]. +type Masked struct { //types:add + + // Tensor that we are a masked view onto. + Tensor Tensor + + // Bool tensor with same shape as source tensor, providing mask. + Mask *Bool +} + +// NewMasked returns a new [Masked] view of given tensor, +// with given [Bool] mask values. +func NewMasked(tsr Tensor, mask ...*Bool) *Masked { + ms := &Masked{Tensor: tsr} + if len(mask) == 1 { + ms.Mask = mask[0] + ms.SyncShape() + } else { + ms.Mask = NewBoolShape(tsr.Shape()) + ms.Mask.SetTrue() + } + return ms +} + +// AsMasked returns the tensor as a [Masked] view. +// If it already is one, then it is returned, otherwise it is wrapped +// with an initially transparent mask. +func AsMasked(tsr Tensor) *Masked { + if ms, ok := tsr.(*Masked); ok { + return ms + } + return NewMasked(tsr) +} + +// SetTensor sets as indexes into given tensor with sequential initial indexes. +func (ms *Masked) SetTensor(tsr Tensor) { + ms.Tensor = tsr + ms.SyncShape() +} + +// SyncShape ensures that [Masked.Mask] shape is the same as source tensor. +func (ms *Masked) SyncShape() { + if ms.Mask == nil { + ms.Mask = NewBoolShape(ms.Tensor.Shape()) + return + } + SetShapeFrom(ms.Mask, ms.Tensor) +} + +// Label satisfies the core.Labeler interface for a summary description of the tensor. +func (ms *Masked) Label() string { + return label(ms.Metadata().Name(), ms.Shape()) +} + +// String satisfies the fmt.Stringer interface for string of tensor data. +func (ms *Masked) String() string { return sprint(ms, 0) } + +// Metadata returns the metadata for this tensor, which can be used +// to encode plotting options, etc. +func (ms *Masked) Metadata() *metadata.Data { return ms.Tensor.Metadata() } + +func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } + +func (ms *Masked) DataType() reflect.Kind { return ms.Tensor.DataType() } + +func (ms *Masked) ShapeInts() []int { return ms.Tensor.ShapeInts() } + +func (ms *Masked) ShapeSizes() *Int { return ms.Tensor.ShapeSizes() } + +func (ms *Masked) Shape() *Shape { return ms.Tensor.Shape() } + +// Len returns the total number of elements in our view of the tensor. +func (ms *Masked) Len() int { return ms.Tensor.Len() } + +// NumDims returns the total number of dimensions. +func (ms *Masked) NumDims() int { return ms.Tensor.NumDims() } + +// DimSize returns the effective view size of given dimension. +func (ms *Masked) DimSize(dim int) int { return ms.Tensor.DimSize(dim) } + +// AsValues returns a copy of this tensor as raw [Values]. +// This "renders" the Masked view into a fully contiguous +// and optimized memory representation of that view. +// Because the masking pattern is unpredictable, only a 1D shape is possible. +func (ms *Masked) AsValues() Values { + dt := ms.Tensor.DataType() + n := ms.Len() + switch { + case ms.Tensor.IsString(): + vals := make([]string, 0, n) + for i := range n { + if !ms.Mask.Bool1D(i) { + continue + } + vals = append(vals, ms.Tensor.String1D(i)) + } + return NewStringFromSlice(vals...) + case reflectx.KindIsFloat(dt): + vals := make([]float64, 0, n) + for i := range n { + if !ms.Mask.Bool1D(i) { + continue + } + vals = append(vals, ms.Tensor.Float1D(i)) + } + return NewFloat64FromSlice(vals...) + default: + vals := make([]int, 0, n) + for i := range n { + if !ms.Mask.Bool1D(i) { + continue + } + vals = append(vals, ms.Tensor.Int1D(i)) + } + return NewIntFromSlice(vals...) + } +} + +/////////////////////////////////////////////// +// Masked access + +///////////////////// Floats + +func (ms *Masked) Float(i ...int) float64 { + if !ms.Mask.Bool(i...) { + return math.NaN() + } + return ms.Tensor.Float(i...) +} + +func (ms *Masked) SetFloat(val float64, i ...int) { + if !ms.Mask.Bool(i...) { + return + } + ms.Tensor.SetFloat(val, i...) +} + +func (ms *Masked) Float1D(i int) float64 { + if !ms.Mask.Bool1D(i) { + return math.NaN() + } + return ms.Tensor.Float1D(i) +} + +func (ms *Masked) SetFloat1D(val float64, i int) { + if !ms.Mask.Bool1D(i) { + return + } + ms.Tensor.SetFloat1D(val, i) +} + +///////////////////// Strings + +func (ms *Masked) StringValue(i ...int) string { + if !ms.Mask.Bool(i...) { + return "" + } + return ms.Tensor.StringValue(i...) +} + +func (ms *Masked) SetString(val string, i ...int) { + if !ms.Mask.Bool(i...) { + return + } + ms.Tensor.SetString(val, i...) +} + +func (ms *Masked) String1D(i int) string { + if !ms.Mask.Bool1D(i) { + return "" + } + return ms.Tensor.String1D(i) +} + +func (ms *Masked) SetString1D(val string, i int) { + if !ms.Mask.Bool1D(i) { + return + } + ms.Tensor.SetString1D(val, i) +} + +///////////////////// Ints + +func (ms *Masked) Int(i ...int) int { + if !ms.Mask.Bool(i...) { + return 0 + } + return ms.Tensor.Int(i...) +} + +func (ms *Masked) SetInt(val int, i ...int) { + if !ms.Mask.Bool(i...) { + return + } + ms.Tensor.SetInt(val, i...) +} + +func (ms *Masked) Int1D(i int) int { + if !ms.Mask.Bool1D(i) { + return 0 + } + return ms.Tensor.Int1D(i) +} + +// SetInt1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ms *Masked) SetInt1D(val int, i int) { + if !ms.Mask.Bool1D(i) { + return + } + ms.Tensor.SetInt1D(val, i) +} + +// Filter sets the mask values using given Filter function. +// The filter function gets the 1D index into the source tensor. +func (ms *Masked) Filter(filterer func(tsr Tensor, idx int) bool) { + n := ms.Tensor.Len() + for i := range n { + ms.Mask.SetBool1D(filterer(ms.Tensor, i), i) + } +} + +// check for interface impl +var _ Tensor = (*Masked)(nil) diff --git a/tensor/rows.go b/tensor/rows.go index edf57fb9ef..604f013315 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -108,7 +108,7 @@ func (rw *Rows) ShapeInts() []int { return sh } -func (rw *Rows) ShapeSizes() Tensor { +func (rw *Rows) ShapeSizes() *Int { if rw.Indexes == nil { return rw.Tensor.ShapeSizes() } diff --git a/tensor/shape.go b/tensor/shape.go index 20a6751f96..5b54ed522f 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -45,7 +45,7 @@ func (sh *Shape) SetShape(shp Tensor) { } // AsTensor returns shape sizes as an Int Tensor. -func (sh *Shape) AsTensor() Tensor { +func (sh *Shape) AsTensor() *Int { return NewIntFromSlice(sh.Sizes...) } diff --git a/tensor/sliced.go b/tensor/sliced.go index ac21c88656..eee15a714e 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -16,15 +16,15 @@ import ( ) // Sliced is a fully indexed wrapper around another [Tensor] that provides a -// re-sliced view onto the Tensor defined by the set of [SlicedIndexes], +// re-sliced view onto the Tensor defined by the set of [Sliced.Indexes], // for each dimension (must have at least 1 per dimension). // Thus, every dimension can be transformed in arbitrary ways relative // to the original tensor. There is some additional cost for every // access operation associated with the additional indexed indirection. // See also [Indexed] for a version that only indexes the outermost row dimension, // which is much more efficient for this common use-case. -// To produce a new [Tensor] that has its raw data actually organized according -// to the indexed order (i.e., the copy function of numpy), call [Sliced.NewTensor]. +// To produce a new concrete [Values] that has raw data actually organized according +// to the indexed order (i.e., the copy function of numpy), call [Sliced.CloneValues]. type Sliced struct { //types:add // Tensor that we are an indexed view onto. @@ -192,7 +192,7 @@ func (sl *Sliced) ShapeInts() []int { return sh } -func (sl *Sliced) ShapeSizes() Tensor { +func (sl *Sliced) ShapeSizes() *Int { return NewIntFromSlice(sl.ShapeInts()...) } @@ -295,13 +295,13 @@ func (sl *Sliced) SetFloat1D(val float64, i int) { ///////////////////// Strings // StringValue returns the value of given index as a string. -// The first index value is indirected through the indexes. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) StringValue(i ...int) string { return sl.Tensor.StringValue(sl.SliceIndexes(i...)...) } // SetString sets the value of given index as a string -// The first index value is indirected through the [Sliced.Indexes]. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) SetString(val string, i ...int) { sl.Tensor.SetString(val, sl.SliceIndexes(i...)...) } @@ -321,13 +321,13 @@ func (sl *Sliced) SetString1D(val string, i int) { ///////////////////// Ints // Int returns the value of given index as an int. -// The first index value is indirected through the indexes. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) Int(i ...int) int { return sl.Tensor.Int(sl.SliceIndexes(i...)...) } // SetInt sets the value of given index as an int -// The first index value is indirected through the [Sliced.Indexes]. +// The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) SetInt(val int, i ...int) { sl.Tensor.SetInt(val, sl.SliceIndexes(i...)...) } @@ -401,9 +401,9 @@ func (sl *Sliced) SortStableFunc(dim int, cmp func(tsr Tensor, dim, i, j int) in sl.Indexes[dim] = ix } -// Filter filters the indexes using given Filter function. -// The Filter function operates directly on row numbers into the Tensor -// as these row numbers have already been projected through the indexes. +// Filter filters the indexes using the given Filter function +// for setting the indexes for given dimension, and index into the +// source data. func (sl *Sliced) Filter(dim int, filterer func(tsr Tensor, dim, idx int) bool) { sl.IndexesNeeded(dim) ix := sl.Indexes[dim] diff --git a/tensor/tensor.go b/tensor/tensor.go index 56e89f4195..38329338ad 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -62,7 +62,7 @@ type Tensor interface { Shape() *Shape // ShapeSizes returns the sizes of each dimension as an int tensor. - ShapeSizes() Tensor + ShapeSizes() *Int // ShapeInts returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index fbe64bdd6b..1b57315e27 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -171,6 +171,43 @@ func TestSliced(t *testing.T) { } +func TestMasked(t *testing.T) { + ft := NewFloat64(3, 4) + for y := range 3 { + for x := range 4 { + v := y*10 + x + ft.SetFloat(float64(v), y, x) + } + } + ms := NewMasked(ft) + + res := `[3, 4] +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 +` + assert.Equal(t, res, ms.String()) + + ms.Filter(func(tsr Tensor, idx int) bool { + val := tsr.Float1D(idx) + return int(val)%10 == 2 + }) + res = `[3, 4] +[0]: NaN NaN 2 NaN +[1]: NaN NaN 12 NaN +[2]: NaN NaN 22 NaN +` + // fmt.Println(ms.String()) + assert.Equal(t, res, ms.String()) + + res = `[3] 2 12 22 +` + + vl := ms.AsValues() + // fmt.Println(vl.String()) + assert.Equal(t, res, vl.String()) +} + func TestSortFilter(t *testing.T) { tsr := NewRows(NewFloat64(5)) for i := range 5 { diff --git a/tensor/typegen.go b/tensor/typegen.go index e6bc08a311..e83773f93e 100644 --- a/tensor/typegen.go +++ b/tensor/typegen.go @@ -6,4 +6,8 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is an indexed wrapper around a tensor.Tensor that provides a\nspecific view onto the Tensor defined by the set of indexes, which\napply to the outermost row dimension (with default row-major indexing).\nThis is the universal representation of a homogenous data type in the\n[tensor] package framework, from scalar to vector, matrix, and beyond,\nbecause it can efficiently represent any kind of element with sufficient\nflexibility to enable a full range of computations to be elegantly expressed.\nFor example, sorting and filtering a tensor only requires\nupdating the indexes while doing nothing to the Tensor itself.\nTo produce a new [Tensor] that has its raw data actually organized according\nto the indexed order, call the [NewTensor] method.\nUse the [Set]FloatRowCell methods wherever possible, for the most efficient\nand natural indirection through the indexes. The 1D methods on underlying\ntensor data do not indirect through the indexes and must be called directly\non the [Tensor].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "IndexesNeeded", Doc: "IndexesNeeded is called prior to an operation that needs actual indexes,\ne.g., Sort, Filter. If Indexes == nil, they are set to all rows, otherwise\ncurrent indexes are left as is. Use Sequential, then IndexesNeeded to ensure\nall rows are represented.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing1D", Doc: "ExcludeMissing1D deletes indexes for a 1D tensor (only) where\nthe values are missing, as indicated by NaN.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows.\nOnly set if order is different from default sequential order.\nUse the Index() method for nil-aware logic."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Masked", IDName: "masked", Doc: "Masked is a wrapper around another [Tensor] that provides a\nbit-masked view onto the Tensor defined by\nfor each dimension (must have at least 1 per dimension).\nThus, every dimension can be transformed in arbitrary ways relative\nto the original tensor. There is some additional cost for every\naccess operation associated with the additional indexed indirection.\nSee also [Indexed] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case.\nTo produce a new [Tensor] that has its raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Masked.CloneValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Rows", IDName: "rows", Doc: "Rows is a row-indexed wrapper around a [Values] [Tensor] that provides a\nspecific view onto the Tensor defined by the set of [Rows.Indexes],\nwhich apply to the outermost row dimension (with default row-major indexing).\nSorting and filtering a tensor only requires updating the indexes while\nleaving the underlying Tensor alone.\nUse [CloneValues] to obtain a concrete [Values] representation with the current row\nsorting. Use the [Set]FloatRow[Cell] methods wherever possible,\nfor the most efficient and natural indirection through the indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing", Doc: "ExcludeMissing deletes indexes where the values are missing, as indicated by NaN.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values compared to given\nstring. Includes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse the named const args [Include], [Exclude], [Contains], [Equals],\n[IgnoreCase], [UseCase] for greater clarity.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"str", "exclude", "contains", "ignoreCase"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nUse the Index() method for nil-aware logic."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Sliced", IDName: "sliced", Doc: "Sliced is a fully indexed wrapper around another [Tensor] that provides a\nre-sliced view onto the Tensor defined by the set of [Sliced.Indexes],\nfor each dimension (must have at least 1 per dimension).\nThus, every dimension can be transformed in arbitrary ways relative\nto the original tensor. There is some additional cost for every\naccess operation associated with the additional indexed indirection.\nSee also [Indexed] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case.\nTo produce a new [Tensor] that has its raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Sliced.CloneValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) From 5b6c72d1c076bafbccbec68c8429f63409a73b46 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 01:55:58 -0700 Subject: [PATCH 092/311] remove redundant tensor version of ShapeSizes and rename Ints version to that; Indexed mostly implemented. --- tensor/base.go | 15 +- tensor/bool.go | 13 +- tensor/convert.go | 20 +-- tensor/examples/datafs-sim/sim.go | 4 +- tensor/indexed.go | 258 ++++++++++++++++++++++++++++++ tensor/masked.go | 6 +- tensor/matrix/matrix.go | 2 +- tensor/number.go | 4 +- tensor/rows.go | 19 +-- tensor/shape.go | 8 +- tensor/sliced.go | 72 ++++----- tensor/stats/glm/glm.go | 2 +- tensor/stats/metric/matrix.go | 18 +-- tensor/stats/metric/misc.go | 2 +- tensor/stats/metric/vec.go | 4 +- tensor/stats/stats/vec.go | 6 +- tensor/string.go | 2 +- tensor/table/io.go | 4 +- tensor/table/io_test.go | 6 +- tensor/tensor.go | 9 +- tensor/tensor_test.go | 4 +- tensor/tensorcore/table.go | 2 +- tensor/typegen.go | 8 +- tensor/values.go | 4 +- 24 files changed, 360 insertions(+), 132 deletions(-) create mode 100644 tensor/indexed.go diff --git a/tensor/base.go b/tensor/base.go index 7230fc7d7f..d1c074b0bb 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -29,12 +29,9 @@ func (tsr *Base[T]) Metadata() *metadata.Data { return &tsr.Meta } func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } -// ShapeSizes returns the sizes of each dimension as an int tensor. -func (tsr *Base[T]) ShapeSizes() *Int { return tsr.shape.AsTensor() } - -// ShapeInts returns the sizes of each dimension as a slice of ints. +// ShapeSizes returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. -func (tsr *Base[T]) ShapeInts() []int { return slices.Clone(tsr.shape.Sizes) } +func (tsr *Base[T]) ShapeSizes() []int { return slices.Clone(tsr.shape.Sizes) } // SetShape sets the dimension sizes as 1D int values from given tensor. // The backing storage is resized appropriately, retaining all existing data that fits. @@ -44,10 +41,10 @@ func (tsr *Base[T]) SetShape(sizes Tensor) { tsr.Values = slicesx.SetLength(tsr.Values, nln) } -// SetShapeInts sets the dimension sizes of the tensor, and resizes +// SetShapeSizes sets the dimension sizes of the tensor, and resizes // backing storage appropriately, retaining all existing data that fits. -func (tsr *Base[T]) SetShapeInts(sizes ...int) { - tsr.shape.SetShapeInts(sizes...) +func (tsr *Base[T]) SetShapeSizes(sizes ...int) { + tsr.shape.SetShapeSizes(sizes...) nln := tsr.Len() tsr.Values = slicesx.SetLength(tsr.Values, nln) } @@ -129,7 +126,7 @@ func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { return nil } stsr := &Base[T]{} - stsr.SetShapeInts(tsr.shape.Sizes[od:]...) + stsr.SetShapeSizes(tsr.shape.Sizes[od:]...) sti := make([]int, nd) copy(sti, offs) stoff := tsr.shape.IndexTo1D(sti...) diff --git a/tensor/bool.go b/tensor/bool.go index 8109b586b5..6d96fdb06f 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -28,7 +28,7 @@ type Bool struct { // with the given sizes per dimension (shape), and optional dimension names. func NewBool(sizes ...int) *Bool { tsr := &Bool{} - tsr.SetShapeInts(sizes...) + tsr.SetShapeSizes(sizes...) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr } @@ -84,12 +84,9 @@ func (tsr *Bool) Bytes() []byte { return slicesx.ToBytes(tsr.Values) } func (tsr *Bool) Shape() *Shape { return &tsr.shape } -// ShapeSizes returns the sizes of each dimension as an int tensor. -func (tsr *Bool) ShapeSizes() *Int { return tsr.shape.AsTensor() } - -// ShapeInts returns the sizes of each dimension as a slice of ints. +// ShapeSizes returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. -func (tsr *Bool) ShapeInts() []int { return tsr.shape.Sizes } +func (tsr *Bool) ShapeSizes() []int { return tsr.shape.Sizes } // Metadata returns the metadata for this tensor, which can be used // to encode plotting options, etc. @@ -117,8 +114,8 @@ func (tsr *Bool) SetShape(sizes Tensor) { tsr.Values.SetLen(nln) } -func (tsr *Bool) SetShapeInts(sizes ...int) { - tsr.shape.SetShapeInts(sizes...) +func (tsr *Bool) SetShapeSizes(sizes ...int) { + tsr.shape.SetShapeSizes(sizes...) nln := tsr.Len() tsr.Values.SetLen(nln) } diff --git a/tensor/convert.go b/tensor/convert.go index afb0567d89..2bca2e26da 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -8,7 +8,7 @@ import "cogentcore.org/core/base/errors" // Clone returns a copy of the given tensor. // If it is raw [Values] then a [Values.Clone] is returned. -// Otherwise if it is a view, then [Tensor.CloneValues] is returned. +// Otherwise if it is a view, then [Tensor.AsValues] is returned. func Clone(tsr Tensor) Values { if vl, ok := tsr.(Values); ok { return vl.Clone() @@ -25,7 +25,7 @@ func SetShapeFrom(tsr, from Tensor) error { if !ok { return errors.Log(errors.New("tensor.SetShapeFrom: tensor must be a Values type to have shape modified. All function output tensors must be Values!")) } - vl.SetShapeInts(from.ShapeInts()...) + vl.SetShapeSizes(from.ShapeSizes()...) return nil } @@ -35,7 +35,7 @@ func SetShapeFrom(tsr, from Tensor) error { // on the 1D list of values. func New1DViewOf(tsr Values) Values { vw := tsr.View() - vw.SetShapeInts(tsr.Len()) + vw.SetShapeSizes(tsr.Len()) return vw } @@ -52,7 +52,7 @@ func Cells1D(tsr RowMajor, row int) Values { // form the cells dimension. The resulting tensor is a re-shaped view // of the original tensor, sharing the same underlying data. func RowCellSplit(tsr Values, split int) Values { - sizes := tsr.ShapeInts() + sizes := tsr.ShapeSizes() rows := sizes[:split] cells := sizes[split:] nr := 1 @@ -64,7 +64,7 @@ func RowCellSplit(tsr Values, split int) Values { nc *= c } vw := tsr.View() - vw.SetShapeInts(nr, nc) + vw.SetShapeSizes(nr, nc) return vw } @@ -107,7 +107,7 @@ func NewStringFromSlice(vals ...string) *String { n := len(vals) tsr := &String{} tsr.Values = vals - tsr.SetShapeInts(n) + tsr.SetShapeSizes(n) return tsr } @@ -192,7 +192,7 @@ func AsFloat64Tensor(tsr Tensor) *Float64 { if f, ok := tsr.(*Float64); ok { return f } - f := NewFloat64(tsr.ShapeInts()...) + f := NewFloat64(tsr.ShapeSizes()...) f.CopyFrom(tsr.AsValues()) return f } @@ -204,7 +204,7 @@ func AsFloat32Tensor(tsr Tensor) *Float32 { if f, ok := tsr.(*Float32); ok { return f } - f := NewFloat32(AsIntSlice(tsr.ShapeSizes())...) + f := NewFloat32(tsr.ShapeSizes()...) f.CopyFrom(tsr.AsValues()) return f } @@ -218,7 +218,7 @@ func AsStringTensor(tsr Tensor) *String { if f, ok := tsr.(*String); ok { return f } - f := NewString(tsr.ShapeInts()...) + f := NewString(tsr.ShapeSizes()...) f.CopyFrom(tsr.AsValues()) return f } @@ -232,7 +232,7 @@ func AsIntTensor(tsr Tensor) *Int { if f, ok := tsr.(*Int); ok { return f } - f := NewInt(tsr.ShapeInts()...) + f := NewInt(tsr.ShapeSizes()...) f.CopyFrom(tsr.AsValues()) return f } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 4323532da6..5aef740d7e 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -83,11 +83,11 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { nm := st.Metadata().Name() // allocate full size lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) - lt.SetShapeInts(0) // then truncate to 0 + lt.SetShapeSizes(0) // then truncate to 0 lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source tensor.SetCalcFunc(lt, func() error { row := lt.DimSize(0) - lt.SetShapeInts(row + 1) + lt.SetShapeSizes(row + 1) if st.IsString() { lt.SetStringRow(st.StringRow(0), row) } else { diff --git a/tensor/indexed.go b/tensor/indexed.go new file mode 100644 index 0000000000..abe38be53f --- /dev/null +++ b/tensor/indexed.go @@ -0,0 +1,258 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "reflect" + "slices" + + "cogentcore.org/core/base/metadata" + "cogentcore.org/core/base/reflectx" + "cogentcore.org/core/base/slicesx" +) + +// Indexed is a wrapper around another [Tensor] that provides a +// indexed view onto the Tensor provided by an [Int] tensor with +// index coordinates into the source tensor. The innermost dimension +// size of the indexes is equal to the number of dimensions in +// the source tensor, and the remaining outer dimensions provide the +// shape for the [Indexed] tensor view. +// To produce a new concrete [Values] that has raw data actually +// organized according to the indexed order (i.e., the copy function +// of numpy), call [Indexed.AsValues]. +type Indexed struct { //types:add + + // Tensor that we are an indexed view onto. + Tensor Tensor + + // Indexes is the list of indexes into the source tensor, + // with the innermost dimension size equal to the number of + // dimensions in the source tensor, and the remaining outer + // dimensions providing the shape for the [Indexed] tensor. + Indexes *Int +} + +// NewIndexed returns a new [Indexed] view of given tensor, +// with optional tensor of indexes into the source tensor. +func NewIndexed(tsr Tensor, idx ...*Int) *Indexed { + ix := &Indexed{Tensor: tsr} + if len(idx) == 1 { + ix.Indexes = idx[0] + } + ix.ValidIndexes() + return ix +} + +// AsIndexed returns the tensor as a [Indexed] view. +// If it already is one, then it is returned, otherwise it is wrapped. +func AsIndexed(tsr Tensor) *Indexed { + if ix, ok := tsr.(*Indexed); ok { + return ix + } + return NewIndexed(tsr) +} + +// SetTensor sets as indexes into given tensor with sequential initial indexes. +func (ix *Indexed) SetTensor(tsr Tensor) { + ix.Tensor = tsr + ix.ValidIndexes() +} + +// SourceIndexes returns the actual indexes into underlying source tensor +// based on given list of indexes into the [Indexed.Indexes] tensor, +// _excluding_ the final innermost dimension. +func (ix *Indexed) SourceIndexes(i ...int) []int { + idx := slices.Clone(i) + idx = append(idx, 0) // first index + oned := ix.Indexes.Shape().IndexTo1D(idx...) + nd := ix.Tensor.NumDims() + return ix.Indexes.Values[oned : oned+nd] +} + +// SourceIndexesFrom1D returns the full indexes into source tensor based on the +// given 1d index, which is based on the outer dimensions, excluding the +// final innermost dimension. +func (ix *Indexed) SourceIndexesFrom1D(oned int) []int { + nd := ix.Tensor.NumDims() + oned *= nd + return ix.Indexes.Values[oned : oned+nd] +} + +// todo: do this: + +// ValidIndexes ensures that [Indexed.Indexes] are valid, +// removing any out-of-range values and setting the view to nil (full sequential) +// for any dimension with no indexes (which is an invalid condition). +// Call this when any structural changes are made to underlying Tensor. +func (ix *Indexed) ValidIndexes() { + nd := ix.Tensor.NumDims() + ix.Indexes = slicesx.SetLength(ix.Indexes, nd) + for d := range nd { + ni := len(ix.Indexes[d]) + if ni == 0 { // invalid + ix.Indexes[d] = nil // full + continue + } + ds := ix.Tensor.DimSize(d) + ix := ix.Indexes[d] + for i := ni - 1; i >= 0; i-- { + if ix[i] >= ds { + ix = append(ix[:i], ix[i+1:]...) + } + } + ix.Indexes[d] = ix + } +} + +// Label satisfies the core.Labeler interface for a summary description of the tensor. +func (ix *Indexed) Label() string { + return label(ix.Metadata().Name(), ix.Shape()) +} + +// String satisfies the fmt.Stringer interface for string of tensor data. +func (ix *Indexed) String() string { return sprint(ix, 0) } + +func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } + +func (ix *Indexed) IsString() bool { return ix.Tensor.IsString() } + +func (ix *Indexed) DataType() reflect.Kind { return ix.Tensor.DataType() } + +// For each dimension, we return the effective shape sizes using +// the current number of indexes per dimension. +func (ix *Indexed) ShapeSizes() []int { + si := slices.Clone(ix.Indexes.ShapeSizes()) + return si[:len(si)-1] // exclude last dim +} + +// Shape() returns a [Shape] representation of the tensor shape +// (dimension sizes). If we have Indexes, this is the effective +// shape using the current number of indexes per dimension. +func (ix *Indexed) Shape() *Shape { + return NewShape(ix.ShapeSizes()...) +} + +// Len returns the total number of elements in our view of the tensor. +func (ix *Indexed) Len() int { return ix.Shape().Len() } + +// NumDims returns the total number of dimensions. +func (ix *Indexed) NumDims() int { return ix.Indexes.NumDims() - 1 } + +// DimSize returns the effective view size of given dimension. +func (ix *Indexed) DimSize(dim int) int { + return ix.Indexes.DimSize(dim) +} + +// todo: + +// AsValues returns a copy of this tensor as raw [Values]. +// This "renders" the Indexed view into a fully contiguous +// and optimized memory representation of that view, which will be faster +// to access for further processing, and enables all the additional +// functionality provided by the [Values] interface. +func (ix *Indexed) AsValues() Values { + dt := ix.Tensor.DataType() + vt := NewOfType(dt, ix.ShapeSizes()...) + n := ix.Len() + switch { + case ix.Tensor.IsString(): + for i := range n { + vt.SetString1D(ix.String1D(i), i) + } + case reflectx.KindIsFloat(dt): + for i := range n { + vt.SetFloat1D(ix.Float1D(i), i) + } + default: + for i := range n { + vt.SetInt1D(ix.Int1D(i), i) + } + } + return vt +} + +/////////////////////////////////////////////// +// Indexed access + +///////////////////// Floats + +// Float returns the value of given index as a float64. +// The indexes are indirected through the [Indexed.Indexes]. +func (ix *Indexed) Float(i ...int) float64 { + return ix.Tensor.Float(ix.SourceIndexes(i...)...) +} + +// SetFloat sets the value of given index as a float64 +// The indexes are indirected through the [Indexed.Indexes]. +func (ix *Indexed) SetFloat(val float64, i ...int) { + ix.Tensor.SetFloat(val, ix.SourceIndexes(i...)...) +} + +// Float1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ix *Indexed) Float1D(i int) float64 { + return ix.Tensor.Float(ix.SourceIndexesFrom1D(i)...) +} + +// SetFloat1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ix *Indexed) SetFloat1D(val float64, i int) { + ix.Tensor.SetFloat(val, ix.SourceIndexesFrom1D(i)...) +} + +///////////////////// Strings + +// StringValue returns the value of given index as a string. +// The indexes are indirected through the [Indexed.Indexes]. +func (ix *Indexed) StringValue(i ...int) string { + return ix.Tensor.StringValue(ix.SourceIndexes(i...)...) +} + +// SetString sets the value of given index as a string +// The indexes are indirected through the [Indexed.Indexes]. +func (ix *Indexed) SetString(val string, i ...int) { + ix.Tensor.SetString(val, ix.SourceIndexes(i...)...) +} + +// String1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ix *Indexed) String1D(i int) string { + return ix.Tensor.StringValue(ix.SourceIndexesFrom1D(i)...) +} + +// SetString1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ix *Indexed) SetString1D(val string, i int) { + ix.Tensor.SetString(val, ix.SourceIndexesFrom1D(i)...) +} + +///////////////////// Ints + +// Int returns the value of given index as an int. +// The indexes are indirected through the [Indexed.Indexes]. +func (ix *Indexed) Int(i ...int) int { + return ix.Tensor.Int(ix.SourceIndexes(i...)...) +} + +// SetInt sets the value of given index as an int +// The indexes are indirected through the [Indexed.Indexes]. +func (ix *Indexed) SetInt(val int, i ...int) { + ix.Tensor.SetInt(val, ix.SourceIndexes(i...)...) +} + +// Int1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ix *Indexed) Int1D(i int) int { + return ix.Tensor.Int(ix.SourceIndexesFrom1D(i)...) +} + +// SetInt1D is somewhat expensive if indexes are set, because it needs to convert +// the flat index back into a full n-dimensional index and then use that api. +func (ix *Indexed) SetInt1D(val int, i int) { + ix.Tensor.SetInt(val, ix.SourceIndexesFrom1D(i)...) +} + +// check for interface impl +var _ Tensor = (*Indexed)(nil) diff --git a/tensor/masked.go b/tensor/masked.go index 4b5f3c7b01..f1bbce5524 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -18,7 +18,7 @@ import ( // then the corresponding value cannot be set and Float access returns // NaN indicating missing data. // To produce a new [Values] tensor with only the 'true' cases, -// (i.e., the copy function of numpy), call [Masked.CloneValues]. +// (i.e., the copy function of numpy), call [Masked.AsValues]. type Masked struct { //types:add // Tensor that we are a masked view onto. @@ -83,9 +83,7 @@ func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } func (ms *Masked) DataType() reflect.Kind { return ms.Tensor.DataType() } -func (ms *Masked) ShapeInts() []int { return ms.Tensor.ShapeInts() } - -func (ms *Masked) ShapeSizes() *Int { return ms.Tensor.ShapeSizes() } +func (ms *Masked) ShapeSizes() []int { return ms.Tensor.ShapeSizes() } func (ms *Masked) Shape() *Shape { return ms.Tensor.Shape() } diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go index 799b4c5caf..a5f49d1482 100644 --- a/tensor/matrix/matrix.go +++ b/tensor/matrix/matrix.go @@ -89,7 +89,7 @@ func (sy *Symmetric) SymmetricDim() (r int) { // using standard Float64 interface func CopyDense(to tensor.Tensor, dm *mat.Dense) { nr, nc := dm.Dims() - to.SetShapeInts(nr, nc) + to.SetShapeSizes(nr, nc) idx := 0 for ri := 0; ri < nr; ri++ { for ci := 0; ci < nc; ci++ { diff --git a/tensor/number.go b/tensor/number.go index 12b9892afa..8478bc048b 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -67,7 +67,7 @@ func NewByte(sizes ...int) *Byte { // with the given sizes per dimension (shape), and optional dimension names. func NewNumber[T num.Number](sizes ...int) *Number[T] { tsr := &Number[T]{} - tsr.SetShapeInts(sizes...) + tsr.SetShapeSizes(sizes...) tsr.Values = make([]T, tsr.Len()) return tsr } @@ -91,7 +91,7 @@ func NewNumberFromSlice[T num.Number](vals ...T) *Number[T] { n := len(vals) tsr := &Number[T]{} tsr.Values = vals - tsr.SetShapeInts(n) + tsr.SetShapeSizes(n) return tsr } diff --git a/tensor/rows.go b/tensor/rows.go index 604f013315..c6c8d25464 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -21,7 +21,7 @@ import ( // which apply to the outermost row dimension (with default row-major indexing). // Sorting and filtering a tensor only requires updating the indexes while // leaving the underlying Tensor alone. -// Use [CloneValues] to obtain a concrete [Values] representation with the current row +// Use [AsValues] to obtain a concrete [Values] representation with the current row // sorting. Use the [Set]FloatRow[Cell] methods wherever possible, // for the most efficient and natural indirection through the indexes. type Rows struct { //types:add @@ -99,22 +99,15 @@ func (rw *Rows) Metadata() *metadata.Data { return rw.Tensor.Metadata() } // If we have Indexes, this is the effective shape sizes using // the current number of indexes as the outermost row dimension size. -func (rw *Rows) ShapeInts() []int { +func (rw *Rows) ShapeSizes() []int { if rw.Indexes == nil || rw.Tensor.NumDims() == 0 { - return rw.Tensor.ShapeInts() + return rw.Tensor.ShapeSizes() } - sh := slices.Clone(rw.Tensor.ShapeInts()) + sh := slices.Clone(rw.Tensor.ShapeSizes()) sh[0] = len(rw.Indexes) return sh } -func (rw *Rows) ShapeSizes() *Int { - if rw.Indexes == nil { - return rw.Tensor.ShapeSizes() - } - return NewIntFromSlice(rw.ShapeInts()...) -} - // Shape() returns a [Shape] representation of the tensor shape // (dimension sizes). If we have Indexes, this is the effective // shape using the current number of indexes as the outermost row dimension size. @@ -122,7 +115,7 @@ func (rw *Rows) Shape() *Shape { if rw.Indexes == nil { return rw.Tensor.Shape() } - return NewShape(rw.ShapeInts()...) + return NewShape(rw.ShapeSizes()...) } // Len returns the total number of elements in the tensor, @@ -383,7 +376,7 @@ func (rw *Rows) AsValues() Values { if rw.Indexes == nil { return rw.Tensor } - vt := NewOfType(rw.Tensor.DataType(), rw.ShapeInts()...) + vt := NewOfType(rw.Tensor.DataType(), rw.ShapeSizes()...) rows := rw.NumRows() for r := range rows { vt.SetRowTensor(rw.RowTensor(r), r) diff --git a/tensor/shape.go b/tensor/shape.go index 5b54ed522f..430ae71bca 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -27,13 +27,13 @@ type Shape struct { // RowMajor ordering is used by default. func NewShape(sizes ...int) *Shape { sh := &Shape{} - sh.SetShapeInts(sizes...) + sh.SetShapeSizes(sizes...) return sh } -// SetShapeInts sets the shape sizes from list of ints. +// SetShapeSizes sets the shape sizes from list of ints. // RowMajor ordering is used by default. -func (sh *Shape) SetShapeInts(sizes ...int) { +func (sh *Shape) SetShapeSizes(sizes ...int) { sh.Sizes = slices.Clone(sizes) sh.Strides = RowMajorStrides(sizes...) } @@ -41,7 +41,7 @@ func (sh *Shape) SetShapeInts(sizes ...int) { // SetShape sets the shape sizes from int values from Tensor. // RowMajor ordering is used by default. func (sh *Shape) SetShape(shp Tensor) { - sh.SetShapeInts(AsIntSlice(shp)...) + sh.SetShapeSizes(AsIntSlice(shp)...) } // AsTensor returns shape sizes as an Int Tensor. diff --git a/tensor/sliced.go b/tensor/sliced.go index eee15a714e..15676401b2 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -21,10 +21,10 @@ import ( // Thus, every dimension can be transformed in arbitrary ways relative // to the original tensor. There is some additional cost for every // access operation associated with the additional indexed indirection. -// See also [Indexed] for a version that only indexes the outermost row dimension, +// See also [Rows] for a version that only indexes the outermost row dimension, // which is much more efficient for this common use-case. // To produce a new concrete [Values] that has raw data actually organized according -// to the indexed order (i.e., the copy function of numpy), call [Sliced.CloneValues]. +// to the indexed order (i.e., the copy function of numpy), call [Sliced.AsValues]. type Sliced struct { //types:add // Tensor that we are an indexed view onto. @@ -77,9 +77,9 @@ func (sl *Sliced) SetTensor(tsr Tensor) { sl.Sequential() } -// SliceIndex returns the actual index into underlying tensor dimension +// SourceIndex returns the actual index into underlying tensor dimension // based on given index value. -func (sl *Sliced) SliceIndex(dim, idx int) int { +func (sl *Sliced) SourceIndex(dim, idx int) int { ix := sl.Indexes[dim] if ix == nil { return idx @@ -87,22 +87,22 @@ func (sl *Sliced) SliceIndex(dim, idx int) int { return ix[idx] } -// SliceIndexes returns the actual indexes into underlying tensor +// SourceIndexes returns the actual indexes into underlying tensor // based on given list of indexes. -func (sl *Sliced) SliceIndexes(i ...int) []int { +func (sl *Sliced) SourceIndexes(i ...int) []int { ix := slices.Clone(i) for d, idx := range i { - ix[d] = sl.SliceIndex(d, idx) + ix[d] = sl.SourceIndex(d, idx) } return ix } -// IndexFrom1D returns the full indexes into source tensor based on the +// SourceIndexesFrom1D returns the full indexes into source tensor based on the // given 1d index. -func (sl *Sliced) IndexFrom1D(oned int) []int { +func (sl *Sliced) SourceIndexesFrom1D(oned int) []int { sh := sl.Shape() oix := sh.IndexFrom1D(oned) // full indexes in our coords - return sl.SliceIndexes(oix...) + return sl.SourceIndexes(oix...) } // ValidIndexes ensures that [Sliced.Indexes] are valid, @@ -160,30 +160,22 @@ func (sl *Sliced) Label() string { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (sl *Sliced) String() string { - return sprint(sl, 0) -} +func (sl *Sliced) String() string { return sprint(sl, 0) } -// Metadata returns the metadata for this tensor, which can be used -// to encode plotting options, etc. func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } -func (sl *Sliced) IsString() bool { - return sl.Tensor.IsString() -} +func (sl *Sliced) IsString() bool { return sl.Tensor.IsString() } -func (sl *Sliced) DataType() reflect.Kind { - return sl.Tensor.DataType() -} +func (sl *Sliced) DataType() reflect.Kind { return sl.Tensor.DataType() } // For each dimension, we return the effective shape sizes using // the current number of indexes per dimension. -func (sl *Sliced) ShapeInts() []int { +func (sl *Sliced) ShapeSizes() []int { nd := sl.Tensor.NumDims() if nd == 0 { - return sl.Tensor.ShapeInts() + return sl.Tensor.ShapeSizes() } - sh := slices.Clone(sl.Tensor.ShapeInts()) + sh := slices.Clone(sl.Tensor.ShapeSizes()) for d := range nd { if sl.Indexes[d] != nil { sh[d] = len(sl.Indexes[d]) @@ -192,15 +184,11 @@ func (sl *Sliced) ShapeInts() []int { return sh } -func (sl *Sliced) ShapeSizes() *Int { - return NewIntFromSlice(sl.ShapeInts()...) -} - // Shape() returns a [Shape] representation of the tensor shape // (dimension sizes). If we have Indexes, this is the effective // shape using the current number of indexes per dimension. func (sl *Sliced) Shape() *Shape { - return NewShape(sl.ShapeInts()...) + return NewShape(sl.ShapeSizes()...) } // Len returns the total number of elements in our view of the tensor. @@ -226,7 +214,7 @@ func (sl *Sliced) DimSize(dim int) int { // functionality provided by the [Values] interface. func (sl *Sliced) AsValues() Values { dt := sl.Tensor.DataType() - vt := NewOfType(dt, sl.ShapeInts()...) + vt := NewOfType(dt, sl.ShapeSizes()...) n := sl.Len() switch { case sl.Tensor.IsString(): @@ -271,25 +259,25 @@ func (sl *Sliced) AsValues() Values { // Float returns the value of given index as a float64. // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) Float(i ...int) float64 { - return sl.Tensor.Float(sl.SliceIndexes(i...)...) + return sl.Tensor.Float(sl.SourceIndexes(i...)...) } // SetFloat sets the value of given index as a float64 // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) SetFloat(val float64, i ...int) { - sl.Tensor.SetFloat(val, sl.SliceIndexes(i...)...) + sl.Tensor.SetFloat(val, sl.SourceIndexes(i...)...) } // Float1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Float1D(i int) float64 { - return sl.Tensor.Float(sl.IndexFrom1D(i)...) + return sl.Tensor.Float(sl.SourceIndexesFrom1D(i)...) } // SetFloat1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetFloat1D(val float64, i int) { - sl.Tensor.SetFloat(val, sl.IndexFrom1D(i)...) + sl.Tensor.SetFloat(val, sl.SourceIndexesFrom1D(i)...) } ///////////////////// Strings @@ -297,25 +285,25 @@ func (sl *Sliced) SetFloat1D(val float64, i int) { // StringValue returns the value of given index as a string. // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) StringValue(i ...int) string { - return sl.Tensor.StringValue(sl.SliceIndexes(i...)...) + return sl.Tensor.StringValue(sl.SourceIndexes(i...)...) } // SetString sets the value of given index as a string // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) SetString(val string, i ...int) { - sl.Tensor.SetString(val, sl.SliceIndexes(i...)...) + sl.Tensor.SetString(val, sl.SourceIndexes(i...)...) } // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) String1D(i int) string { - return sl.Tensor.StringValue(sl.IndexFrom1D(i)...) + return sl.Tensor.StringValue(sl.SourceIndexesFrom1D(i)...) } // SetString1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetString1D(val string, i int) { - sl.Tensor.SetString(val, sl.IndexFrom1D(i)...) + sl.Tensor.SetString(val, sl.SourceIndexesFrom1D(i)...) } ///////////////////// Ints @@ -323,25 +311,25 @@ func (sl *Sliced) SetString1D(val string, i int) { // Int returns the value of given index as an int. // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) Int(i ...int) int { - return sl.Tensor.Int(sl.SliceIndexes(i...)...) + return sl.Tensor.Int(sl.SourceIndexes(i...)...) } // SetInt sets the value of given index as an int // The indexes are indirected through the [Sliced.Indexes]. func (sl *Sliced) SetInt(val int, i ...int) { - sl.Tensor.SetInt(val, sl.SliceIndexes(i...)...) + sl.Tensor.SetInt(val, sl.SourceIndexes(i...)...) } // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) Int1D(i int) int { - return sl.Tensor.Int(sl.IndexFrom1D(i)...) + return sl.Tensor.Int(sl.SourceIndexesFrom1D(i)...) } // SetInt1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (sl *Sliced) SetInt1D(val int, i int) { - sl.Tensor.SetInt(val, sl.IndexFrom1D(i)...) + sl.Tensor.SetInt(val, sl.SourceIndexesFrom1D(i)...) } // Permuted sets indexes in given dimension to a permuted order. diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index 1045da7ae7..bd7b736ed0 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -110,7 +110,7 @@ func (glm *GLM) Defaults() { func (glm *GLM) init(nIv, nDv int) { glm.NIndepVars = nIv glm.NDepVars = nDv - glm.Coeff.SetShapeInts(nDv, nIv+1) + glm.Coeff.SetShapeSizes(nDv, nIv+1) // glm.Coeff.SetNames("DepVars", "IndepVars") glm.R2 = make([]float64, nDv) glm.ObsVariance = make([]float64, nDv) diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 58b35c4769..b7181c941c 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -34,7 +34,7 @@ func Matrix(funcName string, in, out tensor.Tensor) { if rows == 0 || cells == 0 { return } - out.SetShapeInts(rows, rows) + out.SetShapeSizes(rows, rows) mout := tensor.NewFloat64Scalar(0.0) coords := TriangularLIndicies(rows) nc := len(coords) @@ -73,7 +73,7 @@ func CrossMatrix(funcName string, a, b, out tensor.Tensor) { if brows == 0 || bcells == 0 { return } - out.SetShapeInts(arows, brows) + out.SetShapeSizes(arows, brows) mout := tensor.NewFloat64Scalar(0.0) // note: flops estimating 3 per item on average -- different for different metrics. flops := min(acells, bcells) * 3 @@ -109,10 +109,10 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { } flatvw := in.View() - flatvw.SetShapeInts(in.DimSize(0), cells) + flatvw.SetShapeSizes(in.DimSize(0), cells) mout := tensor.NewFloat64Scalar(0.0) - out.SetShapeInts(cells, cells) + out.SetShapeSizes(cells, cells) var av, bv tensor.Tensor curCoords := vecint.Vector2i{-1, -1} @@ -151,8 +151,8 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { func PCA(covar, eigenvecs, vals tensor.Tensor) { n := covar.DimSize(0) cv := tensor.AsFloat64Tensor(covar) - eigenvecs.SetShapeInts(n, n) - vals.SetShapeInts(n) + eigenvecs.SetShapeSizes(n, n) + vals.SetShapeSizes(n) var eig mat.EigenSym ok := eig.Factorize(cv, true) if !ok { @@ -179,8 +179,8 @@ func PCA(covar, eigenvecs, vals tensor.Tensor) { func SVD(covar, eigenvecs, vals tensor.Tensor) { n := covar.DimSize(0) cv := tensor.AsFloat64Tensor(covar) - eigenvecs.SetShapeInts(n, n) - vals.SetShapeInts(n) + eigenvecs.SetShapeSizes(n, n) + vals.SetShapeSizes(n) var eig mat.SVD ok := eig.Factorize(cv, mat.SVDFull) // todo: benchmark different versions if !ok { @@ -212,7 +212,7 @@ func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) { mout := tensor.NewFloat64() if rows > 0 && cells > 0 { msum := tensor.NewFloat64Scalar(0) - out.SetShapeInts(rows) + out.SetShapeSizes(rows) for i := range rows { tmath.Mul(tensor.Cells1D(vec, i), col, mout) stats.SumFunc(mout, msum) diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 23ecabae31..d15de0482f 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -37,7 +37,7 @@ func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) { mind = d } } - out.SetShapeInts(2) + out.SetShapeSizes(2) out.SetFloat1D(float64(mi), 0) out.SetFloat1D(mind, 1) } diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 90c316dbb5..42f20e6d69 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -21,7 +21,7 @@ func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr } nt := len(tsr) out := tsr[nt-1] - osz := out.ShapeInts() + osz := out.ShapeSizes() out1 = tensor.NewFloat64(osz...) out2 = tensor.NewFloat64(osz...) out3 = tensor.NewFloat64(osz...) @@ -51,7 +51,7 @@ func NFunc(tsr ...tensor.Tensor) int { } a, b, out := tsr[0], tsr[1], tsr[nt-1] na, nb := a.DimSize(0), b.DimSize(0) - out.SetShapeInts(OutShape(a.ShapeInts()...)...) + out.SetShapeSizes(OutShape(a.ShapeSizes()...)...) return min(na, nb) } diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 1adb8deba8..398d2daf74 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -26,7 +26,7 @@ func VectorizeOut64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr } nt := len(tsr) out := tsr[nt-1] - osz := out.ShapeInts() + osz := out.ShapeSizes() o64 := tensor.NewFloat64(osz...) etsr := slices.Clone(tsr) etsr[nt-1] = o64 @@ -50,7 +50,7 @@ func Vectorize2Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr } nt := len(tsr) out := tsr[nt-1] - osz := out.ShapeInts() + osz := out.ShapeSizes() out1 = tensor.NewFloat64(osz...) out2 = tensor.NewFloat64(osz...) tsrs := slices.Clone(tsr[:nt-1]) @@ -80,7 +80,7 @@ func NFunc(tsr ...tensor.Tensor) int { in, out := tsr[0], tsr[nt-1] _ = out n := in.DimSize(0) - out.SetShapeInts(OutShape(in.ShapeInts()...)...) + out.SetShapeSizes(OutShape(in.ShapeSizes()...)...) return n } diff --git a/tensor/string.go b/tensor/string.go index ce35fe048d..c3bffd20ca 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -20,7 +20,7 @@ type String struct { // with the given sizes per dimension (shape). func NewString(sizes ...int) *String { tsr := &String{} - tsr.SetShapeInts(sizes...) + tsr.SetShapeSizes(sizes...) tsr.Values = make([]string, tsr.Len()) return tsr } diff --git a/tensor/table/io.go b/tensor/table/io.go index 15e0b3bf0c..649d92ead6 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -401,7 +401,7 @@ func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { } rc++ } else { - csh := tensor.NewShape(tsr.ShapeInts()[1:]...) // cell shape + csh := tensor.NewShape(tsr.ShapeSizes()[1:]...) // cell shape tc := csh.Len() for ti := 0; ti < tc; ti++ { vl := "" @@ -433,7 +433,7 @@ func (dt *Table) TableHeaders() []string { if tsr.NumDims() == 1 { hdrs = append(hdrs, nm) } else { - csh := tensor.NewShape(tsr.ShapeInts()[1:]...) // cell shape + csh := tensor.NewShape(tsr.ShapeSizes()[1:]...) // cell shape tc := csh.Len() nd := csh.NumDims() fnm := nm + fmt.Sprintf("[%v:", nd) diff --git a/tensor/table/io_test.go b/tensor/table/io_test.go index d4102cb0ac..59e0d1601e 100644 --- a/tensor/table/io_test.go +++ b/tensor/table/io_test.go @@ -9,8 +9,6 @@ import ( "reflect" "strings" "testing" - - "cogentcore.org/core/tensor" ) func TestTableHeaders(t *testing.T) { @@ -36,14 +34,14 @@ func TestTableHeaders(t *testing.T) { if cols[2].DataType() != reflect.Float32 { t.Errorf("TableHeaders: cols[2] != FLOAT32\n") } - shsz := tensor.AsIntSlice(cols[1].ShapeSizes()) + shsz := cols[1].ShapeSizes() if shsz[1] != 5 { t.Errorf("TableHeaders: cols[1].ShapeSizes[1] != 5\n") } if shsz[2] != 5 { t.Errorf("TableHeaders: cols[1].ShapeSizes[2] != 5\n") } - shsz = tensor.AsIntSlice(cols[2].ShapeSizes()) + shsz = cols[2].ShapeSizes() if shsz[1] != 5 { t.Errorf("TableHeaders: cols[2].ShapeSizes[1] != 5\n") } diff --git a/tensor/tensor.go b/tensor/tensor.go index 38329338ad..ce3b530efd 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -58,15 +58,12 @@ type Tensor interface { // (dimension sizes). For tensors that present a view onto another // tensor, this typically must be constructed. // In general, it is better to use the specific [Tensor.ShapeSizes], - // [Tensor.ShapeInts], [Tensor.DimSize] etc as neeed. + // [Tensor.ShapeSizes], [Tensor.DimSize] etc as neeed. Shape() *Shape - // ShapeSizes returns the sizes of each dimension as an int tensor. - ShapeSizes() *Int - - // ShapeInts returns the sizes of each dimension as a slice of ints. + // ShapeSizes returns the sizes of each dimension as a slice of ints. // This is the preferred access for Go code. - ShapeInts() []int + ShapeSizes() []int // Len returns the total number of elements in the tensor, // i.e., the product of all shape dimensions. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 1b57315e27..0e1385cac6 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -41,7 +41,7 @@ func TestTensorString(t *testing.T) { assert.Equal(t, "", cln.StringValue(2, 1)) assert.Equal(t, "testing", tsr.StringValue(2, 1)) - tsr.SetShapeInts(2, 4) + tsr.SetShapeSizes(2, 4) // tsr.SetNames("Vals", "Row") assert.Equal(t, "test", tsr.StringValue(1, 0)) assert.Equal(t, "testing", tsr.StringValue(1, 1)) @@ -100,7 +100,7 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 0.0, cln.Float(2, 1)) assert.Equal(t, 2.17, tsr.Float(2, 1)) - tsr.SetShapeInts(2, 4) + tsr.SetShapeSizes(2, 4) assert.Equal(t, 3.14, tsr.Float(1, 0)) assert.Equal(t, 2.17, tsr.Float(1, 1)) diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index 363cd6669e..6e975a7da9 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -361,7 +361,7 @@ func (tb *Table) ColTensorBlank(cidx int, col tensor.Tensor) *tensor.Float64 { if ctb, has := tb.ColumnTensorBlank[cidx]; has { return ctb } - ctb := tensor.New[float64](col.ShapeInts()...).(*tensor.Float64) + ctb := tensor.New[float64](col.ShapeSizes()...).(*tensor.Float64) tb.ColumnTensorBlank[cidx] = ctb return ctb } diff --git a/tensor/typegen.go b/tensor/typegen.go index e83773f93e..35574646c5 100644 --- a/tensor/typegen.go +++ b/tensor/typegen.go @@ -6,8 +6,10 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Masked", IDName: "masked", Doc: "Masked is a wrapper around another [Tensor] that provides a\nbit-masked view onto the Tensor defined by\nfor each dimension (must have at least 1 per dimension).\nThus, every dimension can be transformed in arbitrary ways relative\nto the original tensor. There is some additional cost for every\naccess operation associated with the additional indexed indirection.\nSee also [Indexed] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case.\nTo produce a new [Tensor] that has its raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Masked.CloneValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is a wrapper around another [Tensor] that provides a\nindexed view onto the Tensor provided by an [Int] tensor with\nindex coordinates into the source tensor. The innermost dimension\nsize of the indexes is equal to the number of dimensions in\nthe source tensor, and the remaining outer dimensions provide the\nshape for the [Indexed] tensor view.\nTo produce a new concrete [Values] that has raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Indexed.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Rows", IDName: "rows", Doc: "Rows is a row-indexed wrapper around a [Values] [Tensor] that provides a\nspecific view onto the Tensor defined by the set of [Rows.Indexes],\nwhich apply to the outermost row dimension (with default row-major indexing).\nSorting and filtering a tensor only requires updating the indexes while\nleaving the underlying Tensor alone.\nUse [CloneValues] to obtain a concrete [Values] representation with the current row\nsorting. Use the [Set]FloatRow[Cell] methods wherever possible,\nfor the most efficient and natural indirection through the indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing", Doc: "ExcludeMissing deletes indexes where the values are missing, as indicated by NaN.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values compared to given\nstring. Includes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse the named const args [Include], [Exclude], [Contains], [Equals],\n[IgnoreCase], [UseCase] for greater clarity.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"str", "exclude", "contains", "ignoreCase"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nUse the Index() method for nil-aware logic."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Masked", IDName: "masked", Doc: "Masked is a wrapper around another [Tensor] that provides a\nbit-masked view onto the Tensor defined by a [Bool] [Values]\ntensor with a matching shape. If the bool mask has a 'false'\nthen the corresponding value cannot be set and Float access returns\nNaN indicating missing data.\nTo produce a new [Values] tensor with only the 'true' cases,\n(i.e., the copy function of numpy), call [Masked.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are a masked view onto."}, {Name: "Mask", Doc: "Bool tensor with same shape as source tensor, providing mask."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Sliced", IDName: "sliced", Doc: "Sliced is a fully indexed wrapper around another [Tensor] that provides a\nre-sliced view onto the Tensor defined by the set of [Sliced.Indexes],\nfor each dimension (must have at least 1 per dimension).\nThus, every dimension can be transformed in arbitrary ways relative\nto the original tensor. There is some additional cost for every\naccess operation associated with the additional indexed indirection.\nSee also [Indexed] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case.\nTo produce a new [Tensor] that has its raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Sliced.CloneValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Rows", IDName: "rows", Doc: "Rows is a row-indexed wrapper around a [Values] [Tensor] that provides a\nspecific view onto the Tensor defined by the set of [Rows.Indexes],\nwhich apply to the outermost row dimension (with default row-major indexing).\nSorting and filtering a tensor only requires updating the indexes while\nleaving the underlying Tensor alone.\nUse [AsValues] to obtain a concrete [Values] representation with the current row\nsorting. Use the [Set]FloatRow[Cell] methods wherever possible,\nfor the most efficient and natural indirection through the indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing", Doc: "ExcludeMissing deletes indexes where the values are missing, as indicated by NaN.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values compared to given\nstring. Includes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse the named const args [Include], [Exclude], [Contains], [Equals],\n[IgnoreCase], [UseCase] for greater clarity.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"str", "exclude", "contains", "ignoreCase"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nUse the Index() method for nil-aware logic."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Sliced", IDName: "sliced", Doc: "Sliced is a fully indexed wrapper around another [Tensor] that provides a\nre-sliced view onto the Tensor defined by the set of [Sliced.Indexes],\nfor each dimension (must have at least 1 per dimension).\nThus, every dimension can be transformed in arbitrary ways relative\nto the original tensor. There is some additional cost for every\naccess operation associated with the additional indexed indirection.\nSee also [Rows] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case.\nTo produce a new concrete [Values] that has raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Sliced.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) diff --git a/tensor/values.go b/tensor/values.go index 5632d306d8..f874095760 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -22,9 +22,9 @@ type Values interface { // The backing storage is resized appropriately, retaining all existing data that fits. SetShape(sizes Tensor) - // SetShapeInts sets the dimension sizes of the tensor, and resizes + // SetShapeSizes sets the dimension sizes of the tensor, and resizes // backing storage appropriately, retaining all existing data that fits. - SetShapeInts(sizes ...int) + SetShapeSizes(sizes ...int) // SetNumRows sets the number of rows (outermost dimension). // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) From 72b29952a511d19f9c8615181e8c2ca9baaeb15e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 11:36:00 -0700 Subject: [PATCH 093/311] tensor Indexed fully implemented and tested; cleaned up docs and added more tests. now ready for braodcasting. --- tensor/indexed.go | 94 +++++++++++-------------------------------- tensor/rows.go | 35 +++++++--------- tensor/sliced.go | 77 +++++++++++------------------------ tensor/tensor_test.go | 66 ++++++++++++++++++++---------- 4 files changed, 107 insertions(+), 165 deletions(-) diff --git a/tensor/indexed.go b/tensor/indexed.go index abe38be53f..0e4966a10e 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -10,54 +10,53 @@ import ( "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" - "cogentcore.org/core/base/slicesx" ) -// Indexed is a wrapper around another [Tensor] that provides a -// indexed view onto the Tensor provided by an [Int] tensor with -// index coordinates into the source tensor. The innermost dimension -// size of the indexes is equal to the number of dimensions in -// the source tensor, and the remaining outer dimensions provide the -// shape for the [Indexed] tensor view. +// Indexed provides an arbitrarily indexed view onto another "source" [Tensor] +// with each index value providing a full n-dimensional index into the source. +// The shape of this view is determined by the shape of the [Indexed.Indexes] +// tensor up to the final innermost dimension, which holds the index values. +// Thus the innermost dimension size of the indexes is equal to the number +// of dimensions in the source tensor. Given the essential role of the +// indexes in this view, it is not usable without the indexes. +// This view is not memory-contiguous and does not support the [RowMajor] +// interface or efficient access to inner-dimensional subspaces. // To produce a new concrete [Values] that has raw data actually // organized according to the indexed order (i.e., the copy function // of numpy), call [Indexed.AsValues]. type Indexed struct { //types:add - // Tensor that we are an indexed view onto. + // Tensor source that we are an indexed view onto. Tensor Tensor // Indexes is the list of indexes into the source tensor, - // with the innermost dimension size equal to the number of - // dimensions in the source tensor, and the remaining outer - // dimensions providing the shape for the [Indexed] tensor. + // with the innermost dimension providing the index values + // (size = number of dimensions in the source tensor), and + // the remaining outer dimensions determine the shape + // of this [Indexed] tensor view. Indexes *Int } // NewIndexed returns a new [Indexed] view of given tensor, -// with optional tensor of indexes into the source tensor. -func NewIndexed(tsr Tensor, idx ...*Int) *Indexed { +// with tensor of indexes into the source tensor. +func NewIndexed(tsr Tensor, idx *Int) *Indexed { ix := &Indexed{Tensor: tsr} - if len(idx) == 1 { - ix.Indexes = idx[0] - } - ix.ValidIndexes() + ix.Indexes = idx return ix } -// AsIndexed returns the tensor as a [Indexed] view. -// If it already is one, then it is returned, otherwise it is wrapped. +// AsIndexed returns the tensor as a [Indexed] view, if it is one. +// Otherwise, it returns nil; there is no usable "null" Indexed view. func AsIndexed(tsr Tensor) *Indexed { if ix, ok := tsr.(*Indexed); ok { return ix } - return NewIndexed(tsr) + return nil } // SetTensor sets as indexes into given tensor with sequential initial indexes. func (ix *Indexed) SetTensor(tsr Tensor) { ix.Tensor = tsr - ix.ValidIndexes() } // SourceIndexes returns the actual indexes into underlying source tensor @@ -80,38 +79,8 @@ func (ix *Indexed) SourceIndexesFrom1D(oned int) []int { return ix.Indexes.Values[oned : oned+nd] } -// todo: do this: - -// ValidIndexes ensures that [Indexed.Indexes] are valid, -// removing any out-of-range values and setting the view to nil (full sequential) -// for any dimension with no indexes (which is an invalid condition). -// Call this when any structural changes are made to underlying Tensor. -func (ix *Indexed) ValidIndexes() { - nd := ix.Tensor.NumDims() - ix.Indexes = slicesx.SetLength(ix.Indexes, nd) - for d := range nd { - ni := len(ix.Indexes[d]) - if ni == 0 { // invalid - ix.Indexes[d] = nil // full - continue - } - ds := ix.Tensor.DimSize(d) - ix := ix.Indexes[d] - for i := ni - 1; i >= 0; i-- { - if ix[i] >= ds { - ix = append(ix[:i], ix[i+1:]...) - } - } - ix.Indexes[d] = ix - } -} +func (ix *Indexed) Label() string { return label(ix.Metadata().Name(), ix.Shape()) } -// Label satisfies the core.Labeler interface for a summary description of the tensor. -func (ix *Indexed) Label() string { - return label(ix.Metadata().Name(), ix.Shape()) -} - -// String satisfies the fmt.Stringer interface for string of tensor data. func (ix *Indexed) String() string { return sprint(ix, 0) } func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } @@ -120,32 +89,18 @@ func (ix *Indexed) IsString() bool { return ix.Tensor.IsString() } func (ix *Indexed) DataType() reflect.Kind { return ix.Tensor.DataType() } -// For each dimension, we return the effective shape sizes using -// the current number of indexes per dimension. func (ix *Indexed) ShapeSizes() []int { si := slices.Clone(ix.Indexes.ShapeSizes()) return si[:len(si)-1] // exclude last dim } -// Shape() returns a [Shape] representation of the tensor shape -// (dimension sizes). If we have Indexes, this is the effective -// shape using the current number of indexes per dimension. -func (ix *Indexed) Shape() *Shape { - return NewShape(ix.ShapeSizes()...) -} +func (ix *Indexed) Shape() *Shape { return NewShape(ix.ShapeSizes()...) } -// Len returns the total number of elements in our view of the tensor. func (ix *Indexed) Len() int { return ix.Shape().Len() } -// NumDims returns the total number of dimensions. func (ix *Indexed) NumDims() int { return ix.Indexes.NumDims() - 1 } -// DimSize returns the effective view size of given dimension. -func (ix *Indexed) DimSize(dim int) int { - return ix.Indexes.DimSize(dim) -} - -// todo: +func (ix *Indexed) DimSize(dim int) int { return ix.Indexes.DimSize(dim) } // AsValues returns a copy of this tensor as raw [Values]. // This "renders" the Indexed view into a fully contiguous @@ -173,9 +128,6 @@ func (ix *Indexed) AsValues() Values { return vt } -/////////////////////////////////////////////// -// Indexed access - ///////////////////// Floats // Float returns the value of given index as a float64. diff --git a/tensor/rows.go b/tensor/rows.go index c6c8d25464..0732fcf999 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -16,22 +16,25 @@ import ( "cogentcore.org/core/base/metadata" ) -// Rows is a row-indexed wrapper around a [Values] [Tensor] that provides a -// specific view onto the Tensor defined by the set of [Rows.Indexes], -// which apply to the outermost row dimension (with default row-major indexing). -// Sorting and filtering a tensor only requires updating the indexes while -// leaving the underlying Tensor alone. -// Use [AsValues] to obtain a concrete [Values] representation with the current row -// sorting. Use the [Set]FloatRow[Cell] methods wherever possible, -// for the most efficient and natural indirection through the indexes. +// Rows is a row-indexed wrapper view around a [Values] [Tensor] that allows +// arbitrary row-wise ordering and filtering according to the [Rows.Indexes]. +// Sorting and filtering a tensor along this outermost row dimension only +// requires updating the indexes while leaving the underlying Tensor alone. +// Unlike the more general [Sliced] view, Rows maintains memory contiguity +// for the inner dimensions ("cells") within each row, and supports the [RowMajor] +// interface, with the [Set]FloatRow[Cell] methods providing efficient access. +// Use [Rows.AsValues] to obtain a concrete [Values] representation with the +// current row sorting. type Rows struct { //types:add - // Tensor that we are an indexed view onto. + // Tensor source that we are an indexed view onto. + // Note that this must be a concrete [Values] tensor, to enable efficient + // [RowMajor] access and subspace functions. Tensor Values // Indexes are the indexes into Tensor rows, with nil = sequential. // Only set if order is different from default sequential order. - // Use the Index() method for nil-aware logic. + // Use the [Rows.RowIndex] method for nil-aware logic. Indexes []int } @@ -83,18 +86,10 @@ func (rw *Rows) NumRows() int { return len(rw.Indexes) } -// String satisfies the fmt.Stringer interface for string of tensor data. -func (rw *Rows) String() string { - return sprint(rw.Tensor, 0) // todo: no need -} +func (rw *Rows) String() string { return sprint(rw.Tensor, 0) } -// Label satisfies the core.Labeler interface for a summary description of the tensor. -func (rw *Rows) Label() string { - return rw.Tensor.Label() -} +func (rw *Rows) Label() string { return rw.Tensor.Label() } -// Metadata returns the metadata for this tensor, which can be used -// to encode plotting options, etc. func (rw *Rows) Metadata() *metadata.Data { return rw.Tensor.Metadata() } // If we have Indexes, this is the effective shape sizes using diff --git a/tensor/sliced.go b/tensor/sliced.go index 15676401b2..46d3e3fa29 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -15,26 +15,30 @@ import ( "cogentcore.org/core/base/slicesx" ) -// Sliced is a fully indexed wrapper around another [Tensor] that provides a -// re-sliced view onto the Tensor defined by the set of [Sliced.Indexes], -// for each dimension (must have at least 1 per dimension). -// Thus, every dimension can be transformed in arbitrary ways relative -// to the original tensor. There is some additional cost for every -// access operation associated with the additional indexed indirection. +// Sliced provides a re-sliced view onto another "source" [Tensor], +// defined by a set of [Sliced.Indexes] for each dimension (must have +// at least 1 index per dimension to avoid a null view). +// Thus, each dimension can be transformed in arbitrary ways relative +// to the original tensor (filtered subsets, reversals, sorting, etc). +// This view is not memory-contiguous and does not support the [RowMajor] +// interface or efficient access to inner-dimensional subspaces. +// A new Sliced view defaults to a full transparent view of the ource tensor. +// There is additional cost for every access operation associated with the +// indexed indirection, and access is always via the full n-dimensional indexes. // See also [Rows] for a version that only indexes the outermost row dimension, -// which is much more efficient for this common use-case. +// which is much more efficient for this common use-case, and does support [RowMajor]. // To produce a new concrete [Values] that has raw data actually organized according // to the indexed order (i.e., the copy function of numpy), call [Sliced.AsValues]. type Sliced struct { //types:add - // Tensor that we are an indexed view onto. + // Tensor source that we are an indexed view onto. Tensor Tensor // Indexes are the indexes for each dimension, with dimensions as the outer // slice (enforced to be the same length as the NumDims of the source Tensor), // and a list of dimension index values (within range of DimSize(d)). - // A nil list of indexes automatically provides a full, sequential view of that - // dimension. + // A nil list of indexes for a dimension automatically provides a full, + // sequential view of that dimension. Indexes [][]int } @@ -71,13 +75,14 @@ func AsSliced(tsr Tensor) *Sliced { return NewSliced(tsr) } -// SetTensor sets as indexes into given tensor with sequential initial indexes. +// SetTensor sets tensor as source for this view, and initializes a full +// transparent view onto source (calls [Sliced.Sequential]). func (sl *Sliced) SetTensor(tsr Tensor) { sl.Tensor = tsr sl.Sequential() } -// SourceIndex returns the actual index into underlying tensor dimension +// SourceIndex returns the actual index into source tensor dimension // based on given index value. func (sl *Sliced) SourceIndex(dim, idx int) int { ix := sl.Indexes[dim] @@ -87,8 +92,8 @@ func (sl *Sliced) SourceIndex(dim, idx int) int { return ix[idx] } -// SourceIndexes returns the actual indexes into underlying tensor -// based on given list of indexes. +// SourceIndexes returns the actual n-dimensional indexes into source tensor +// based on given list of indexes based on the Sliced view shape. func (sl *Sliced) SourceIndexes(i ...int) []int { ix := slices.Clone(i) for d, idx := range i { @@ -97,8 +102,8 @@ func (sl *Sliced) SourceIndexes(i ...int) []int { return ix } -// SourceIndexesFrom1D returns the full indexes into source tensor based on the -// given 1d index. +// SourceIndexesFrom1D returns the n-dimensional indexes into source tensor +// based on the given 1D index based on the Sliced view shape. func (sl *Sliced) SourceIndexesFrom1D(oned int) []int { sh := sl.Shape() oix := sh.IndexFrom1D(oned) // full indexes in our coords @@ -154,12 +159,8 @@ func (sl *Sliced) IndexesNeeded(d int) { sl.Indexes[d] = ix } -// Label satisfies the core.Labeler interface for a summary description of the tensor. -func (sl *Sliced) Label() string { - return label(sl.Metadata().Name(), sl.Shape()) -} +func (sl *Sliced) Label() string { return label(sl.Metadata().Name(), sl.Shape()) } -// String satisfies the fmt.Stringer interface for string of tensor data. func (sl *Sliced) String() string { return sprint(sl, 0) } func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } @@ -184,19 +185,10 @@ func (sl *Sliced) ShapeSizes() []int { return sh } -// Shape() returns a [Shape] representation of the tensor shape -// (dimension sizes). If we have Indexes, this is the effective -// shape using the current number of indexes per dimension. -func (sl *Sliced) Shape() *Shape { - return NewShape(sl.ShapeSizes()...) -} +func (sl *Sliced) Shape() *Shape { return NewShape(sl.ShapeSizes()...) } -// Len returns the total number of elements in our view of the tensor. -func (sl *Sliced) Len() int { - return sl.Shape().Len() -} +func (sl *Sliced) Len() int { return sl.Shape().Len() } -// NumDims returns the total number of dimensions. func (sl *Sliced) NumDims() int { return sl.Tensor.NumDims() } // DimSize returns the effective view size of given dimension. @@ -233,27 +225,6 @@ func (sl *Sliced) AsValues() Values { return vt } -// // CloneIndexes returns a copy of the current Sliced view with new indexes, -// // with a pointer to the same underlying Tensor as the source. -// func (sl *Sliced) CloneIndexes() *Sliced { -// nix := &Sliced{} -// nix.Tensor = sl.Tensor -// nix.CopyIndexes(sl) -// return nix -// } -// -// // CopyIndexes copies indexes from other Sliced view. -// func (sl *Sliced) CopyIndexes(oix *Sliced) { -// if oix.Indexes == nil { -// sl.Indexes = nil -// } else { -// sl.Indexes = slices.Clone(oix.Indexes) -// } -// } - -/////////////////////////////////////////////// -// Sliced access - ///////////////////// Floats // Float returns the value of given index as a float64. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 0e1385cac6..a1ac205078 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -144,31 +144,23 @@ func TestSliced(t *testing.T) { [0]: 23 22 [1]: 13 12 ` - sl := NewSlicedIndexes(ft, []int{2, 1}, []int{3, 2}) // fmt.Println(sl) assert.Equal(t, res, sl.String()) - /* - - ft := NewFloat64(2, 3, 4) - for r := range 2 { - for y := range 3 { - for x := range 4 { - v := (r+1)*100 + y*10 + x - ft.SetFloat(float64(v), r, y, x) - } - } - } - - fmt.Println(ft) - - sl := NewSliced(ft, []int{1, 0}, []int{1, 0}, []int{1, 0}) - fmt.Println(sl) - - assert.Equal(t, res, sf.Tensor.String()) - */ + vl := sl.AsValues() + assert.Equal(t, res, vl.String()) + res = `[3, 1] +[0]: 2 +[1]: 12 +[2]: 22 +` + sl = NewSliced(ft, Slice{}, Slice{2, 3, 0}) + // fmt.Println(sl) + assert.Equal(t, res, sl.String()) + vl = sl.AsValues() + assert.Equal(t, res, vl.String()) } func TestMasked(t *testing.T) { @@ -204,7 +196,39 @@ func TestMasked(t *testing.T) { ` vl := ms.AsValues() - // fmt.Println(vl.String()) + assert.Equal(t, res, vl.String()) +} + +func TestIndexed(t *testing.T) { + ft := NewFloat64(3, 4) + for y := range 3 { + for x := range 4 { + v := y*10 + x + ft.SetFloat(float64(v), y, x) + } + } + ixs := NewIntFromSlice([]int{ + 0, 1, + 0, 1, + 0, 2, + 0, 2, + 1, 1, + 1, 1, + 2, 2, + 2, 2, + }...) + + ixs.SetShapeSizes(2, 2, 2, 2) + ix := NewIndexed(ft, ixs) + + res := `[2, 2, 2] +[0]: 1 1 11 11 +[0]: 2 2 22 22 +` + // fmt.Println(ix.String()) + assert.Equal(t, res, ix.String()) + + vl := ix.AsValues() assert.Equal(t, res, vl.String()) } From 35e600bc599b59581c8025e35a6648890d4a7fcb Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 13:03:44 -0700 Subject: [PATCH 094/311] more docs cleanup in README etc -- now ready for broadcast / alignment --- goal/transpile/math.go | 10 ++--- goal/transpile/transpile_test.go | 2 +- tensor/README.md | 42 ++++++++++--------- tensor/convert.go | 22 +++++----- tensor/funcs_test.go | 2 +- tensor/masked.go | 65 +++++++++++++++++++----------- tensor/number.go | 4 +- tensor/shape.go | 2 +- tensor/sliced.go | 7 +++- tensor/stats/metric/metric_test.go | 4 +- tensor/stats/stats/stats_test.go | 4 +- tensor/tensor.go | 2 +- tensor/tensor_test.go | 4 +- tensor/tmath/math_test.go | 2 +- tensor/tmath/ops_test.go | 2 +- 15 files changed, 100 insertions(+), 74 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index fe6d2306d4..362c61172c 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -261,7 +261,7 @@ type funWrap struct { wrap string } -// nis: NewIntScalar, nifs: NewIntFromSlice +// nis: NewIntScalar, nifs: NewIntFromValues var numpyProps = map[string]funWrap{ "ndim": {"NumDims()", "nis"}, "len": {"Len()", "nis"}, @@ -287,13 +287,13 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { case "nss": mp.out.Add(token.IDENT, "tensor.NewStringScalar") case "nifs": - mp.out.Add(token.IDENT, "tensor.NewIntFromSlice") + mp.out.Add(token.IDENT, "tensor.NewIntFromValues") elip = true case "nffs": - mp.out.Add(token.IDENT, "tensor.NewFloat64FromSlice") + mp.out.Add(token.IDENT, "tensor.NewFloat64FromValues") elip = true case "nsfs": - mp.out.Add(token.IDENT, "tensor.NewStringFromSlice") + mp.out.Add(token.IDENT, "tensor.NewStringFromValues") elip = true } mp.out.Add(token.LPAREN) @@ -328,7 +328,7 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { typ = "string" fun = "String" } - mp.out.Add(token.IDENT, "tensor.New"+fun+"FromSlice") + mp.out.Add(token.IDENT, "tensor.New"+fun+"FromValues") mp.out.Add(token.LPAREN) mp.out.Add(token.IDENT, "[]"+typ) mp.addToken(token.LBRACE) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 7cafac4bd0..c866a98c3e 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ func TestMath(t *testing.T) { // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - // {"# a := [1,2,3,4]", `a := tensor.NewIntFromSlice([]int { 1, 2, 3, 4 } ...)`}, + // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# a := zeros(3, 4)", `a := tensor.NewFloat64Tensor(3, 4)`}, } diff --git a/tensor/README.md b/tensor/README.md index 2b47f8c047..31cf3dc715 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -2,25 +2,27 @@ Tensor and related sub-packages provide a simple yet powerful framework for representing n-dimensional data of various types, providing similar functionality to the widely used [NumPy](https://numpy.org/doc/stable/index.html) and [pandas](https://pandas.pydata.org/) libraries in Python, and the commercial MATLAB framework. -The [Goal](../goal) augmented version of the _Go_ language directly supports numpy-like operations on tensors. A `Tensor` is comparable to the NumPy `ndarray` type, and it provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix and beyond. All functions take and return `Tensor` arguments. +The [Goal](../goal) augmented version of the _Go_ language directly supports NumPy-like operations on tensors. A `Tensor` is comparable to the NumPy `ndarray` type, and it provides the universal representation of a homogenous data type throughout all the packages here, from scalar to vector, matrix and beyond. All functions take and return `Tensor` arguments. -The `Tensor` interface is implemented at the basic level with n-dimensional indexing into flat Go slices of any numeric data type (by `Number`), along with `String`, and `Bool` (which uses [bitslice](bitslice) for maximum efficiency). The `Shape` type provides all the n-dimensional indexing with arbitrary strides to allow any ordering, although _row major_ is the default and other orders have to be manually imposed. +The `Tensor` interface is implemented at the basic level with n-dimensional indexing into flat Go slices of any numeric data type (by `Number`), along with `String`, and `Bool` (which uses [bitslice](bitslice) for maximum efficiency). These implementations satisfy the `Values` sub-interface of Tensor, which supports the most direct and efficient operations on contiguous memory data. The `Shape` type provides all the n-dimensional indexing with arbitrary strides to allow any ordering, although _row major_ is the default and other orders have to be manually imposed. -In addition, there are three important "view" implementations of `Tensor` that wrap another Tensor to provide more flexible and efficient access to the data, consistent with the NumPy functionality. See [Basic and Advanced Indexing](#basic-and-advanced-indexing) for more info. +In addition, there are four important "view" implementations of `Tensor` that wrap another "source" Tensor to provide more flexible and efficient access to the data, consistent with the NumPy functionality. See [Basic and Advanced Indexing](#basic-and-advanced-indexing) below for more info. -* `Rows` provides an index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension. Organizing data systematically along the row dimension eliminates many sources of ambiguity about how to process higher-dimensional data, and any given n-dimensional structure can be reshaped to fit in this row-based format. +* `Rows` provides a row index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension, and provides similar functionality to pandas and particularly [xarray](http://xarray.pydata.org/en/stable/) in Python. Organizing data systematically along the row dimension eliminates many sources of ambiguity about how to process higher-dimensional data, and any given n-dimensional structure can be reshaped to fit in this row-based format. -* `Sliced` is a more general version of `Rows` that provides a sub-sliced view into the wrapped `Tensor`, using an indexed list along each dimension, not just the outermost one. Thus, it can provide a systematic, reordered and filtered view onto the raw data, and it has a well-defined shape in terms of the number of indexes per dimension. This corresponds well with the NumPy basic sliced indexing model. +* `Sliced` is a more general version of `Rows` that provides a sub-sliced view into the wrapped `Tensor` source, using an indexed list along each dimension, not just the outermost one. Thus, it can provide a reordered and filtered view onto the raw data, and it has a well-defined shape in terms of the number of indexes per dimension. This corresponds to the NumPy basic sliced indexing model. -* `Masked` provides a `Bool` masked view onto each element in the wrapped `Tensor`, where the two maintain the same shape). Any cell with a `false` value in the bool mask returns a `NaN` (missing data), and `Set` functions are no-ops. +* `Masked` provides a `Bool` masked view onto each element in the wrapped `Tensor`, where the two maintain the same shape). Any cell with a `false` value in the bool mask returns a `NaN` (missing data), and `Set` functions are no-ops, such that the tensor functions automatically only process the mask-filtered data. -* `Indexed` uses a tensor of indexes where the final, innermost dimension is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. +* `Indexed` has a tensor of indexes into the source data, where the final, innermost dimension of the indexes is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. + +Each view type implements the `AsValues` method to create a concrete "rendered" version of the view (as a `Values` tensor) where the actual underlying data is organized as it appears in the view. This is like the `copy` function in NumPy, disconnecting the view from the original source data. Note that unlike NumPy, `Masked` and `Indexed` remain views into the underlying source data -- see [Basic and Advanced Indexing](#basic-and-advanced-indexing) below. The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. -The `[Set]FloatRow[Cell]` methods are used wherever possible, for the most efficient and natural indirection for row-major organized data. See [Standard shapes](#standard-shapes) for more info. +There is also a `RowMajor` sub-interface for tensors (implemented by the `Values` and `Rows` types), which supports `[Set]FloatRow[Cell]` methods that provide optimized access to row major data. See [Standard shapes](#standard-shapes) for more info. -The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, which is applied to every index, and a varargs list of indexed tensors. In general it is completely up to the compute function how to interpret the index, although we also support the [broadcasting](#broadcasting) principles from NumPy for binary functions operating on two tensors. There is a Threaded version of this for parallelizable functions, and a GPU version. +The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, and a varargs list of tensors. In general it is completely up to the compute function how to interpret the index, although we also support the "broadcasting" principles from NumPy for binary functions operating on two tensors, as discussed below. There is a Threaded version of this for parallelizable functions, and a GPU version in the [gosl](../gpu/gosl) Go-as-a-shading-language package. All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensor for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. These functions are the equivalent of the [ufunc](https://numpy.org/doc/stable/user/basics.ufuncs.html) universal functions in NumPy. @@ -124,7 +126,7 @@ The [Goal](../goal) language provides a reasonably faithful translation of NumPy ## Basic and Advanced Indexing -NumPy distinguishes between _basic indexing_ (using a single index or sliced ranges of indexes along each dimension) versus _advanced indexing_ (using an array of indexes or bools) indexing. Basic indexing returns a **view** into the original data (where changes to the view directly affect the underlying type), while advanced indexing returns a **copy**. +NumPy distinguishes between _basic indexing_ (using a single index or sliced ranges of indexes along each dimension) versus _advanced indexing_ (using an array of indexes or bools). Basic indexing returns a **view** into the original data (where changes to the view directly affect the underlying type), while advanced indexing returns a **copy**. However, rather confusingly (per this [stack overflow question](https://stackoverflow.com/questions/15691740/does-assignment-with-advanced-indexing-copy-array-data)), you can do direct assignment through advanced indexing (more on this below): ```Python @@ -132,19 +134,21 @@ a[np.array([1,2])] = 5 # or: a[a > 0.5] = 1 # boolean advanced indexing ``` -Although powerful, the semantics of all of this is a bit confusing. In the `tensor` package, we provide what are hopefully more clear and concrete _view_ types that have well-defined semantics, and cover the relevant functionality, while hopefully being a bit easier to reason with. These were described at the start of this README. The correspondence to NumPy indexing is as follows: +Although powerful, the semantics of all of this is a bit confusing. In the `tensor` package, we provide what are hopefully more clear and concrete _view_ types that have well-defined semantics, and cover the relevant functionality, while perhaps being a bit easier to reason with. These were described at the start of this README. The correspondence to NumPy indexing is as follows: -* Basic indexing by individual integer index coordinate values is supported by the basic `Number`, `String`, `Bool` value `Tensor`s. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) view then completes the basic indexing with arbitrary reordering and filtering along entire dimension values. +* Basic indexing by individual integer index coordinate values is supported by the basic `Number`, `String`, `Bool` `Values` Tensors. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) view then completes the basic indexing with arbitrary reordering and filtering along entire dimension values. -* The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `AsValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `CloneIndexes` method returns a 1D list of indexes of the `true` values, which can be used for the `Indexed` view. +* The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `AsValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `SourceIndexes` method returns a 1D list of indexes of the `true` (or `false`) values, which can be used for the `Indexed` view. -* The `Indexed` view corresponds to the array-based advanced indexing case in NumPy, but again it is a view, not a copy, so the assignment semantics are as expected from a view (and how NumPy behaves some of the time). Note that the NumPy version uses `n` separate index tensors, where each such tensor specifies the value of a corresponding dimension index, and all such tensors _must have the same shape_; that form can be converted into the single Indexes form with a utility function. Also, NumPy advanced indexing has a somewhat confusing property where it de-duplicates index references during some operations, such that `+=1` only increments +1 even when there are multiple elements in the view. The tensor version does not implement that special case, due to its direct view semantics. +* The `Indexed` view corresponds to the array-based advanced indexing case in NumPy, but again it is a view, not a copy, so the assignment semantics are as expected from a view (and how NumPy behaves some of the time). Note that the NumPy version uses `n` separate index tensors, where each such tensor specifies the value of a corresponding dimension index, and all such tensors _must have the same shape_; that form can be converted into the single Indexes form with a utility function. Also, NumPy advanced indexing has a somewhat confusing property where it de-duplicates index references during some operations, such that `a+=1` only increments +1 even when there are multiple elements in the view. The tensor version does not implement that special case, due to its direct view semantics. To reiterate, all view tensors have a `AsValues` function, equivalent to the `copy` function in NumPy, which turns the view into a corresponding basic concrete value Tensor, so the copy semantics of advanced indexing (modulo the direct assignment behavior) can be achieved when assigning to a new variable. -## Broadcasting +## Alignment of shapes for computations ("broadcasting") + +The NumPy concept of [broadcasting](https://numpy.org/doc/stable/user/basics.broadcasting.html) is critical for flexibly defining the semantics for how functions taking two n-dimensional Tensor arguments behave when they have different shapes. Ultimately, the computation operates by iterating over the length of the longest tensor, and the question is how to _align_ the shapes so that a meaningful computation results from this. -The NumPy concept of [broadcasting](https://numpy.org/doc/stable/user/basics.broadcasting.html) is critical for flexibly defining the semantics for how functions taking two n-dimensional Tensor arguments behave when they have different shapes. If both tensors are 1D and the same length, then a simple matched iteration over both can take place. However, the broadcasting logic defines what happens when there is a systematic relationship between the two, enabling powerful (but sometimes difficult to understand) computations to be specified. +If both tensors are 1D and the same length, then a simple matched iteration over both can take place. However, the broadcasting logic defines what happens when there is a systematic relationship between the two, enabling powerful (but sometimes difficult to understand) computations to be specified. The following examples demonstrate the logic: @@ -155,13 +159,13 @@ Scale (1d array): 3 Result (3d array): 256 x 256 x 3 ``` -Anything with a dimension size of 1 will match against any other sized dimension: +Anything with a dimension size of 1 (a "singleton") will match against any other sized dimension: ``` A (4d array): 8 x 1 x 6 x 1 B (3d array): 7 x 1 x 5 Result (4d array): 8 x 7 x 6 x 5 ``` -In the innermost dimension here, the single value in A acts like a "scalar" in relationship to the 5 values in B along that same dimension, operating on each one in turn. +In the innermost dimension here, the single value in A acts like a "scalar" in relationship to the 5 values in B along that same dimension, operating on each one in turn. Likewise for the singleton second-to-last dimension in B. Any non-1 mismatch represents an error: ``` @@ -169,7 +173,7 @@ A (2d array): 2 x 1 B (3d array): 8 x 4 x 3 # second from last dimensions mismatched ``` -Computationally, the broadcast logic is straightforward to implement, in terms of computing the resulting shape. Any missing outer dimensions can be replaced with 1s, and the full 1D product index on the result shape can be applied directly to the source shapes, using the modulo operator on length so it just repeatedly samples as needed. +Computationally, the broadcast logic is straightforward to implement, in terms of computing the resulting shape. Any missing outer dimensions can be replaced with 1s, and the full 1D product index on the result shape can be applied directly to the source shapes, using the modulo operator on length so it just repeatedly samples as needed (the computation routines automatically apply this modulo operator). # History diff --git a/tensor/convert.go b/tensor/convert.go index 2bca2e26da..a3b64eaa9e 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -71,39 +71,39 @@ func RowCellSplit(tsr Values, split int) Values { // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. func NewFloat64Scalar(val float64) *Float64 { - return NewNumberFromSlice(val) + return NewNumberFromValues(val) } // NewIntScalar is a convenience method for a Tensor // representation of a single int scalar value. func NewIntScalar(val int) *Int { - return NewNumberFromSlice(val) + return NewNumberFromValues(val) } // NewStringScalar is a convenience method for a Tensor // representation of a single string scalar value. func NewStringScalar(val string) *String { - return NewStringFromSlice(val) + return NewStringFromValues(val) } -// NewFloat64FromSlice returns a new 1-dimensional tensor of given value type +// NewFloat64FromValues returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewFloat64FromSlice(vals ...float64) *Float64 { - return NewNumberFromSlice(vals...) +func NewFloat64FromValues(vals ...float64) *Float64 { + return NewNumberFromValues(vals...) } -// NewIntFromSlice returns a new 1-dimensional tensor of given value type +// NewIntFromValues returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewIntFromSlice(vals ...int) *Int { - return NewNumberFromSlice(vals...) +func NewIntFromValues(vals ...int) *Int { + return NewNumberFromValues(vals...) } -// NewStringFromSlice returns a new 1-dimensional tensor of given value type +// NewStringFromValues returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewStringFromSlice(vals ...string) *String { +func NewStringFromValues(vals ...string) *String { n := len(vals) tsr := &String{} tsr.Values = vals diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 5171d59c59..186374bb71 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -30,7 +30,7 @@ func TestFuncs(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := NewNumberFromSlice(vals...) + oned := NewNumberFromValues(vals...) oneout := oned.Clone() err = Call("Abs", oned, oneout) diff --git a/tensor/masked.go b/tensor/masked.go index f1bbce5524..2d4a37f2f4 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -12,11 +12,12 @@ import ( "cogentcore.org/core/base/reflectx" ) -// Masked is a wrapper around another [Tensor] that provides a -// bit-masked view onto the Tensor defined by a [Bool] [Values] +// Masked is a filtering wrapper around another "source" [Tensor], +// that provides a bit-masked view onto the Tensor defined by a [Bool] [Values] // tensor with a matching shape. If the bool mask has a 'false' -// then the corresponding value cannot be set and Float access returns -// NaN indicating missing data. +// then the corresponding value cannot be Set, and Float access returns +// NaN indicating missing data (other type access returns the zero value). +// A new Masked view defaults to a full transparent view of the source tensor. // To produce a new [Values] tensor with only the 'true' cases, // (i.e., the copy function of numpy), call [Masked.AsValues]. type Masked struct { //types:add @@ -29,7 +30,8 @@ type Masked struct { //types:add } // NewMasked returns a new [Masked] view of given tensor, -// with given [Bool] mask values. +// with given [Bool] mask values. If no mask is provided, +// a default full transparent (all bool values = true) mask is used. func NewMasked(tsr Tensor, mask ...*Bool) *Masked { ms := &Masked{Tensor: tsr} if len(mask) == 1 { @@ -44,7 +46,7 @@ func NewMasked(tsr Tensor, mask ...*Bool) *Masked { // AsMasked returns the tensor as a [Masked] view. // If it already is one, then it is returned, otherwise it is wrapped -// with an initially transparent mask. +// with an initially fully transparent mask. func AsMasked(tsr Tensor) *Masked { if ms, ok := tsr.(*Masked); ok { return ms @@ -52,31 +54,32 @@ func AsMasked(tsr Tensor) *Masked { return NewMasked(tsr) } -// SetTensor sets as indexes into given tensor with sequential initial indexes. +// SetTensor sets the given source tensor. If the shape does not match +// the current Mask, then a new transparent mask is established. func (ms *Masked) SetTensor(tsr Tensor) { ms.Tensor = tsr ms.SyncShape() } // SyncShape ensures that [Masked.Mask] shape is the same as source tensor. +// If the Mask does not exist or is a different shape from the source, +// then it is created or reshaped, and all values set to true ("transparent"). func (ms *Masked) SyncShape() { if ms.Mask == nil { ms.Mask = NewBoolShape(ms.Tensor.Shape()) + ms.Mask.SetTrue() return } - SetShapeFrom(ms.Mask, ms.Tensor) + if !ms.Mask.Shape().IsEqual(ms.Tensor.Shape()) { + SetShapeFrom(ms.Mask, ms.Tensor) + ms.Mask.SetTrue() + } } -// Label satisfies the core.Labeler interface for a summary description of the tensor. -func (ms *Masked) Label() string { - return label(ms.Metadata().Name(), ms.Shape()) -} +func (ms *Masked) Label() string { return label(ms.Metadata().Name(), ms.Shape()) } -// String satisfies the fmt.Stringer interface for string of tensor data. func (ms *Masked) String() string { return sprint(ms, 0) } -// Metadata returns the metadata for this tensor, which can be used -// to encode plotting options, etc. func (ms *Masked) Metadata() *metadata.Data { return ms.Tensor.Metadata() } func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } @@ -87,13 +90,10 @@ func (ms *Masked) ShapeSizes() []int { return ms.Tensor.ShapeSizes() } func (ms *Masked) Shape() *Shape { return ms.Tensor.Shape() } -// Len returns the total number of elements in our view of the tensor. func (ms *Masked) Len() int { return ms.Tensor.Len() } -// NumDims returns the total number of dimensions. func (ms *Masked) NumDims() int { return ms.Tensor.NumDims() } -// DimSize returns the effective view size of given dimension. func (ms *Masked) DimSize(dim int) int { return ms.Tensor.DimSize(dim) } // AsValues returns a copy of this tensor as raw [Values]. @@ -112,7 +112,7 @@ func (ms *Masked) AsValues() Values { } vals = append(vals, ms.Tensor.String1D(i)) } - return NewStringFromSlice(vals...) + return NewStringFromValues(vals...) case reflectx.KindIsFloat(dt): vals := make([]float64, 0, n) for i := range n { @@ -121,7 +121,7 @@ func (ms *Masked) AsValues() Values { } vals = append(vals, ms.Tensor.Float1D(i)) } - return NewFloat64FromSlice(vals...) + return NewFloat64FromValues(vals...) default: vals := make([]int, 0, n) for i := range n { @@ -130,12 +130,31 @@ func (ms *Masked) AsValues() Values { } vals = append(vals, ms.Tensor.Int1D(i)) } - return NewIntFromSlice(vals...) + return NewIntFromValues(vals...) } } -/////////////////////////////////////////////// -// Masked access +// SourceIndexes returns a flat [Int] tensor of the mask values +// that match the given getTrue argument state. +// These can be used as indexes in the [Indexed] view, for example. +// The resulting tensor is 2D with inner dimension = number of source +// tensor dimensions, to hold the indexes, and outer dimension = number +// of indexes. +func (ms *Masked) SourceIndexes(getTrue bool) *Int { + n := ms.Len() + nd := ms.Tensor.NumDims() + idxs := make([]int, 0, n*nd) + for i := range n { + if ms.Mask.Bool1D(i) != getTrue { + continue + } + ix := ms.Tensor.Shape().IndexFrom1D(i) + idxs = append(idxs, ix...) + } + it := NewIntFromValues(idxs...) + it.SetShapeSizes(len(idxs)/nd, nd) + return it +} ///////////////////// Floats diff --git a/tensor/number.go b/tensor/number.go index 8478bc048b..21b174f719 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -84,10 +84,10 @@ func NewNumberShape[T num.Number](shape *Shape) *Number[T] { // todo: this should in principle work with yaegi:add but it is crashing // will come back to it later. -// NewNumberFromSlice returns a new 1-dimensional tensor of given value type +// NewNumberFromValues returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. -func NewNumberFromSlice[T num.Number](vals ...T) *Number[T] { +func NewNumberFromValues[T num.Number](vals ...T) *Number[T] { n := len(vals) tsr := &Number[T]{} tsr.Values = vals diff --git a/tensor/shape.go b/tensor/shape.go index 430ae71bca..f6018b7a8e 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -46,7 +46,7 @@ func (sh *Shape) SetShape(shp Tensor) { // AsTensor returns shape sizes as an Int Tensor. func (sh *Shape) AsTensor() *Int { - return NewIntFromSlice(sh.Sizes...) + return NewIntFromValues(sh.Sizes...) } // CopyShape copies the shape parameters from another Shape struct. diff --git a/tensor/sliced.go b/tensor/sliced.go index 46d3e3fa29..a45151ec1f 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -22,7 +22,7 @@ import ( // to the original tensor (filtered subsets, reversals, sorting, etc). // This view is not memory-contiguous and does not support the [RowMajor] // interface or efficient access to inner-dimensional subspaces. -// A new Sliced view defaults to a full transparent view of the ource tensor. +// A new Sliced view defaults to a full transparent view of the source tensor. // There is additional cost for every access operation associated with the // indexed indirection, and access is always via the full n-dimensional indexes. // See also [Rows] for a version that only indexes the outermost row dimension, @@ -44,6 +44,7 @@ type Sliced struct { //types:add // NewSlicedIndexes returns a new [Sliced] view of given tensor, // with optional list of indexes for each dimension (none / nil = sequential). +// Any dimensions without indexes default to nil = full sequential view. func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { sl := &Sliced{Tensor: tsr, Indexes: idxs} sl.ValidIndexes() @@ -52,6 +53,7 @@ func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { // NewSliced returns a new [Sliced] view of given tensor, // with given slices for each dimension (none / nil = sequential). +// Any dimensions without indexes default to nil = full sequential view. func NewSliced(tsr Tensor, sls ...Slice) *Sliced { ns := len(sls) if ns == 0 { @@ -67,7 +69,8 @@ func NewSliced(tsr Tensor, sls ...Slice) *Sliced { } // AsSliced returns the tensor as a [Sliced] view. -// If it already is one, then it is returned, otherwise it is wrapped. +// If it already is one, then it is returned, otherwise it is wrapped +// in a new Sliced, with default full sequential ("transparent") view. func AsSliced(tsr Tensor) *Sliced { if sl, ok := tsr.(*Sliced); ok { return sl diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index c2f5a7310a..aa78ad7603 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -21,8 +21,8 @@ func TestFuncs(t *testing.T) { tol := 1.0e-8 - atsr := tensor.NewNumberFromSlice(a64...) - btsr := tensor.NewNumberFromSlice(b64...) + atsr := tensor.NewNumberFromValues(a64...) + btsr := tensor.NewNumberFromValues(b64...) out := tensor.NewFloat64(1) EuclideanFunc(atsr, btsr, out) diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index 2ab67091b4..a4b2473c43 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -14,7 +14,7 @@ import ( func TestFuncs64(t *testing.T) { vals := []float64{0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1} - ix := tensor.NewNumberFromSlice(vals...) + ix := tensor.NewNumberFromValues(vals...) out := tensor.NewFloat64(1) results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} @@ -89,7 +89,7 @@ func TestFuncs64(t *testing.T) { func TestFuncsInt(t *testing.T) { vals := []int{0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100} - tsr := tensor.NewNumberFromSlice(vals...) + tsr := tensor.NewNumberFromValues(vals...) ix := tensor.NewRows(tsr) out := tensor.NewInt(1) diff --git a/tensor/tensor.go b/tensor/tensor.go index ce3b530efd..009f811dbd 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -39,7 +39,7 @@ var MaxSprintLength = 1000 // as all of the data analysis and plot packages skip NaNs. // View Tensor types provide different ways of viewing a source tensor, // including [Sliced] for arbitrary slices of dimension indexes, -// [Masked] for boolean masked access and setting of individual indexes, +// [Masked] for boolean masked access of individual elements, // and [Indexed] for arbitrary indexes of values, organized into the // shape of the indexes, not the original source data. // The [Rows] view provides an optimized row-indexed view for [table.Table] data. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index a1ac205078..b2c59293ac 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -207,7 +207,7 @@ func TestIndexed(t *testing.T) { ft.SetFloat(float64(v), y, x) } } - ixs := NewIntFromSlice([]int{ + ixs := NewIntFromValues( 0, 1, 0, 1, 0, 2, @@ -216,7 +216,7 @@ func TestIndexed(t *testing.T) { 1, 1, 2, 2, 2, 2, - }...) + ) ixs.SetShapeSizes(2, 2, 2, 2) ix := NewIndexed(ft, ixs) diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 42bd1958f3..0f317af81d 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -32,7 +32,7 @@ func TestMath(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := tensor.NewNumberFromSlice(vals...) + oned := tensor.NewNumberFromValues(vals...) oneout := oned.Clone() cell2d := tensor.NewFloat32(5, 2, 6) diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 02bc0db7d3..e67b670196 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -20,7 +20,7 @@ func TestAdd(t *testing.T) { vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} - oned := tensor.NewNumberFromSlice(vals...) + oned := tensor.NewNumberFromValues(vals...) oneout := oned.Clone() cell2d := tensor.NewFloat32(5, 2, 6) From 549f35c6ac97396f8aa88575945d40b08461d01b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 14:55:44 -0700 Subject: [PATCH 095/311] reshaped view added -- key for computation --- tensor/README.md | 2 +- tensor/base.go | 10 +--- tensor/bool.go | 10 +--- tensor/broadcast.go | 5 -- tensor/indexed.go | 24 +++----- tensor/masked.go | 29 ++++----- tensor/number.go | 2 +- tensor/reshaped.go | 134 ++++++++++++++++++++++++++++++++++++++++++ tensor/rowmajor.go | 4 +- tensor/rows.go | 10 +--- tensor/shape.go | 14 ++--- tensor/sliced.go | 21 +++---- tensor/string.go | 2 +- tensor/tensor.go | 4 +- tensor/tensor_test.go | 29 +++++++++ tensor/values.go | 10 ++-- 16 files changed, 215 insertions(+), 95 deletions(-) delete mode 100644 tensor/broadcast.go create mode 100644 tensor/reshaped.go diff --git a/tensor/README.md b/tensor/README.md index 31cf3dc715..4c9f1934c1 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -173,7 +173,7 @@ A (2d array): 2 x 1 B (3d array): 8 x 4 x 3 # second from last dimensions mismatched ``` -Computationally, the broadcast logic is straightforward to implement, in terms of computing the resulting shape. Any missing outer dimensions can be replaced with 1s, and the full 1D product index on the result shape can be applied directly to the source shapes, using the modulo operator on length so it just repeatedly samples as needed (the computation routines automatically apply this modulo operator). +Computationally, the broadcast logic is straightforward to implement, in terms of computing the resulting shape. Any missing outer dimensions are implicitly 1s, and any singleton dimension wraps around during the indexing process. # History diff --git a/tensor/base.go b/tensor/base.go index d1c074b0bb..7341627a1a 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -33,14 +33,6 @@ func (tsr *Base[T]) Shape() *Shape { return &tsr.shape } // This is the preferred access for Go code. func (tsr *Base[T]) ShapeSizes() []int { return slices.Clone(tsr.shape.Sizes) } -// SetShape sets the dimension sizes as 1D int values from given tensor. -// The backing storage is resized appropriately, retaining all existing data that fits. -func (tsr *Base[T]) SetShape(sizes Tensor) { - tsr.shape.SetShape(sizes) - nln := tsr.Len() - tsr.Values = slicesx.SetLength(tsr.Values, nln) -} - // SetShapeSizes sets the dimension sizes of the tensor, and resizes // backing storage appropriately, retaining all existing data that fits. func (tsr *Base[T]) SetShapeSizes(sizes ...int) { @@ -96,7 +88,7 @@ func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[i] = val } // view is implementation of View -- needs final casting to tensor type. func (tsr *Base[T]) view() *Base[T] { nw := &Base[T]{} - nw.shape.CopyShape(&tsr.shape) + nw.shape.CopyFrom(&tsr.shape) nw.Values = tsr.Values nw.Meta = tsr.Meta return nw diff --git a/tensor/bool.go b/tensor/bool.go index 6d96fdb06f..de5c8c8781 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -37,7 +37,7 @@ func NewBool(sizes ...int) *Bool { // using given shape. func NewBoolShape(shape *Shape) *Bool { tsr := &Bool{} - tsr.shape.CopyShape(shape) + tsr.shape.CopyFrom(shape) tsr.Values = bitslice.Make(tsr.Len(), 0) return tsr } @@ -108,12 +108,6 @@ func (tsr *Bool) RowCellSize() (rows, cells int) { return tsr.shape.RowCellSize() } -func (tsr *Bool) SetShape(sizes Tensor) { - tsr.shape.SetShape(sizes) - nln := tsr.Len() - tsr.Values.SetLen(nln) -} - func (tsr *Bool) SetShapeSizes(sizes ...int) { tsr.shape.SetShapeSizes(sizes...) nln := tsr.Len() @@ -299,7 +293,7 @@ func (tsr *Bool) Clone() Values { func (tsr *Bool) View() Values { nw := &Bool{} - nw.shape.CopyShape(&tsr.shape) + nw.shape.CopyFrom(&tsr.shape) nw.Values = tsr.Values nw.Meta = tsr.Meta return nw diff --git a/tensor/broadcast.go b/tensor/broadcast.go deleted file mode 100644 index f5543518e0..0000000000 --- a/tensor/broadcast.go +++ /dev/null @@ -1,5 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package tensor diff --git a/tensor/indexed.go b/tensor/indexed.go index 0e4966a10e..96ee963216 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -79,29 +79,21 @@ func (ix *Indexed) SourceIndexesFrom1D(oned int) []int { return ix.Indexes.Values[oned : oned+nd] } -func (ix *Indexed) Label() string { return label(ix.Metadata().Name(), ix.Shape()) } - -func (ix *Indexed) String() string { return sprint(ix, 0) } - +func (ix *Indexed) Label() string { return label(ix.Metadata().Name(), ix.Shape()) } +func (ix *Indexed) String() string { return sprint(ix, 0) } func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } - -func (ix *Indexed) IsString() bool { return ix.Tensor.IsString() } - -func (ix *Indexed) DataType() reflect.Kind { return ix.Tensor.DataType() } +func (ix *Indexed) IsString() bool { return ix.Tensor.IsString() } +func (ix *Indexed) DataType() reflect.Kind { return ix.Tensor.DataType() } +func (ix *Indexed) Shape() *Shape { return NewShape(ix.ShapeSizes()...) } +func (ix *Indexed) Len() int { return ix.Shape().Len() } +func (ix *Indexed) NumDims() int { return ix.Indexes.NumDims() - 1 } +func (ix *Indexed) DimSize(dim int) int { return ix.Indexes.DimSize(dim) } func (ix *Indexed) ShapeSizes() []int { si := slices.Clone(ix.Indexes.ShapeSizes()) return si[:len(si)-1] // exclude last dim } -func (ix *Indexed) Shape() *Shape { return NewShape(ix.ShapeSizes()...) } - -func (ix *Indexed) Len() int { return ix.Shape().Len() } - -func (ix *Indexed) NumDims() int { return ix.Indexes.NumDims() - 1 } - -func (ix *Indexed) DimSize(dim int) int { return ix.Indexes.DimSize(dim) } - // AsValues returns a copy of this tensor as raw [Values]. // This "renders" the Indexed view into a fully contiguous // and optimized memory representation of that view, which will be faster diff --git a/tensor/masked.go b/tensor/masked.go index 2d4a37f2f4..0cf1e6f776 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -22,7 +22,7 @@ import ( // (i.e., the copy function of numpy), call [Masked.AsValues]. type Masked struct { //types:add - // Tensor that we are a masked view onto. + // Tensor source that we are a masked view onto. Tensor Tensor // Bool tensor with same shape as source tensor, providing mask. @@ -76,25 +76,16 @@ func (ms *Masked) SyncShape() { } } -func (ms *Masked) Label() string { return label(ms.Metadata().Name(), ms.Shape()) } - -func (ms *Masked) String() string { return sprint(ms, 0) } - +func (ms *Masked) Label() string { return label(ms.Metadata().Name(), ms.Shape()) } +func (ms *Masked) String() string { return sprint(ms, 0) } func (ms *Masked) Metadata() *metadata.Data { return ms.Tensor.Metadata() } - -func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } - -func (ms *Masked) DataType() reflect.Kind { return ms.Tensor.DataType() } - -func (ms *Masked) ShapeSizes() []int { return ms.Tensor.ShapeSizes() } - -func (ms *Masked) Shape() *Shape { return ms.Tensor.Shape() } - -func (ms *Masked) Len() int { return ms.Tensor.Len() } - -func (ms *Masked) NumDims() int { return ms.Tensor.NumDims() } - -func (ms *Masked) DimSize(dim int) int { return ms.Tensor.DimSize(dim) } +func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } +func (ms *Masked) DataType() reflect.Kind { return ms.Tensor.DataType() } +func (ms *Masked) ShapeSizes() []int { return ms.Tensor.ShapeSizes() } +func (ms *Masked) Shape() *Shape { return ms.Tensor.Shape() } +func (ms *Masked) Len() int { return ms.Tensor.Len() } +func (ms *Masked) NumDims() int { return ms.Tensor.NumDims() } +func (ms *Masked) DimSize(dim int) int { return ms.Tensor.DimSize(dim) } // AsValues returns a copy of this tensor as raw [Values]. // This "renders" the Masked view into a fully contiguous diff --git a/tensor/number.go b/tensor/number.go index 21b174f719..07580758af 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -76,7 +76,7 @@ func NewNumber[T num.Number](sizes ...int) *Number[T] { // using given shape. func NewNumberShape[T num.Number](shape *Shape) *Number[T] { tsr := &Number[T]{} - tsr.shape.CopyShape(shape) + tsr.shape.CopyFrom(shape) tsr.Values = make([]T, tsr.Len()) return tsr } diff --git a/tensor/reshaped.go b/tensor/reshaped.go new file mode 100644 index 0000000000..c8e76beb04 --- /dev/null +++ b/tensor/reshaped.go @@ -0,0 +1,134 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "reflect" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/metadata" +) + +// Reshaped is a reshaping wrapper around another "source" [Tensor], +// that provides a length-preserving reshaped view onto the source Tensor. +// Reshaping by adding new size=1 dimensions (via [NewAxis] value) is +// often important for properly aligning two tensors in a computationally +// compatible manner; see the [AlignShapes] function. +// [Reshaped.AsValues] on this view returns a new [Values] with the view +// shape, calling [Clone] on the source tensor to get the values. +type Reshaped struct { //types:add + + // Tensor source that we are a masked view onto. + Tensor Tensor + + // Reshape is the effective shape we use for access. + // This must have the same Len() as the source Tensor. + Reshape Shape +} + +// NewReshaped returns a new [Reshaped] view of given tensor, +// with given shape sizes. If no such sizes are provided, +// the source shape is used. +func NewReshaped(tsr Tensor, sizes ...int) *Reshaped { + rs := &Reshaped{Tensor: tsr} + if len(sizes) == 0 { + rs.Reshape.CopyFrom(tsr.Shape()) + } else { + errors.Log(rs.SetShapeSizes(sizes...)) + } + return rs +} + +// AsReshaped returns the tensor as a [Reshaped] view. +// If it already is one, then it is returned, otherwise it is wrapped +// with an initial shape equal to the source tensor. +func AsReshaped(tsr Tensor) *Reshaped { + if rs, ok := tsr.(*Reshaped); ok { + return rs + } + return NewReshaped(tsr) +} + +// NewAxis can be used in [Reshaped.SetShapeSizes] to indicate where a +// new dimension (axis) is being added relative to the source shape. +const NewAxis = 1 + +// SetShapeSizes sets our shape sizes to the given values, which must result in +// the same length as the source tensor. An error is returned if not. +// If a different subset of content is desired, use another view such as [Sliced]. +// Note that any number of size = 1 dimensions can be added without affecting +// the length, and the [NewAxis] value can be used to semantically +// indicate when such a new dimension is being inserted. This is often useful +// for aligning two tensors to achieve a desired computation; see [AlignShapes] +// function. +func (rs *Reshaped) SetShapeSizes(sizes ...int) error { + rs.Reshape.SetShapeSizes(sizes...) + if rs.Reshape.Len() != rs.Tensor.Len() { + return errors.New("tensor.Reshaped SetShapeSizes: new length is different from source tensor; use Sliced or other views to change view content") + } + return nil +} + +func (rs *Reshaped) Label() string { return label(rs.Metadata().Name(), rs.Shape()) } +func (rs *Reshaped) String() string { return sprint(rs, 0) } +func (rs *Reshaped) Metadata() *metadata.Data { return rs.Tensor.Metadata() } +func (rs *Reshaped) IsString() bool { return rs.Tensor.IsString() } +func (rs *Reshaped) DataType() reflect.Kind { return rs.Tensor.DataType() } +func (rs *Reshaped) ShapeSizes() []int { return rs.Reshape.Sizes } +func (rs *Reshaped) Shape() *Shape { return &rs.Reshape } +func (rs *Reshaped) Len() int { return rs.Reshape.Len() } +func (rs *Reshaped) NumDims() int { return rs.Reshape.NumDims() } +func (rs *Reshaped) DimSize(dim int) int { return rs.Reshape.DimSize(dim) } + +// AsValues returns a copy of this tensor as raw [Values], with +// the same shape as our view. This calls [Clone] on the source +// tensor to get the Values and then sets our shape sizes to it. +func (rs *Reshaped) AsValues() Values { + vals := Clone(rs.Tensor) + vals.SetShapeSizes(rs.Reshape.Sizes...) + return vals +} + +///////////////////// Floats + +func (rs *Reshaped) Float(i ...int) float64 { + return rs.Tensor.Float1D(rs.Reshape.IndexTo1D(i...)) +} + +func (rs *Reshaped) SetFloat(val float64, i ...int) { + rs.Tensor.SetFloat1D(val, rs.Reshape.IndexTo1D(i...)) +} + +func (rs *Reshaped) Float1D(i int) float64 { return rs.Tensor.Float1D(i) } +func (rs *Reshaped) SetFloat1D(val float64, i int) { rs.Tensor.SetFloat1D(val, i) } + +///////////////////// Strings + +func (rs *Reshaped) StringValue(i ...int) string { + return rs.Tensor.String1D(rs.Reshape.IndexTo1D(i...)) +} + +func (rs *Reshaped) SetString(val string, i ...int) { + rs.Tensor.SetString1D(val, rs.Reshape.IndexTo1D(i...)) +} + +func (rs *Reshaped) String1D(i int) string { return rs.Tensor.String1D(i) } +func (rs *Reshaped) SetString1D(val string, i int) { rs.Tensor.SetString1D(val, i) } + +///////////////////// Ints + +func (rs *Reshaped) Int(i ...int) int { + return rs.Tensor.Int1D(rs.Reshape.IndexTo1D(i...)) +} + +func (rs *Reshaped) SetInt(val int, i ...int) { + rs.Tensor.SetInt1D(val, rs.Reshape.IndexTo1D(i...)) +} + +func (rs *Reshaped) Int1D(i int) int { return rs.Tensor.Int1D(i) } +func (rs *Reshaped) SetInt1D(val int, i int) { rs.Tensor.SetInt1D(val, i) } + +// check for interface impl +var _ Tensor = (*Reshaped)(nil) diff --git a/tensor/rowmajor.go b/tensor/rowmajor.go index e254685c58..2e331fdaef 100644 --- a/tensor/rowmajor.go +++ b/tensor/rowmajor.go @@ -4,13 +4,13 @@ package tensor -// RowMajor is subclass of [Tensor] that maintains a row major memory organization +// RowMajor is subtype of [Tensor] that maintains a row major memory organization // that thereby supports efficient access via the outermost 'row' dimension, // with all remaining inner dimensions comprising the 'cells' of data per row // (1 scalar value in the case of a 1D tensor). // It is implemented by raw [Values] tensors, and the [Rows] indexed view of // raw Values tensors. Other views however do not retain the underlying -// outer to inner row major memory structure and thus do not benefit from this interface. +// outer to inner row major memory structure and thus do not implement this interface. type RowMajor interface { Tensor diff --git a/tensor/rows.go b/tensor/rows.go index 0732fcf999..411c2b75ea 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -86,11 +86,10 @@ func (rw *Rows) NumRows() int { return len(rw.Indexes) } -func (rw *Rows) String() string { return sprint(rw.Tensor, 0) } - -func (rw *Rows) Label() string { return rw.Tensor.Label() } - +func (rw *Rows) String() string { return sprint(rw.Tensor, 0) } +func (rw *Rows) Label() string { return rw.Tensor.Label() } func (rw *Rows) Metadata() *metadata.Data { return rw.Tensor.Metadata() } +func (rw *Rows) NumDims() int { return rw.Tensor.NumDims() } // If we have Indexes, this is the effective shape sizes using // the current number of indexes as the outermost row dimension size. @@ -122,9 +121,6 @@ func (rw *Rows) Len() int { return cells * rows } -// NumDims returns the total number of dimensions. -func (rw *Rows) NumDims() int { return rw.Tensor.NumDims() } - // DimSize returns size of given dimension, returning NumRows() // for first dimension. func (rw *Rows) DimSize(dim int) int { diff --git a/tensor/shape.go b/tensor/shape.go index f6018b7a8e..e9a6dcaa30 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -38,20 +38,20 @@ func (sh *Shape) SetShapeSizes(sizes ...int) { sh.Strides = RowMajorStrides(sizes...) } -// SetShape sets the shape sizes from int values from Tensor. +// SetShapeSizesFromTensor sets the shape sizes from given tensor. // RowMajor ordering is used by default. -func (sh *Shape) SetShape(shp Tensor) { - sh.SetShapeSizes(AsIntSlice(shp)...) +func (sh *Shape) SetShapeSizesFromTensor(sizes Tensor) { + sh.SetShapeSizes(AsIntSlice(sizes)...) } -// AsTensor returns shape sizes as an Int Tensor. -func (sh *Shape) AsTensor() *Int { +// SizesAsTensor returns shape sizes as an Int Tensor. +func (sh *Shape) SizesAsTensor() *Int { return NewIntFromValues(sh.Sizes...) } -// CopyShape copies the shape parameters from another Shape struct. +// CopyFrom copies the shape parameters from another Shape struct. // copies the data so it is not accidentally subject to updates. -func (sh *Shape) CopyShape(cp *Shape) { +func (sh *Shape) CopyFrom(cp *Shape) { sh.Sizes = slices.Clone(cp.Sizes) sh.Strides = slices.Clone(cp.Strides) } diff --git a/tensor/sliced.go b/tensor/sliced.go index a45151ec1f..3b5a9d9773 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -162,15 +162,14 @@ func (sl *Sliced) IndexesNeeded(d int) { sl.Indexes[d] = ix } -func (sl *Sliced) Label() string { return label(sl.Metadata().Name(), sl.Shape()) } - -func (sl *Sliced) String() string { return sprint(sl, 0) } - +func (sl *Sliced) Label() string { return label(sl.Metadata().Name(), sl.Shape()) } +func (sl *Sliced) String() string { return sprint(sl, 0) } func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } - -func (sl *Sliced) IsString() bool { return sl.Tensor.IsString() } - -func (sl *Sliced) DataType() reflect.Kind { return sl.Tensor.DataType() } +func (sl *Sliced) IsString() bool { return sl.Tensor.IsString() } +func (sl *Sliced) DataType() reflect.Kind { return sl.Tensor.DataType() } +func (sl *Sliced) Shape() *Shape { return NewShape(sl.ShapeSizes()...) } +func (sl *Sliced) Len() int { return sl.Shape().Len() } +func (sl *Sliced) NumDims() int { return sl.Tensor.NumDims() } // For each dimension, we return the effective shape sizes using // the current number of indexes per dimension. @@ -188,12 +187,6 @@ func (sl *Sliced) ShapeSizes() []int { return sh } -func (sl *Sliced) Shape() *Shape { return NewShape(sl.ShapeSizes()...) } - -func (sl *Sliced) Len() int { return sl.Shape().Len() } - -func (sl *Sliced) NumDims() int { return sl.Tensor.NumDims() } - // DimSize returns the effective view size of given dimension. func (sl *Sliced) DimSize(dim int) int { if sl.Indexes[dim] != nil { diff --git a/tensor/string.go b/tensor/string.go index c3bffd20ca..3c060e938d 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -29,7 +29,7 @@ func NewString(sizes ...int) *String { // using given shape. func NewStringShape(shape *Shape) *String { tsr := &String{} - tsr.shape.CopyShape(shape) + tsr.shape.CopyFrom(shape) tsr.Values = make([]string, tsr.Len()) return tsr } diff --git a/tensor/tensor.go b/tensor/tensor.go index 009f811dbd..2aab1e56c5 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -42,7 +42,9 @@ var MaxSprintLength = 1000 // [Masked] for boolean masked access of individual elements, // and [Indexed] for arbitrary indexes of values, organized into the // shape of the indexes, not the original source data. -// The [Rows] view provides an optimized row-indexed view for [table.Table] data. +// [Reshaped] provides length preserving reshaping (mostly for computational +// alignment purposes), and [Rows] provides an optimized row-indexed +// view for [table.Table] data. type Tensor interface { fmt.Stringer diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index b2c59293ac..36db0a7860 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -232,6 +232,35 @@ func TestIndexed(t *testing.T) { assert.Equal(t, res, vl.String()) } +func TestReshaped(t *testing.T) { + ft := NewFloat64(3, 4) + for y := range 3 { + for x := range 4 { + v := y*10 + x + ft.SetFloat(float64(v), y, x) + } + } + + res := `[4, 3] +[0]: 0 1 2 +[1]: 3 10 11 +[2]: 12 13 20 +[3]: 21 22 23 +` + rs := NewReshaped(ft, 4, 3) + // fmt.Println(rs) + assert.Equal(t, res, rs.String()) + + res = `[1, 3, 4] +[0]: 0 1 2 3 +[0]: 10 11 12 13 +[0]: 20 21 22 23 +` + rs = NewReshaped(ft, NewAxis, 3, 4) + // fmt.Println(rs) + assert.Equal(t, res, rs.String()) +} + func TestSortFilter(t *testing.T) { tsr := NewRows(NewFloat64(5)) for i := range 5 { diff --git a/tensor/values.go b/tensor/values.go index f874095760..ed641ec941 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -18,10 +18,6 @@ import ( type Values interface { RowMajor - // SetShape sets the dimension sizes as 1D int values from given tensor. - // The backing storage is resized appropriately, retaining all existing data that fits. - SetShape(sizes Tensor) - // SetShapeSizes sets the dimension sizes of the tensor, and resizes // backing storage appropriately, retaining all existing data that fits. SetShapeSizes(sizes ...int) @@ -133,3 +129,9 @@ func SetShapeNames(md *metadata.Data, names ...string) { func ShapeNames(md *metadata.Data) []string { return errors.Log1(metadata.Get[[]string](*md, "ShapeNames")) } + +// SetShapeSizesFromTensor sets the dimension sizes as 1D int values from given tensor. +// The backing storage is resized appropriately, retaining all existing data that fits. +func SetShapeSizesFromTensor(vals Values, sizes Tensor) { + vals.SetShapeSizes(AsIntSlice(sizes)...) +} From c89aa9aaa9154ef372b4db79a13ca666e01f4b0a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 16:14:13 -0700 Subject: [PATCH 096/311] math ops working with new logic! --- tensor/align.go | 71 ++++++++ tensor/convert.go | 54 +++++- tensor/shape.go | 6 +- tensor/stats/stats/README.md | 7 + tensor/{tmath => stats/stats}/norm.go | 17 +- tensor/stats/stats/stats_test.go | 37 ++++ tensor/tmath/README.md | 5 - tensor/tmath/math.go | 149 ++++++++++++---- tensor/tmath/math_test.go | 2 +- tensor/tmath/ops.go | 240 +++++--------------------- tensor/tmath/ops_test.go | 33 +--- tensor/values.go | 6 - 12 files changed, 327 insertions(+), 300 deletions(-) create mode 100644 tensor/align.go rename tensor/{tmath => stats/stats}/norm.go (87%) diff --git a/tensor/align.go b/tensor/align.go new file mode 100644 index 0000000000..70c7f0396f --- /dev/null +++ b/tensor/align.go @@ -0,0 +1,71 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "fmt" + "slices" +) + +// AlignShapes aligns the shapes of two tensors, a and b for a binary +// computation producing an output, returning the effective aligned shapes +// for a, b, and the output, all with the same number of dimensions. +// Alignment proceeds from the innermost dimension out, with 1s provided +// beyond the number of dimensions for a or b. +// The output has the max of the dimension sizes for each dimension. +// An error is returned if the rules of alignment are violated: +// each dimension size must be either the same, or one of them +// is equal to 1. This corresponds to the "broadcasting" logic of NumPy. +func AlignShapes(a, b Tensor) (as, bs, os *Shape, err error) { + asz := a.ShapeSizes() + bsz := b.ShapeSizes() + an := len(asz) + bn := len(bsz) + n := max(an, bn) + osizes := make([]int, n) + asizes := make([]int, n) + bsizes := make([]int, n) + for d := range n { + ai := an - 1 - d + bi := bn - 1 - d + oi := n - 1 - d + ad := 1 + bd := 1 + if ai >= 0 { + ad = asz[ai] + } + if bi >= 0 { + bd = bsz[bi] + } + if ad != bd && !(ad == 1 || bd == 1) { + err = fmt.Errorf("tensor.AlignShapes: output dimension %d does not align for a=%d b=%d: must be either the same or one of them is a 1", oi, ad, bd) + return + } + od := max(ad, bd) + osizes[oi] = od + asizes[oi] = ad + bsizes[oi] = bd + } + as = NewShape(asizes...) + bs = NewShape(bsizes...) + os = NewShape(osizes...) + return +} + +// WrapIndex1D returns the 1d flat index for given n-dimensional index +// based on given shape, where any singleton dimension sizes cause the +// resulting index value to remain at 0, effectively causing that dimension +// to wrap around, while the other tensor is presumably using the full range +// of the values along this dimension. See [AlignShapes] for more info. +func WrapIndex1D(sh *Shape, i ...int) int { + nd := sh.NumDims() + ai := slices.Clone(i) + for d := range nd { + if sh.DimSize(d) == 1 { + ai[d] = 0 + } + } + return sh.IndexTo1D(ai...) +} diff --git a/tensor/convert.go b/tensor/convert.go index a3b64eaa9e..3154a7661a 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -16,16 +16,56 @@ func Clone(tsr Tensor) Values { return tsr.AsValues() } -// SetShapeFrom sets shape of given tensor from a source tensor. -// This will return an error if the destination tensor is not a Values type. -// This is used extensively for output tensors in functions, and all such -// output tensors _must_ be Values tensors. -func SetShapeFrom(tsr, from Tensor) error { +// MustBeValues returns the given tensor as a [Values] subtype, or nil and +// an error if it is not one. Typically outputs of compute operations must +// be values, and are reshaped to hold the results as needed. +func MustBeValues(tsr Tensor) (Values, error) { vl, ok := tsr.(Values) if !ok { - return errors.Log(errors.New("tensor.SetShapeFrom: tensor must be a Values type to have shape modified. All function output tensors must be Values!")) + return nil, errors.New("tensor.MustBeValues: tensor must be a Values type") + } + return vl, nil +} + +// SetShape sets the dimension sizes from given Shape +func SetShape(vals Values, sh *Shape) { + vals.SetShapeSizes(sh.Sizes...) +} + +// SetShapeMustBeValues sets the dimension sizes from given Shape, +// calling [MustBeValues] on the destination tensor to ensure it is a [Values], +// type, returning an error if not. This is used extensively for output +// tensors in functions, and all such output tensors _must_ be Values tensors. +func SetShapeMustBeValues(tsr Tensor, sh *Shape) error { + vals, err := MustBeValues(tsr) + if err != nil { + return err + } + vals.SetShapeSizes(sh.Sizes...) + return nil +} + +// SetShapeSizesFromTensor sets the dimension sizes as 1D int values from given tensor. +// The backing storage is resized appropriately, retaining all existing data that fits. +func SetShapeSizesFromTensor(vals Values, sizes Tensor) { + vals.SetShapeSizes(AsIntSlice(sizes)...) +} + +// SetShapeFrom sets shape of given tensor from a source tensor. +func SetShapeFrom(vals Values, from Tensor) { + vals.SetShapeSizes(from.ShapeSizes()...) +} + +// SetShapeFromMustBeValues sets shape of given tensor from a source tensor, +// calling [MustBeValues] on the destination tensor to ensure it is a [Values], +// type, returning an error if not. This is used extensively for output +// tensors in functions, and all such output tensors _must_ be Values tensors. +func SetShapeFromMustBeValues(tsr, from Tensor) error { + vals, err := MustBeValues(tsr) + if err != nil { + return err } - vl.SetShapeSizes(from.ShapeSizes()...) + vals.SetShapeSizes(from.ShapeSizes()...) return nil } diff --git a/tensor/shape.go b/tensor/shape.go index e9a6dcaa30..d061450c8f 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -74,9 +74,9 @@ func (sh *Shape) NumDims() int { return len(sh.Sizes) } // DimSize returns the size of given dimension. func (sh *Shape) DimSize(i int) int { - if sh.Sizes == nil { - return 0 - } + // if sh.Sizes == nil { + // return 0 + // } return sh.Sizes[i] } diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index a6b2ec626d..ad58bc7d09 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -48,6 +48,13 @@ All stats functions skip over NaN's, as a missing value. Stats functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. +## norm functions + +* `UnitNorm` subtracts `min` and divides by resulting `max` to normalize to 0..1 unit range. +* `ZScore` subtracts the mean and divides by the standard deviation. +* `Clamp` enforces min, max range, clamping values to those bounds if they exceed them. +* `Binarize` sets all values below a given threshold to 0, and those above to 1. + ## Groups The `Groups` function (and `TableGroups` convenience function for `table.Table` columns) creates lists of indexes for each unique value in a 1D tensor, and `GroupStats` calls a stats function on those groups, thereby creating a "pivot table" that summarizes data in terms of the groups present within it. The data is stored in a [datafs](../datafs) data filesystem, which can be visualized and further manipulated. diff --git a/tensor/tmath/norm.go b/tensor/stats/stats/norm.go similarity index 87% rename from tensor/tmath/norm.go rename to tensor/stats/stats/norm.go index f70ea10013..cf599a75d2 100644 --- a/tensor/tmath/norm.go +++ b/tensor/stats/stats/norm.go @@ -2,31 +2,30 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package tmath +package stats import ( "cogentcore.org/core/math32" "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/stats" ) // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. func ZScore(a, out tensor.Tensor) { mout := tensor.NewFloat64() - std, mean, _ := stats.StdFuncOut64(a, mout) - Sub(a, mean, out) - Div(out, std, out) + std, mean, _ := StdFuncOut64(a, mout) + tmath.Sub(a, mean, out) + tmath.Div(out, std, out) } // UnitNorm computes unit normalized values into given output tensor, // subtracting the Min value and dividing by the Max of the remaining numbers. func UnitNorm(a, out tensor.Tensor) { mout := tensor.NewFloat64() - stats.MinFunc(a, mout) - Sub(a, mout, out) - stats.MaxFunc(out, mout) - Div(out, mout, out) + MinFunc(a, mout) + tmath.Sub(a, mout, out) + MaxFunc(out, mout) + tmath.Div(out, mout, out) } // Clamp ensures that all values are within min, max limits, clamping diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index a4b2473c43..ee64b757e0 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -179,3 +179,40 @@ func TestFuncsCell(t *testing.T) { assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) } } + +func TestNorm(t *testing.T) { + vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0.1, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} + + oned := tensor.NewNumberFromValues(vals...) + oneout := oned.Clone() + + ZScore(oned, oneout) + mout := tensor.NewFloat64() + std, mean, _ := stats.StdFuncOut64(oneout, mout) + assert.InDelta(t, 1.0, std.Float1D(0), 1.0e-6) + assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) + + UnitNorm(oned, oneout) + stats.MinFunc(oneout, mout) + assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) + stats.MaxFunc(oneout, mout) + assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) + // fmt.Println(oneout) + + minv := tensor.NewFloat64Scalar(0) + maxv := tensor.NewFloat64Scalar(1) + Clamp(oned, minv, maxv, oneout) + stats.MinFunc(oneout, mout) + assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) + stats.MaxFunc(oneout, mout) + assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) + // fmt.Println(oneout) + + thr := tensor.NewFloat64Scalar(0.5) + Binarize(oned, thr, oneout) + stats.MinFunc(oneout, mout) + assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) + stats.MaxFunc(oneout, mout) + assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) + // fmt.Println(oneout) +} diff --git a/tensor/tmath/README.md b/tensor/tmath/README.md index 5748b3610a..95cda68594 100644 --- a/tensor/tmath/README.md +++ b/tensor/tmath/README.md @@ -10,9 +10,4 @@ The output result tensor(s) can be the same as the input for all functions (exce The standard `Add`, `Sub`, `Mul`, `Div` (`+, -, *, /`) mathematical operators all operate element-wise, with a separate MatMul for matrix multiplication, which operates through gonum routines, for 2D Float64 tensor shapes with no indexes, so that the raw float64 values can be passed directly to gonum. -# norm functions -* `UnitNorm` subtracts `min` and divides by resulting `max` to normalize to 0..1 unit range. -* `ZScore` subtracts the mean and divides by the standard deviation. -* `Clamp` enforces min, max range, clamping values to those bounds if they exceed them. -* `Binarize` sets all values below a given threshold to 0, and those above to 1. diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index 386efd9c97..9525b026e3 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -7,6 +7,7 @@ package tmath import ( "math" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -51,259 +52,333 @@ func init() { } func Abs(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) } func Acos(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(idx)), idx) }, in, out) } func Acosh(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(idx)), idx) }, in, out) } func Asin(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(idx)), idx) }, in, out) } func Asinh(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(idx)), idx) }, in, out) } func Atan(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(idx)), idx) }, in, out) } func Atanh(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(idx)), idx) }, in, out) } func Cbrt(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(idx)), idx) }, in, out) } func Ceil(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(idx)), idx) }, in, out) } func Cos(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(idx)), idx) }, in, out) } func Cosh(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(idx)), idx) }, in, out) } func Erf(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(idx)), idx) }, in, out) } func Erfc(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(idx)), idx) }, in, out) } func Erfcinv(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(idx)), idx) }, in, out) } func Erfinv(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(idx)), idx) }, in, out) } func Exp(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(idx)), idx) }, in, out) } func Exp2(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(idx)), idx) }, in, out) } func Expm1(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(idx)), idx) }, in, out) } func Floor(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(idx)), idx) }, in, out) } func Gamma(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(idx)), idx) }, in, out) } func J0(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(idx)), idx) }, in, out) } func J1(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(idx)), idx) }, in, out) } func Log(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(idx)), idx) }, in, out) } func Log10(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(idx)), idx) }, in, out) } func Log1p(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(idx)), idx) }, in, out) } func Log2(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(idx)), idx) }, in, out) } func Logb(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(idx)), idx) }, in, out) } func Round(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(idx)), idx) }, in, out) } func RoundToEven(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(idx)), idx) }, in, out) } func Sin(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(idx)), idx) }, in, out) } func Sinh(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(idx)), idx) }, in, out) } func Sqrt(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(idx)), idx) }, in, out) } func Tan(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(idx)), idx) }, in, out) } func Tanh(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(idx)), idx) }, in, out) } func Trunc(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(idx)), idx) }, in, out) } func Y0(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(idx)), idx) }, in, out) } func Y1(in, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) + if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(idx)), idx) }, in, out) diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 0f317af81d..cb4c810237 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -76,5 +76,5 @@ func TestOps(t *testing.T) { b := tensor.CallOut("Add", x, y) c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y), tensor.CallOut("Mul", a, b)) - assert.Equal(t, 14, c.IntRow(0)) + assert.Equal(t, 14.0, c.Float1D(0)) } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 3e04ef693c..7b12fbc7c1 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -5,6 +5,7 @@ package tmath import ( + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -16,248 +17,85 @@ func init() { } // Add adds two tensors into output. -// If one is a scalar, it is added to all elements. -// If one is the same size as the Cell SubSpace of the other -// then the SubSpace is added to each row of the other. -// Otherwise an element-wise addition is performed -// for overlapping cells. func Add(a, b, out tensor.Tensor) { - if b.Len() == 1 { - AddScalar(b.FloatRow(0), a, out) + as, bs, os, err := tensor.AlignShapes(a, b) + if errors.Log(err) != nil { return } - if a.Len() == 1 { - AddScalar(a.FloatRow(0), b, out) + if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { return } - arows, acells := a.RowCellSize() - brows, bcells := b.RowCellSize() - if brows*bcells == acells { - AddSubSpace(a, b, out) - return - } - if arows*acells == bcells { - AddSubSpace(b, a, out) - return - } - // just do element-wise - tensor.SetShapeFrom(out, a) + olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return tensor.NMinLen(2, tsr...) + return olen }, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(tsr[0].Float1D(idx)+tsr[1].Float1D(idx), idx) + oi := os.IndexFrom1D(idx) + ai := tensor.WrapIndex1D(as, oi...) + bi := tensor.WrapIndex1D(bs, oi...) + out.SetFloat1D(tsr[0].Float1D(ai)+tsr[1].Float1D(bi), idx) }, a, b, out) } -// AddScalar adds a scalar to given tensor into output. -func AddScalar(scalar float64, a, out tensor.Tensor) { - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(a.Float1D(idx)+scalar, idx) - }, a, out) -} - -// AddSubSpace adds the subspace tensor to each row in the given tensor, -// into the output tensor. -func AddSubSpace(a, sub, out tensor.Tensor) { - _, acells := a.RowCellSize() - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - si := idx % acells - out.SetFloat1D(a.Float1D(idx)+sub.Float1D(si), idx) - }, a, sub, out) -} - -//////////////////////////////////////////////////////////// -// Sub - // Sub subtracts two tensors into output. -// If one is a scalar, it is subtracted from all elements. -// If one is the same size as the Cell SubSpace of the other -// then the SubSpace is subtracted from each row of the other. -// Otherwise an element-wise subtractition is performed -// for overlapping cells. func Sub(a, b, out tensor.Tensor) { - if b.Len() == 1 { - SubScalar(1, b.FloatRow(0), a, out) - return - } - if a.Len() == 1 { - SubScalar(-1, a.FloatRow(0), b, out) - return - } - arows, acells := a.RowCellSize() - brows, bcells := b.RowCellSize() - if brows*bcells == acells { - SubSubSpace(1, a, b, out) + as, bs, os, err := tensor.AlignShapes(a, b) + if errors.Log(err) != nil { return } - if arows*acells == bcells { - SubSubSpace(-1, b, a, out) + if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { return } - // just do element-wise - tensor.SetShapeFrom(out, a) + olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return tensor.NMinLen(2, tsr...) + return olen }, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(tsr[0].Float1D(idx)-tsr[1].Float1D(idx), idx) + oi := os.IndexFrom1D(idx) + ai := tensor.WrapIndex1D(as, oi...) + bi := tensor.WrapIndex1D(bs, oi...) + out.SetFloat1D(tsr[0].Float1D(ai)-tsr[1].Float1D(bi), idx) }, a, b, out) } -// SubScalar subtracts a scalar from given tensor into output. -// sign determines which way the subtraction goes: 1 = a-scalar, -1 = scalar-a -func SubScalar(sign float64, scalar float64, a, out tensor.Tensor) { - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(sign*(a.Float1D(idx)-scalar), idx) - }, a, out) -} - -// SubSubSpace subtracts the subspace tensor to each row in the given tensor, -// into the output tensor. -// sign determines which way the subtraction goes: 1 = a-sub, -1 = sub-a -func SubSubSpace(sign float64, a, sub, out tensor.Tensor) { - _, acells := a.RowCellSize() - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - si := idx % acells - out.SetFloat1D(sign*(a.Float1D(idx)-sub.Float1D(si)), idx) - }, a, sub, out) -} - -//////////////////////////////////////////////////////////// -// Mul - -// Mul does element-wise multiplication between two tensors into output. -// If one is a scalar, it multiplies all elements. -// If one is the same size as the Cell SubSpace of the other -// then the SubSpace multiplies each row of the other. -// Otherwise an element-wise multiplication is performed -// for overlapping cells. +// Mul multiplies two tensors into output. func Mul(a, b, out tensor.Tensor) { - if b.Len() == 1 { - MulScalar(b.FloatRow(0), a, out) - return - } - if a.Len() == 1 { - MulScalar(a.FloatRow(0), b, out) + as, bs, os, err := tensor.AlignShapes(a, b) + if errors.Log(err) != nil { return } - arows, acells := a.RowCellSize() - brows, bcells := b.RowCellSize() - if brows*bcells == acells { - MulSubSpace(a, b, out) + if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { return } - if arows*acells == bcells { - MulSubSpace(b, a, out) - return - } - // just do element-wise - tensor.SetShapeFrom(out, a) + olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return tensor.NMinLen(2, tsr...) + return olen }, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(tsr[0].Float1D(idx)*tsr[1].Float1D(idx), idx) + oi := os.IndexFrom1D(idx) + ai := tensor.WrapIndex1D(as, oi...) + bi := tensor.WrapIndex1D(bs, oi...) + out.SetFloat1D(tsr[0].Float1D(ai)*tsr[1].Float1D(bi), idx) }, a, b, out) } -// MulScalar multiplies a scalar to given tensor into output. -func MulScalar(scalar float64, a, out tensor.Tensor) { - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(a.Float1D(idx)*scalar, idx) - }, a, out) -} - -// MulSubSpace multiplies the subspace tensor to each row in the given tensor, -// into the output tensor. -func MulSubSpace(a, sub, out tensor.Tensor) { - _, acells := a.RowCellSize() - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - si := idx % acells - out.SetFloat1D(a.Float1D(idx)*sub.Float1D(si), idx) - }, a, sub, out) -} - -//////////////////////////////////////////////////////////// -// Div - -// Div does element-wise division between two tensors into output. -// If one is a scalar, it divides all elements. -// If one is the same size as the Cell SubSpace of the other -// then the SubSpace divides each row of the other. -// Otherwise an element-wise division is performed -// for overlapping cells. +// Div divides two tensors into output. func Div(a, b, out tensor.Tensor) { - if b.Len() == 1 { - DivScalar(b.FloatRow(0), a, out) + as, bs, os, err := tensor.AlignShapes(a, b) + if errors.Log(err) != nil { return } - if a.Len() == 1 { - DivScalarInv(a.FloatRow(0), b, out) + if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { return } - arows, acells := a.RowCellSize() - brows, bcells := b.RowCellSize() - if brows*bcells == acells { - DivSubSpace(a, b, out) - return - } - if arows*acells == bcells { - DivSubSpaceInv(b, a, out) - return - } - // just do element-wise - tensor.SetShapeFrom(out, a) + olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return tensor.NMinLen(2, tsr...) + return olen }, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(tsr[0].Float1D(idx)/tsr[1].Float1D(idx), idx) + oi := os.IndexFrom1D(idx) + ai := tensor.WrapIndex1D(as, oi...) + bi := tensor.WrapIndex1D(bs, oi...) + out.SetFloat1D(tsr[0].Float1D(ai)/tsr[1].Float1D(bi), idx) }, a, b, out) } - -// DivScalar divides given tensor elements by scalar into output. -func DivScalar(scalar float64, a, out tensor.Tensor) { - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(a.Float1D(idx)/scalar, idx) - }, a, out) -} - -// DivScalarInv divides scalar by given tensor elements into output -// (inverse of [DivScalar]). -func DivScalarInv(scalar float64, a, out tensor.Tensor) { - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(scalar/a.Float1D(idx), idx) - }, a, out) -} - -// DivSubSpace divides each row of the given tensor by the subspace tensor elements, -// into the output tensor. -func DivSubSpace(a, sub, out tensor.Tensor) { - _, acells := a.RowCellSize() - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - si := idx % acells - out.SetFloat1D(a.Float1D(idx)/sub.Float1D(si), idx) - }, a, sub, out) -} - -// DivSubSpaceInv divides the subspace tensor by each row of the given tensor, -// into the output tensor (inverse of [DivSubSpace]) -func DivSubSpaceInv(a, sub, out tensor.Tensor) { - _, acells := a.RowCellSize() - tensor.SetShapeFrom(out, a) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - si := idx % acells - out.SetFloat1D(sub.Float1D(si)/a.Float1D(idx), idx) - }, a, sub, out) -} diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index e67b670196..585646eaf2 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -8,7 +8,6 @@ import ( "testing" "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/stats/stats" "github.com/stretchr/testify/assert" ) @@ -23,7 +22,7 @@ func TestAdd(t *testing.T) { oned := tensor.NewNumberFromValues(vals...) oneout := oned.Clone() - cell2d := tensor.NewFloat32(5, 2, 6) + cell2d := tensor.NewFloat32(5, 12) _, cells := cell2d.RowCellSize() assert.Equal(t, cells, 12) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { @@ -32,6 +31,7 @@ func TestAdd(t *testing.T) { }, cell2d) // cell2d.DeleteRows(3, 1) cellout := cell2d.Clone() + _ = cellout Add(scalar, scb, scout) assert.Equal(t, -5.5+-4, scout.Float1D(0)) @@ -129,33 +129,4 @@ func TestAdd(t *testing.T) { } } - ZScore(oned, oneout) - mout := tensor.NewFloat64() - std, mean, _ := stats.StdFuncOut64(oneout, mout) - assert.InDelta(t, 1.0, std.Float1D(0), 1.0e-6) - assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) - - UnitNorm(oned, oneout) - stats.MinFunc(oneout, mout) - assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - stats.MaxFunc(oneout, mout) - assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) - // fmt.Println(oneout) - - minv := tensor.NewFloat64Scalar(0) - maxv := tensor.NewFloat64Scalar(1) - Clamp(oned, minv, maxv, oneout) - stats.MinFunc(oneout, mout) - assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - stats.MaxFunc(oneout, mout) - assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) - // fmt.Println(oneout) - - thr := tensor.NewFloat64Scalar(0.5) - Binarize(oned, thr, oneout) - stats.MinFunc(oneout, mout) - assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - stats.MaxFunc(oneout, mout) - assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) - // fmt.Println(oneout) } diff --git a/tensor/values.go b/tensor/values.go index ed641ec941..5cd6b22472 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -129,9 +129,3 @@ func SetShapeNames(md *metadata.Data, names ...string) { func ShapeNames(md *metadata.Data) []string { return errors.Log1(metadata.Get[[]string](*md, "ShapeNames")) } - -// SetShapeSizesFromTensor sets the dimension sizes as 1D int values from given tensor. -// The backing storage is resized appropriately, retaining all existing data that fits. -func SetShapeSizesFromTensor(vals Values, sizes Tensor) { - vals.SetShapeSizes(AsIntSlice(sizes)...) -} From 4d039c1e6e37c037b59130233abc2b3973152223 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 16:24:43 -0700 Subject: [PATCH 097/311] testing, etc --- tensor/funcs_test.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 186374bb71..70a30a940e 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -8,11 +8,14 @@ import ( "math" "testing" + "cogentcore.org/core/base/errors" "github.com/stretchr/testify/assert" ) func abs(in, out Tensor) { - SetShapeFrom(out, in) + if err := SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { + return + } VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...Tensor) { tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) @@ -45,3 +48,28 @@ func TestFuncs(t *testing.T) { // assert.NoError(t, err) assert.Equal(t, AsFloat64Scalar(oneout), AsFloat64Scalar(out)) } + +func TestAlign(t *testing.T) { + a := NewFloat64(3, 4) + b := NewFloat64(1, 3, 4) + as, bs, os, err := AlignShapes(a, b) + assert.NoError(t, err) + assert.Equal(t, []int{1, 3, 4}, os.Sizes) + assert.Equal(t, []int{1, 3, 4}, as.Sizes) + assert.Equal(t, []int{1, 3, 4}, bs.Sizes) + + ars := NewReshaped(a, 12) + as, bs, os, err = AlignShapes(ars, b) + assert.Error(t, err) + + brs := NewReshaped(b, 12) + as, bs, os, err = AlignShapes(ars, brs) + assert.NoError(t, err) + + ars = NewReshaped(a, 3, 1, 4) + as, bs, os, err = AlignShapes(ars, b) + assert.NoError(t, err) + assert.Equal(t, []int{3, 3, 4}, os.Sizes) + assert.Equal(t, []int{3, 1, 4}, as.Sizes) + assert.Equal(t, []int{1, 3, 4}, bs.Sizes) +} From d32c6085c684fd1e893d947d364e07ff66727cb7 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 17:00:24 -0700 Subject: [PATCH 098/311] finish docs --- tensor/README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index 4c9f1934c1..a21e1580eb 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -6,16 +6,20 @@ The [Goal](../goal) augmented version of the _Go_ language directly supports Num The `Tensor` interface is implemented at the basic level with n-dimensional indexing into flat Go slices of any numeric data type (by `Number`), along with `String`, and `Bool` (which uses [bitslice](bitslice) for maximum efficiency). These implementations satisfy the `Values` sub-interface of Tensor, which supports the most direct and efficient operations on contiguous memory data. The `Shape` type provides all the n-dimensional indexing with arbitrary strides to allow any ordering, although _row major_ is the default and other orders have to be manually imposed. -In addition, there are four important "view" implementations of `Tensor` that wrap another "source" Tensor to provide more flexible and efficient access to the data, consistent with the NumPy functionality. See [Basic and Advanced Indexing](#basic-and-advanced-indexing) below for more info. +In addition, there are five important "view" implementations of `Tensor` that wrap another "source" Tensor to provide more flexible and efficient access to the data, consistent with the NumPy functionality. See [Basic and Advanced Indexing](#basic-and-advanced-indexing) below for more info. -* `Rows` provides a row index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension, and provides similar functionality to pandas and particularly [xarray](http://xarray.pydata.org/en/stable/) in Python. Organizing data systematically along the row dimension eliminates many sources of ambiguity about how to process higher-dimensional data, and any given n-dimensional structure can be reshaped to fit in this row-based format. - -* `Sliced` is a more general version of `Rows` that provides a sub-sliced view into the wrapped `Tensor` source, using an indexed list along each dimension, not just the outermost one. Thus, it can provide a reordered and filtered view onto the raw data, and it has a well-defined shape in terms of the number of indexes per dimension. This corresponds to the NumPy basic sliced indexing model. +* `Sliced` provides a sub-sliced view into the wrapped `Tensor` source, using an indexed list along each dimension. Thus, it can provide a reordered and filtered view onto the raw data, and it has a well-defined shape in terms of the number of indexes per dimension. This corresponds to the NumPy basic sliced indexing model. * `Masked` provides a `Bool` masked view onto each element in the wrapped `Tensor`, where the two maintain the same shape). Any cell with a `false` value in the bool mask returns a `NaN` (missing data), and `Set` functions are no-ops, such that the tensor functions automatically only process the mask-filtered data. * `Indexed` has a tensor of indexes into the source data, where the final, innermost dimension of the indexes is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. +* `Reshaped` applies a different `Shape` to the source tensor, with the constraint that the new shape has the same length of total elements as the source tensor. It is particularly useful for aligning different tensors binary operation between them produces the desired results, for example by adding a new axis or collapsing multiple dimensions into one. + +* `Rows` is a specialized version of `Sliced` that provides a row index-based view, with the `Indexes` applying to the outermost _row_ dimension, which allows sorting and filtering to operate only on the indexes, leaving the underlying Tensor unchanged. This view is returned by the [table](table) data table, which organizes multiple heterogenous Tensor columns along a common outer row dimension, and provides similar functionality to pandas and particularly [xarray](http://xarray.pydata.org/en/stable/) in Python. + +Note that any view can be "stacked" on top of another, to produce more complex net views. + Each view type implements the `AsValues` method to create a concrete "rendered" version of the view (as a `Values` tensor) where the actual underlying data is organized as it appears in the view. This is like the `copy` function in NumPy, disconnecting the view from the original source data. Note that unlike NumPy, `Masked` and `Indexed` remain views into the underlying source data -- see [Basic and Advanced Indexing](#basic-and-advanced-indexing) below. The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used as universal input / output types, and for intermediate computation in the math functions. Any performance-critical code can be optimized for a specific data type, but these universal interfaces are suitable for misc ad-hoc data analysis. @@ -68,6 +72,8 @@ The `SetNumRows` function can be used to progressively increase the number of ro # Cheat Sheet +TODO: update + `ix` is the `Rows` tensor for these examples: ## Tensor Access @@ -122,7 +128,7 @@ val := ix.Float(3,2,1) # `Tensor` vs. Python NumPy -The [Goal](../goal) language provides a reasonably faithful translation of NumPy `array` syntax into the corresponding Go tensor package implementations. For those already familiar with NumPy, it should mostly "just work", but the following provides a more in-depth explanation for how the two relate, and when you might get different results. +The [Goal](../goal) language provides a reasonably faithful translation of NumPy `ndarray` syntax into the corresponding Go tensor package implementations. For those already familiar with NumPy, it should mostly "just work", but the following provides a more in-depth explanation for how the two relate, and when you might get different results. ## Basic and Advanced Indexing @@ -136,7 +142,7 @@ a[a > 0.5] = 1 # boolean advanced indexing Although powerful, the semantics of all of this is a bit confusing. In the `tensor` package, we provide what are hopefully more clear and concrete _view_ types that have well-defined semantics, and cover the relevant functionality, while perhaps being a bit easier to reason with. These were described at the start of this README. The correspondence to NumPy indexing is as follows: -* Basic indexing by individual integer index coordinate values is supported by the basic `Number`, `String`, `Bool` `Values` Tensors. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) view then completes the basic indexing with arbitrary reordering and filtering along entire dimension values. +* Basic indexing by individual integer index coordinate values is supported by the `Number`, `String`, `Bool` `Values` Tensors. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) and `Reshaped` views then complete the basic indexing with arbitrary reordering and filtering along entire dimension values, and reshaping dimensions. * The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `AsValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `SourceIndexes` method returns a 1D list of indexes of the `true` (or `false`) values, which can be used for the `Indexed` view. @@ -173,7 +179,7 @@ A (2d array): 2 x 1 B (3d array): 8 x 4 x 3 # second from last dimensions mismatched ``` -Computationally, the broadcast logic is straightforward to implement, in terms of computing the resulting shape. Any missing outer dimensions are implicitly 1s, and any singleton dimension wraps around during the indexing process. +The `AlignShapes` function performs this shape alignment logic, and the `WrapIndex1D` function is used to compute a 1D index into a given shape, based on the total output shape sizes, wrapping any singleton dimensions around as needed. These are used in the [tmath](tmath) package for example to implement the basic binary math operators. # History From bfff608e73d54633055412422bfc8a21c9aef025 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 17:29:14 -0700 Subject: [PATCH 099/311] some fixes from kai comments on tfunc pr. --- plot/plotcore/ploteditor_test.go | 2 +- tensor/cmd/tablecat/tablecat.go | 4 ++-- tensor/databrowser/datatab.go | 4 ++-- tensor/datafs/dir.go | 2 +- tensor/examples/grids/grids.go | 2 +- tensor/examples/planets/planets.go | 2 +- tensor/slices.go | 26 ++++++++++++------------ tensor/stats/cluster/clust_test.go | 2 +- tensor/stats/cluster/cluster.go | 2 +- tensor/stats/histogram/histogram_test.go | 2 +- tensor/stats/metric/metric_test.go | 4 ++-- tensor/stats/stats/group_test.go | 4 ++-- tensor/stats/stats/norm.go | 1 + tensor/table/indexes.go | 6 +++--- tensor/table/io_test.go | 4 ++-- tensor/table/slicetable.go | 2 +- tensor/table/table.go | 8 ++++---- tensor/table/table_test.go | 6 +++--- 18 files changed, 42 insertions(+), 41 deletions(-) diff --git a/plot/plotcore/ploteditor_test.go b/plot/plotcore/ploteditor_test.go index 411f15943b..301991d56b 100644 --- a/plot/plotcore/ploteditor_test.go +++ b/plot/plotcore/ploteditor_test.go @@ -21,7 +21,7 @@ type Data struct { func TestTablePlotEditor(t *testing.T) { b := core.NewBody() - epc := table.NewTable("epc") + epc := table.New("epc") epc.OpenCSV("testdata/ra25epoch.tsv", tensor.Tab) pl := NewPlotEditor(b) diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index afabc8f4ad..2e6b214c23 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -110,7 +110,7 @@ func RawCat(files []string) { func AvgCat(files []string) { dts := make([]*table.Table, 0, len(files)) for _, fn := range files { - dt := table.NewTable() + dt := table.New() err := dt.OpenCSV(core.Filename(fn), tensor.Tab) if err != nil { fmt.Println("Error opening file: ", err) @@ -136,7 +136,7 @@ func AvgCat(files []string) { // If column is empty, averages across all rows. func AvgByColumn(files []string, column string) { for _, fn := range files { - dt := table.NewTable() + dt := table.New() err := dt.OpenCSV(core.Filename(fn), tensor.Tab) if err != nil { fmt.Println("Error opening file: ", err) diff --git a/tensor/databrowser/datatab.go b/tensor/databrowser/datatab.go index 53b430b9b8..5d78f1adb1 100644 --- a/tensor/databrowser/datatab.go +++ b/tensor/databrowser/datatab.go @@ -34,7 +34,7 @@ func NewTab[T any](br *Browser, label string, mkfun func(tab *core.Frame) T) T { func (br *Browser) NewTabTensorTable(label string, dt *table.Table) *tensorcore.Table { tv := NewTab[*tensorcore.Table](br, label, func(tab *core.Frame) *tensorcore.Table { tb := core.NewToolbar(tab) - tv := tensorcore.NewTable(tab) + tv := tensorcore.New(tab) tb.Maker(tv.MakeToolbar) return tv }) @@ -85,7 +85,7 @@ func (br *Browser) NewTabPlot(label string, dt *table.Table) *plotcore.PlotEdito // to view the given slice of structs. func (br *Browser) NewTabSliceTable(label string, slc any) *core.Table { tv := NewTab[*core.Table](br, label, func(tab *core.Frame) *core.Table { - return core.NewTable(tab) + return core.New(tab) }) tv.SetSlice(slc) br.Update() diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 6bcac3fc58..9e0b70dc7e 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -370,7 +370,7 @@ func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { return d.DirTable } its := d.FlatItemsFunc(fun) - dt := table.NewTable(fsx.DirAndFile(string(d.Path()))) + dt := table.New(fsx.DirAndFile(string(d.Path()))) for _, it := range its { tsr := it.Data rows := tsr.DimSize(0) diff --git a/tensor/examples/grids/grids.go b/tensor/examples/grids/grids.go index 9415813604..0fbe0ee05c 100644 --- a/tensor/examples/grids/grids.go +++ b/tensor/examples/grids/grids.go @@ -18,7 +18,7 @@ import ( var tsv embed.FS func main() { - pats := table.NewTable("TrainPats") + pats := table.New("TrainPats") pats.Meta.SetDoc("Training patterns") // todo: meta data for grid size errors.Log(pats.OpenFS(tsv, "random_5x5_25.tsv", tensor.Tab)) diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go index 6b85ccda22..c4cd2f0e3e 100644 --- a/tensor/examples/planets/planets.go +++ b/tensor/examples/planets/planets.go @@ -27,7 +27,7 @@ var csv embed.FS // // https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html func AnalyzePlanets(dir *datafs.Data) { - Planets := table.NewTable("planets") + Planets := table.New("planets") Planets.OpenFS(csv, "planets.csv", tensor.Comma) vals := []string{"number", "orbital_period", "mass", "distance", "year"} diff --git a/tensor/slices.go b/tensor/slices.go index e5b67915b6..6a1b96b84d 100644 --- a/tensor/slices.go +++ b/tensor/slices.go @@ -14,7 +14,7 @@ package tensor // the zero value results in a list of all values in the dimension, with Step = 1 if 0. // The behavior is identical to the NumPy slice. type Slice struct { - // Starting value. If 0 and Step < 0, = size-1; + // Start is the starting value. If 0 and Step < 0, = size-1; // If negative, = size+Start. Start int @@ -33,8 +33,8 @@ func NewSlice(start, stop, step int) Slice { return Slice{Start: start, Stop: stop, Step: step} } -// StartActual is the actual start value given the size of the dimension. -func (sl Slice) StartActual(size int) int { +// GetStart is the actual start value given the size of the dimension. +func (sl Slice) GetStart(size int) int { if sl.Start == 0 && sl.Step < 0 { return size - 1 } @@ -44,8 +44,8 @@ func (sl Slice) StartActual(size int) int { return sl.Start } -// StopActual is the actual end value given the size of the dimension. -func (sl Slice) StopActual(size int) int { +// GetStop is the actual end value given the size of the dimension. +func (sl Slice) GetStop(size int) int { if sl.Stop == 0 && sl.Step >= 0 { return size } @@ -58,8 +58,8 @@ func (sl Slice) StopActual(size int) int { return min(sl.Stop, size) } -// StepActual is the actual increment value. -func (sl Slice) StepActual() int { +// GetStep is the actual increment value. +func (sl Slice) GetStep() int { if sl.Step == 0 { return 1 } @@ -69,9 +69,9 @@ func (sl Slice) StepActual() int { // Len is the number of elements in the actual slice given // size of the dimension. func (sl Slice) Len(size int) int { - s := sl.StartActual(size) - e := sl.StopActual(size) - i := sl.StepActual() + s := sl.GetStart(size) + e := sl.GetStop(size) + i := sl.GetStep() n := max((e-s)/i, 0) pe := s + n*i if i < 0 { @@ -94,9 +94,9 @@ func (sl Slice) ToIntSlice(size int, ints []int) { if n == 0 { return } - s := sl.StartActual(size) - e := sl.StopActual(size) - inc := sl.StepActual() + s := sl.GetStart(size) + e := sl.GetStop(size) + inc := sl.GetStep() idx := 0 if inc < 0 { for i := s; i > e; i += inc { diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 561579c32a..fb58983a7d 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -28,7 +28,7 @@ var clustres = ` 3.605551275463989: Wendy_sad Wendy_happy ` func TestClust(t *testing.T) { - dt := table.NewTable() + dt := table.New() err := dt.OpenCSV("testdata/faces.dat", tensor.Tab) if err != nil { t.Error(err) diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index 6b9992a414..313bd7ead3 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -19,7 +19,7 @@ import ( // todo: all of this data goes into the datafs // Cluster makes a new dir, stuffs results in there! // need a global "cwd" that it uses, so basically you cd -// to a dir, then cal it. +// to a dir, then call it. // Node is one node in the cluster type Node struct { diff --git a/tensor/stats/histogram/histogram_test.go b/tensor/stats/histogram/histogram_test.go index c25452b06a..4806692268 100644 --- a/tensor/stats/histogram/histogram_test.go +++ b/tensor/stats/histogram/histogram_test.go @@ -20,7 +20,7 @@ func TestHistogram64(t *testing.T) { assert.Equal(t, ex, res) // exvals := []float64{0, 0.3333, 0.6667} - // dt := table.NewTable() + // dt := table.New() // F64Table(dt, vals, 3, 0, 1) // for ri, v := range ex { // vv := dt.Float("Value", ri) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index aa78ad7603..567109ab5c 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -85,7 +85,7 @@ func TestMatrix(t *testing.T) { [10]: 5.291502622129181 6.324555320336759 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 4.242640687119285 5.477225575051661 9.16515138991168 9.539392014169456 0 3.4641016151377544 [11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 ` - dt := table.NewTable() + dt := table.New() err := dt.OpenCSV("../cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) in := dt.Column("Input") @@ -99,7 +99,7 @@ func TestPCAIris(t *testing.T) { // note: these results are verified against this example: // https://plot.ly/ipython-notebooks/principal-component-analysis/ - dt := table.NewTable() + dt := table.New() dt.AddFloat64Column("data", 4) dt.AddStringColumn("class") err := dt.OpenCSV("testdata/iris.data", tensor.Comma) diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index cb3ea6bd14..01adfb0fd1 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -14,7 +14,7 @@ import ( ) func TestGroup(t *testing.T) { - dt := table.NewTable().SetNumRows(4) + dt := table.New().SetNumRows(4) dt.AddStringColumn("Name") dt.AddFloat32Column("Value") for i := range dt.NumRows() { @@ -44,7 +44,7 @@ func TestGroup(t *testing.T) { /* func TestAggEmpty(t *testing.T) { - dt := table.NewTable().SetNumRows(4) + dt := table.New().SetNumRows(4) dt.AddStringColumn("Group") dt.AddFloat32Column("Value") for i := 0; i < dt.Rows; i++ { diff --git a/tensor/stats/stats/norm.go b/tensor/stats/stats/norm.go index cf599a75d2..9304daeddb 100644 --- a/tensor/stats/stats/norm.go +++ b/tensor/stats/stats/norm.go @@ -7,6 +7,7 @@ package stats import ( "cogentcore.org/core/math32" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/tmath" ) // ZScore computes Z-normalized values into given output tensor, diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 8964cb57a8..87697be915 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -213,10 +213,10 @@ func (dt *Table) FilterString(columnName string, str string, exclude, contains, return nil } -// NewTable returns a new table with column data organized according to +// New returns a new table with column data organized according to // the indexes. If Indexes are nil, a clone of the current tensor is returned // but this function is only sensible if there is an indexed view in place. -func (dt *Table) NewTable() *Table { +func (dt *Table) New() *Table { if dt.Indexes == nil { return dt.Clone() } @@ -239,7 +239,7 @@ func (dt *Table) NewTable() *Table { // DeleteRows deletes n rows of Indexes starting at given index in the list of indexes. // This does not affect the underlying tensor data; To create an actual in-memory -// ordering with rows deleted, use [Table.NewTable]. +// ordering with rows deleted, use [Table.New]. func (dt *Table) DeleteRows(at, n int) { dt.IndexesNeeded() dt.Indexes = append(dt.Indexes[:at], dt.Indexes[at+n:]...) diff --git a/tensor/table/io_test.go b/tensor/table/io_test.go index 59e0d1601e..2534d6ca2c 100644 --- a/tensor/table/io_test.go +++ b/tensor/table/io_test.go @@ -15,7 +15,7 @@ func TestTableHeaders(t *testing.T) { hdrstr := `$Name %Input[2:0,0]<2:5,5> %Input[2:1,0] %Input[2:2,0] %Input[2:3,0] %Input[2:4,0] %Input[2:0,1] %Input[2:1,1] %Input[2:2,1] %Input[2:3,1] %Input[2:4,1] %Input[2:0,2] %Input[2:1,2] %Input[2:2,2] %Input[2:3,2] %Input[2:4,2] %Input[2:0,3] %Input[2:1,3] %Input[2:2,3] %Input[2:3,3] %Input[2:4,3] %Input[2:0,4] %Input[2:1,4] %Input[2:2,4] %Input[2:3,4] %Input[2:4,4] %Output[2:0,0]<2:5,5> %Output[2:1,0] %Output[2:2,0] %Output[2:3,0] %Output[2:4,0] %Output[2:0,1] %Output[2:1,1] %Output[2:2,1] %Output[2:3,1] %Output[2:4,1] %Output[2:0,2] %Output[2:1,2] %Output[2:2,2] %Output[2:3,2] %Output[2:4,2] %Output[2:0,3] %Output[2:1,3] %Output[2:2,3] %Output[2:3,3] %Output[2:4,3] %Output[2:0,4] %Output[2:1,4] %Output[2:2,4] %Output[2:3,4] %Output[2:4,4] ` hdrs := strings.Split(hdrstr, "\t") - dt := NewTable() + dt := New() err := ConfigFromHeaders(dt, hdrs, nil) if err != nil { t.Error(err) @@ -69,7 +69,7 @@ func TestReadTableDat(t *testing.T) { if err != nil { t.Error(err) } - dt := NewTable() + dt := New() err = dt.ReadCSV(fp, '\t') // tsv if err != nil { t.Error(err) diff --git a/tensor/table/slicetable.go b/tensor/table/slicetable.go index 2adfcd1c1e..cdc2468e8f 100644 --- a/tensor/table/slicetable.go +++ b/tensor/table/slicetable.go @@ -22,7 +22,7 @@ func NewSliceTable(st any) (*Table, error) { if eltyp.Kind() != reflect.Struct { return nil, fmt.Errorf("NewSliceTable: element type is not a struct") } - dt := NewTable() + dt := New() for i := 0; i < eltyp.NumField(); i++ { f := eltyp.Field(i) diff --git a/tensor/table/table.go b/tensor/table/table.go index f7ec5c6262..79b6dbd47b 100644 --- a/tensor/table/table.go +++ b/tensor/table/table.go @@ -42,9 +42,9 @@ type Table struct { //types:add Meta metadata.Data } -// NewTable returns a new Table with its own (empty) set of Columns. +// New returns a new Table with its own (empty) set of Columns. // Can pass an optional name which calls metadata SetName. -func NewTable(name ...string) *Table { +func New(name ...string) *Table { dt := &Table{} dt.Columns = NewColumns() if len(name) > 0 { @@ -250,7 +250,7 @@ func (dt *Table) AddRows(n int) *Table { //types:add // InsertRows adds n rows to end of underlying Table, and to the indexes starting at // given index in this view, providing an efficient insertion operation that only -// exists in the indexed view. To create an in-memory ordering, use [Table.NewTable]. +// exists in the indexed view. To create an in-memory ordering, use [Table.New]. func (dt *Table) InsertRows(at, n int) *Table { dt.IndexesNeeded() strow := dt.Columns.Rows @@ -299,7 +299,7 @@ func (dt *Table) SetNumRowsToMax() { // Clone returns a complete copy of this table, including cloning // the underlying Columns tensors, and the current [Table.Indexes]. -// See also [Table.NewTable] to flatten the current indexes. +// See also [Table.New] to flatten the current indexes. func (dt *Table) Clone() *Table { cp := &Table{} cp.Columns = dt.Columns.Clone() diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index 55ba2b870a..c16be732c6 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -13,7 +13,7 @@ import ( ) func TestAdd3DCol(t *testing.T) { - dt := NewTable() + dt := New() dt.AddFloat32Column("Values", 11, 1, 16) col := dt.Column("Values").Tensor @@ -39,7 +39,7 @@ func TestAdd3DCol(t *testing.T) { } func NewTestTable() *Table { - dt := NewTable() + dt := New() dt.AddStringColumn("Str") dt.AddFloat64Column("Flt64") dt.AddIntColumn("Int") @@ -114,7 +114,7 @@ func TestInsertDeleteRows(t *testing.T) { } func TestCells(t *testing.T) { - dt := NewTable() + dt := New() err := dt.OpenCSV("../stats/cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) in := dt.Column("Input") From ec34572cbc898e34efb9b8d356797e25563b0bea Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 22 Sep 2024 21:47:19 -0700 Subject: [PATCH 100/311] more fixes from kai comments on tfunc pr. --- tensor/bool.go | 2 +- tensor/examples/planets/planets.go | 5 ++--- tensor/number.go | 12 ++++++------ tensor/shape.go | 6 +++--- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/tensor/bool.go b/tensor/bool.go index de5c8c8781..9a633a82b9 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -25,7 +25,7 @@ type Bool struct { } // NewBool returns a new n-dimensional tensor of bit values -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewBool(sizes ...int) *Bool { tsr := &Bool{} tsr.SetShapeSizes(sizes...) diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go index c4cd2f0e3e..8a34822953 100644 --- a/tensor/examples/planets/planets.go +++ b/tensor/examples/planets/planets.go @@ -23,9 +23,8 @@ import ( var csv embed.FS // AnalyzePlanets analyzes planets.csv data following some of the examples -// given here, using pandas: -// -// https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html +// in pandas from: +// https://jakevdp.github.io/PythonDataScienceHandbook/03.08-aggregation-and-grouping.html func AnalyzePlanets(dir *datafs.Data) { Planets := table.New("planets") Planets.OpenFS(csv, "planets.csv", tensor.Comma) diff --git a/tensor/number.go b/tensor/number.go index 07580758af..cd81b2dc75 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -34,37 +34,37 @@ type Int32 = Number[int32] type Byte = Number[byte] // NewFloat32 returns a new [Float32] tensor -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewFloat32(sizes ...int) *Float32 { return New[float32](sizes...).(*Float32) } // NewFloat64 returns a new [Float64] tensor -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewFloat64(sizes ...int) *Float64 { return New[float64](sizes...).(*Float64) } // NewInt returns a new Int tensor -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewInt(sizes ...int) *Int { return New[int](sizes...).(*Int) } // NewInt32 returns a new Int32 tensor -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewInt32(sizes ...int) *Int32 { return New[int32](sizes...).(*Int32) } // NewByte returns a new Byte tensor -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewByte(sizes ...int) *Byte { return New[uint8](sizes...).(*Byte) } // NewNumber returns a new n-dimensional tensor of numerical values -// with the given sizes per dimension (shape), and optional dimension names. +// with the given sizes per dimension (shape). func NewNumber[T num.Number](sizes ...int) *Number[T] { tsr := &Number[T]{} tsr.SetShapeSizes(sizes...) diff --git a/tensor/shape.go b/tensor/shape.go index d061450c8f..3e6b4d4e19 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -9,7 +9,7 @@ import ( "slices" ) -// Shape manages a tensor's shape information, including strides and dimension names +// Shape manages a tensor's shape information, including sizes and strides, // and can compute the flat index into an underlying 1D data storage array based on an // n-dimensional index (and vice-versa). // Per Go / C / Python conventions, indexes are Row-Major, ordered from @@ -192,9 +192,9 @@ func RowMajorStrides(sizes ...int) []int { return strides } -// ColMajorStrides returns strides for sizes where the first dimension is inner-most +// ColumnMajorStrides returns strides for sizes where the first dimension is inner-most // and subsequent dimensions are progressively outer -func ColMajorStrides(sizes ...int) []int { +func ColumnMajorStrides(sizes ...int) []int { total := int(1) for _, v := range sizes { if v == 0 { From 94b3b37b0b96847b51eecbb815a38aa450cf621a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 02:44:45 -0700 Subject: [PATCH 101/311] many fixes: funcs return errors, improved reshaping, Cells1D, etc functions; various fixes from kai comments; stats now working and passing tests. --- tensor/README.md | 6 +- tensor/base.go | 9 -- tensor/bool.go | 8 - tensor/convert.go | 105 +++++++------ tensor/funcs.go | 108 ++++++------- tensor/funcs_test.go | 8 +- tensor/io.go | 6 +- tensor/number.go | 27 ---- tensor/reshaped.go | 22 +++ tensor/shape.go | 13 ++ tensor/stats/metric/matrix.go | 8 +- tensor/stats/stats/README.md | 2 +- tensor/stats/stats/funcs.go | 188 +++++++++++++--------- tensor/stats/stats/norm.go | 30 +++- tensor/stats/stats/quantiles.go | 18 ++- tensor/stats/stats/stats.go | 24 +-- tensor/stats/stats/stats_test.go | 19 +-- tensor/stats/stats/vec.go | 48 +++--- tensor/string.go | 4 - tensor/table/io.go | 2 +- tensor/table/table_test.go | 1 + tensor/tensorcore/tensorgrid.go | 2 +- tensor/tensormpi/tensor.go | 2 +- tensor/tmath/math.go | 260 ++++++++++++++++++------------- tensor/tmath/math_test.go | 4 +- tensor/tmath/ops.go | 45 +++--- tensor/tmath/ops_test.go | 2 +- tensor/values.go | 6 - 28 files changed, 528 insertions(+), 449 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index a21e1580eb..717f84857b 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -56,11 +56,9 @@ All tensor package functions are registered using a global name-to-function map In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: -* **Flat, 1D**: this is the simplest data shape. For example, the [stats](stats) functions report summary statistics for all values of such data, across the one dimension. `Rows` views of this 1D data provide fine-grained filtering and sorting of all the data. Any `Tensor` can be accessed via a flat 1D index, which goes directly into the underlying Go slice for the basic types, and is appropriately (though somewhat expensively in some cases) indirected through the effective geometry in `Sliced` and `Rows` types. +* **Flat, 1D**: this is the simplest data shape, and any tensor can be turned into a flat 1D list using the `New1DView` function, which returns a `Reshaped` view. The [stats](stats) functions for example report summary statistics across the outermost row dimension, so converting data to this 1D view gives stats across all the data. -* **Row, Cell 2D**: The outermost row dimension can be sorted, filtered in an `Rows` view, and the inner "cells" of data are organized in a simple flat 1D `SubSpace`, so they can be easily processed. In the [stats](stats) and [metric](metric) packages, 2+ dimensional data will be automatically re-shaped into this Row, Cell format, and processed as row-wise list of cell-wise patterns. For example, `stats` will aggregate each cell separately across rows, so you end up with the "average pattern" when you do `stats.Mean` for example. The [tmath](tmath) package, which defines binary functions, uses the [broadcasting](#broadcasting) logic to align n-dimensional data, and the row, cell structure provides a concrete simplification for thinking about how that works. - - A higher-dimensional tensor can also be re-shaped into this row, cell format by collapsing any number of additional outer dimensions into a longer, effective "row" index, with the remaining inner dimensions forming the cell-wise patterns. You can decide where to make the cut, and the `RowCellSplit` function makes it easy to create a new view of an existing tensor with this split made at a given dimension. +* **Row, Cell 2D**: This is the natural shape for tabular data, and the `RowMajor` type and `Rows` view provide methods for efficiently accessing data in this way. In addition, the [stats](stats) and [metric](metric) packages automatically compute statistics across the outermost row dimension, aggregating results across rows for each cell. Thus, you end up with the "average cell-wise pattern" when you do `stats.Mean` for example. The `NewRowCellsView` function returns a `Reshaped` view of any tensor organized into this 2D shape, with the row vs. cell split specified at any point in the list of dimensions, which can be useful in obtaining the desired results. * **Matrix 2D**: For matrix algebra functions, a 2D tensor is treated as a standard row-major 2D matrix, which can be processed using `gonum` based matrix and vector operations, as in the [matrix](matrix) package. diff --git a/tensor/base.go b/tensor/base.go index 7341627a1a..1d8c488c9f 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -85,15 +85,6 @@ func (tsr *Base[T]) Set(val T, i ...int) { func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[i] = val } -// view is implementation of View -- needs final casting to tensor type. -func (tsr *Base[T]) view() *Base[T] { - nw := &Base[T]{} - nw.shape.CopyFrom(&tsr.shape) - nw.Values = tsr.Values - nw.Meta = tsr.Meta - return nw -} - // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) // it is best to first set the anticipated full size, which allocates the diff --git a/tensor/bool.go b/tensor/bool.go index 9a633a82b9..b7bc4cac1f 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -291,14 +291,6 @@ func (tsr *Bool) Clone() Values { return csr } -func (tsr *Bool) View() Values { - nw := &Bool{} - nw.shape.CopyFrom(&tsr.shape) - nw.Values = tsr.Values - nw.Meta = tsr.Meta - return nw -} - // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. diff --git a/tensor/convert.go b/tensor/convert.go index 3154a7661a..353498d63c 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -4,7 +4,11 @@ package tensor -import "cogentcore.org/core/base/errors" +import ( + "math" + + "cogentcore.org/core/base/errors" +) // Clone returns a copy of the given tensor. // If it is raw [Values] then a [Values.Clone] is returned. @@ -32,19 +36,6 @@ func SetShape(vals Values, sh *Shape) { vals.SetShapeSizes(sh.Sizes...) } -// SetShapeMustBeValues sets the dimension sizes from given Shape, -// calling [MustBeValues] on the destination tensor to ensure it is a [Values], -// type, returning an error if not. This is used extensively for output -// tensors in functions, and all such output tensors _must_ be Values tensors. -func SetShapeMustBeValues(tsr Tensor, sh *Shape) error { - vals, err := MustBeValues(tsr) - if err != nil { - return err - } - vals.SetShapeSizes(sh.Sizes...) - return nil -} - // SetShapeSizesFromTensor sets the dimension sizes as 1D int values from given tensor. // The backing storage is resized appropriately, retaining all existing data that fits. func SetShapeSizesFromTensor(vals Values, sizes Tensor) { @@ -60,52 +51,43 @@ func SetShapeFrom(vals Values, from Tensor) { // calling [MustBeValues] on the destination tensor to ensure it is a [Values], // type, returning an error if not. This is used extensively for output // tensors in functions, and all such output tensors _must_ be Values tensors. -func SetShapeFromMustBeValues(tsr, from Tensor) error { +func SetShapeFromMustBeValues(tsr Tensor, from Tensor) error { + return SetShapeSizesMustBeValues(tsr, from.ShapeSizes()...) +} + +// SetShapeSizesMustBeValues sets shape of given tensor from given sizes, +// calling [MustBeValues] on the destination tensor to ensure it is a [Values], +// type, returning an error if not. This is used extensively for output +// tensors in functions, and all such output tensors _must_ be Values tensors. +func SetShapeSizesMustBeValues(tsr Tensor, sizes ...int) error { vals, err := MustBeValues(tsr) if err != nil { return err } - vals.SetShapeSizes(from.ShapeSizes()...) + vals.SetShapeSizes(sizes...) return nil } -// New1DViewOf returns a 1D view into the given tensor, using the same -// underlying values, and just changing the shape to a 1D view. -// This can be useful e.g., for stats and metric functions that report -// on the 1D list of values. -func New1DViewOf(tsr Values) Values { - vw := tsr.View() - vw.SetShapeSizes(tsr.Len()) - return vw -} - -// Cells1D returns a flat 1D [Values] view of the cells for given row index. -// This is useful for passing to other functions e.g., -// in stats or metrics that process a 1D tensor. -func Cells1D(tsr RowMajor, row int) Values { - return New1DViewOf(tsr.SubSpace(row)) +// As1D returns a 1D tensor, which is either the input tensor if it is +// already 1D, or a new [Reshaped] 1D view of it. +// This can be useful e.g., for stats and metric functions that operate +// on a 1D list of values. +func As1D(tsr Tensor) Tensor { + if tsr.NumDims() == 1 { + return tsr + } + return NewReshaped(tsr, tsr.Len()) } -// RowCellSplit splits the given tensor into a standard 2D row, cell -// shape at the given split dimension index. All dimensions prior to -// split are collapsed into the row dimension, and from split onward -// form the cells dimension. The resulting tensor is a re-shaped view -// of the original tensor, sharing the same underlying data. -func RowCellSplit(tsr Values, split int) Values { - sizes := tsr.ShapeSizes() - rows := sizes[:split] - cells := sizes[split:] - nr := 1 - for _, r := range rows { - nr *= r - } - nc := 1 - for _, c := range cells { - nc *= c +// Cells1D returns a flat 1D view of the innermost cells for given row index. +// For a [RowMajor] tensor, it uses the [RowTensor] subspace directly, +// otherwise it uses [Sliced] to extract the cells. In either case, +// [As1D] is used to ensure the result is a 1D tensor. +func Cells1D(tsr Tensor, row int) Tensor { + if rm, ok := tsr.(RowMajor); ok { + return As1D(rm.RowTensor(row)) } - vw := tsr.View() - vw.SetShapeSizes(nr, nc) - return vw + return As1D(NewSlicedIndexes(tsr, []int{row})) } // NewFloat64Scalar is a convenience method for a Tensor @@ -276,3 +258,26 @@ func AsIntTensor(tsr Tensor) *Int { f.CopyFrom(tsr.AsValues()) return f } + +// Range returns the min, max (and associated indexes, -1 = no values) for the tensor. +// This is needed for display and is thus in the tensor api on Values. +func Range(vals Values) (min, max float64, minIndex, maxIndex int) { + minIndex = -1 + maxIndex = -1 + n := vals.Len() + for j := range n { + fv := vals.Float1D(n) + if math.IsNaN(fv) { + continue + } + if fv < min || minIndex < 0 { + min = fv + minIndex = j + } + if fv > max || maxIndex < 0 { + max = fv + maxIndex = j + } + } + return +} diff --git a/tensor/funcs.go b/tensor/funcs.go index 69530325a5..359e0aa6c8 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -7,7 +7,6 @@ package tensor import ( "fmt" "reflect" - "strings" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" @@ -68,7 +67,7 @@ var Funcs map[string]*Func // AddFunc adds given named function to the global tensor named function // registry, which is used in goal to call functions by name, and // in specific packages to call functions by enum String() names. -// Use the standard Go CamelCase name -- will be auto-lowercased. +// Use the standard Go CamelCase name. // The number of output arguments must be provided here, // along with an optional first string argument if present; // the number of input arguments is automatically set from that. @@ -76,8 +75,7 @@ func AddFunc(name string, fun any, out int, stringFirst ...bool) error { if Funcs == nil { Funcs = make(map[string]*Func) } - nm := strings.ToLower(name) - _, ok := Funcs[nm] + _, ok := Funcs[name] if ok { return errors.Log(fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name)) } @@ -85,7 +83,7 @@ func AddFunc(name string, fun any, out int, stringFirst ...bool) error { if errors.Log(err) != nil { return err } - Funcs[nm] = fn + Funcs[name] = fn // note: can record orig camel name if needed for docs etc later. return nil } @@ -96,8 +94,7 @@ func AddFunc(name string, fun any, out int, stringFirst ...bool) error { // in the Funcs global function registry, or the argument count // does not match. func Call(name string, tsr ...Tensor) error { - nm := strings.ToLower(name) - fn, ok := Funcs[nm] + fn, ok := Funcs[name] if !ok { return errors.Log(fmt.Errorf("tensor.Call: function of name %q not registered", name)) } @@ -111,8 +108,7 @@ func Call(name string, tsr ...Tensor) error { // does not match. This version of [Call] is for functions that // have an initial string argument func CallString(name, first string, tsr ...Tensor) error { - nm := strings.ToLower(name) - fn, ok := Funcs[nm] + fn, ok := Funcs[name] if !ok { return errors.Log(fmt.Errorf("tensor.Call: function of name %q not registered", name)) } @@ -124,10 +120,9 @@ func CallString(name, first string, tsr ...Tensor) error { // output tensor, for the common case with just one return value. // An error is logged if the function name has not been registered // in the Funcs global function registry, or the argument count -// does not match. +// does not match, or an error is returned by the function. func CallOut(name string, tsr ...Tensor) Tensor { - nm := strings.ToLower(name) - fn, ok := Funcs[nm] + fn, ok := Funcs[name] if !ok { errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) return nil @@ -140,10 +135,9 @@ func CallOut(name string, tsr ...Tensor) Tensor { // output tensors, for the rare case of multiple return values. // An error is logged if the function name has not been registered // in the Funcs global function registry, or the argument count -// does not match. +// does not match, or an error is returned by the function. func CallOutMulti(name string, tsr ...Tensor) []Tensor { - nm := strings.ToLower(name) - fn, ok := Funcs[nm] + fn, ok := Funcs[name] if !ok { errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) return nil @@ -156,34 +150,34 @@ func CallOutMulti(name string, tsr ...Tensor) []Tensor { func (fn *Func) ArgCount() int { nargs := -1 switch fn.Fun.(type) { - case func(a Tensor): + case func(a Tensor) error: nargs = 1 - case func(a, b Tensor): + case func(a, b Tensor) error: nargs = 2 - case func(a, b, c Tensor): + case func(a, b, c Tensor) error: nargs = 3 - case func(a, b, c, d Tensor): + case func(a, b, c, d Tensor) error: nargs = 4 - case func(a, b, c, d, e Tensor): + case func(a, b, c, d, e Tensor) error: nargs = 5 // string cases: - case func(s string, a Tensor): + case func(s string, a Tensor) error: nargs = 1 - case func(s string, a, b Tensor): + case func(s string, a, b Tensor) error: nargs = 2 - case func(s string, a, b, c Tensor): + case func(s string, a, b, c Tensor) error: nargs = 3 - case func(s string, a, b, c, d Tensor): + case func(s string, a, b, c, d Tensor) error: nargs = 4 - case func(s string, a, b, c, d, e Tensor): + case func(s string, a, b, c, d, e Tensor) error: nargs = 5 } return nargs } -// ArgCheck returns an error if the number of args in list does not +// argCheck returns an error if the number of args in list does not // match the number required as specified. -func (fn *Func) ArgCheck(n int, tsr ...Tensor) error { +func (fn *Func) argCheck(n int, tsr ...Tensor) error { if len(tsr) != n { return fmt.Errorf("tensor.Call: args passed to %q: %d does not match required: %d", fn.Name, len(tsr), n) } @@ -197,31 +191,31 @@ func (fn *Func) Call(tsr ...Tensor) error { return fmt.Errorf("tensor.Call: function %q: requires a first string argument", fn.Name) } switch f := fn.Fun.(type) { - case func(a Tensor): - if err := fn.ArgCheck(1, tsr...); err != nil { + case func(a Tensor) error: + if err := fn.argCheck(1, tsr...); err != nil { return err } - f(tsr[0]) - case func(a, b Tensor): - if err := fn.ArgCheck(2, tsr...); err != nil { + return f(tsr[0]) + case func(a, b Tensor) error: + if err := fn.argCheck(2, tsr...); err != nil { return err } - f(tsr[0], tsr[1]) - case func(a, b, c Tensor): - if err := fn.ArgCheck(3, tsr...); err != nil { + return f(tsr[0], tsr[1]) + case func(a, b, c Tensor) error: + if err := fn.argCheck(3, tsr...); err != nil { return err } - f(tsr[0], tsr[1], tsr[2]) - case func(a, b, c, d Tensor): - if err := fn.ArgCheck(4, tsr...); err != nil { + return f(tsr[0], tsr[1], tsr[2]) + case func(a, b, c, d Tensor) error: + if err := fn.argCheck(4, tsr...); err != nil { return err } - f(tsr[0], tsr[1], tsr[2], tsr[3]) - case func(a, b, c, d, e Tensor): - if err := fn.ArgCheck(5, tsr...); err != nil { + return f(tsr[0], tsr[1], tsr[2], tsr[3]) + case func(a, b, c, d, e Tensor) error: + if err := fn.argCheck(5, tsr...); err != nil { return err } - f(tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) + return f(tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) } return nil } @@ -234,31 +228,31 @@ func (fn *Func) CallString(s string, tsr ...Tensor) error { return fmt.Errorf("tensor.CallString: function %q: does not take a first string argument", fn.Name) } switch f := fn.Fun.(type) { - case func(s string, a Tensor): - if err := fn.ArgCheck(1, tsr...); err != nil { + case func(s string, a Tensor) error: + if err := fn.argCheck(1, tsr...); err != nil { return err } - f(s, tsr[0]) - case func(s string, a, b Tensor): - if err := fn.ArgCheck(2, tsr...); err != nil { + return f(s, tsr[0]) + case func(s string, a, b Tensor) error: + if err := fn.argCheck(2, tsr...); err != nil { return err } - f(s, tsr[0], tsr[1]) - case func(s string, a, b, c Tensor): - if err := fn.ArgCheck(3, tsr...); err != nil { + return f(s, tsr[0], tsr[1]) + case func(s string, a, b, c Tensor) error: + if err := fn.argCheck(3, tsr...); err != nil { return err } - f(s, tsr[0], tsr[1], tsr[2]) - case func(s string, a, b, c, d Tensor): - if err := fn.ArgCheck(4, tsr...); err != nil { + return f(s, tsr[0], tsr[1], tsr[2]) + case func(s string, a, b, c, d Tensor) error: + if err := fn.argCheck(4, tsr...); err != nil { return err } - f(s, tsr[0], tsr[1], tsr[2], tsr[3]) - case func(s string, a, b, c, d, e Tensor): - if err := fn.ArgCheck(5, tsr...); err != nil { + return f(s, tsr[0], tsr[1], tsr[2], tsr[3]) + case func(s string, a, b, c, d, e Tensor) error: + if err := fn.argCheck(5, tsr...); err != nil { return err } - f(s, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) + return f(s, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) } return nil } diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 70a30a940e..99d4e5742a 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -8,17 +8,17 @@ import ( "math" "testing" - "cogentcore.org/core/base/errors" "github.com/stretchr/testify/assert" ) -func abs(in, out Tensor) { - if err := SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func abs(in, out Tensor) error { + if err := SetShapeFromMustBeValues(out, in); err != nil { + return err } VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...Tensor) { tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } func TestFuncs(t *testing.T) { diff --git a/tensor/io.go b/tensor/io.go index 02cc175a17..44adaa4b70 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -50,10 +50,10 @@ func SetPrecision(md metadata.Data, prec int) { md.Set("precision", prec) } -// GetPrecision gets the "precision" metadata value that determines +// Precision gets the "precision" metadata value that determines // the precision to use in writing floating point numbers to files. // returns an error if not set. -func GetPrecision(md metadata.Data) (int, error) { +func Precision(md metadata.Data) (int, error) { return metadata.Get[int](md, "precision") } @@ -96,7 +96,7 @@ func OpenCSV(tsr Tensor, filename core.Filename, delim Delims) error { // Reading just grabs all values and doesn't care about shape. func WriteCSV(tsr Tensor, w io.Writer, delim Delims) error { prec := -1 - if ps, err := GetPrecision(*tsr.Metadata()); err == nil { + if ps, err := Precision(*tsr.Metadata()); err == nil { prec = ps } cw := csv.NewWriter(w) diff --git a/tensor/number.go b/tensor/number.go index cd81b2dc75..c773b5a85a 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -6,7 +6,6 @@ package tensor import ( "fmt" - "math" "strconv" "cogentcore.org/core/base/num" @@ -237,10 +236,6 @@ func (tsr *Number[T]) SetZeros() { } } -func (tsr *Number[T]) View() Values { - return &Number[T]{*tsr.view()} -} - // Clone clones this tensor, creating a duplicate copy of itself with its // own separate memory representation of all the values. func (tsr *Number[T]) Clone() Values { @@ -334,25 +329,3 @@ func (tsr *Number[T]) SetRowTensor(val Values, row int) { mx := min(val.Len(), cells) tsr.CopyCellsFrom(val, st, 0, mx) } - -// Range returns the min, max (and associated indexes, -1 = no values) for the tensor. -// This is needed for display and is thus in the core api in optimized form -func (tsr *Number[T]) Range() (min, max float64, minIndex, maxIndex int) { - minIndex = -1 - maxIndex = -1 - for j, vl := range tsr.Values { - fv := float64(vl) - if math.IsNaN(fv) { - continue - } - if fv < min || minIndex < 0 { - min = fv - minIndex = j - } - if fv > max || maxIndex < 0 { - max = fv - maxIndex = j - } - } - return -} diff --git a/tensor/reshaped.go b/tensor/reshaped.go index c8e76beb04..0e57601863 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -41,6 +41,28 @@ func NewReshaped(tsr Tensor, sizes ...int) *Reshaped { return rs } +// NewRowCellsView returns a 2D [Reshaped] view onto the given tensor, +// with a single outer "row" dimension and a single inner "cells" dimension, +// with the given 'split' dimension specifying where the cells start. +// All dimensions prior to split are collapsed to form the new outer row dimension, +// and the remainder are collapsed to form the 1D cells dimension. +// This is useful for stats, metrics and other packages that operate +// on data in this shape. +func NewRowCellsView(tsr Values, split int) *Reshaped { + sizes := tsr.ShapeSizes() + rows := sizes[:split] + cells := sizes[split:] + nr := 1 + for _, r := range rows { + nr *= r + } + nc := 1 + for _, c := range cells { + nc *= c + } + return NewReshaped(tsr, nr, nc) +} + // AsReshaped returns the tensor as a [Reshaped] view. // If it already is one, then it is returned, otherwise it is wrapped // with an initial shape equal to the source tensor. diff --git a/tensor/shape.go b/tensor/shape.go index 3e6b4d4e19..c49abcca11 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -224,3 +224,16 @@ func AddShapes(shape1, shape2 *Shape) *Shape { sh := NewShape(nsh...) return sh } + +// CellsSizes returns the sizes of inner cells dimensions given +// overall tensor sizes. It returns []int{1} for the 1D case. +// Used for ensuring cell-wise outputs are the right size. +func CellsSize(sizes []int) []int { + csz := slices.Clone(sizes) + if len(csz) == 1 { + csz[0] = 1 + } else { + csz = csz[1:] + } + return csz +} diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index b7181c941c..a78cefc17c 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -24,7 +24,9 @@ func init() { // Matrix computes the rows x rows square distance / similarity matrix // between the patterns for each row of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, -// and within that, 1+dimensional patterns (cells). +// and within that, 1+dimensional patterns (cells). Use [tensor.NewRowCellsView] +// to organize data into the desired split between a 1D outermost Row dimension +// and the remaining Cells dimension. // The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied @@ -42,8 +44,8 @@ func Matrix(funcName string, in, out tensor.Tensor) { tensor.VectorizeThreaded(cells*3, func(tsr ...tensor.Tensor) int { return nc }, func(idx int, tsr ...tensor.Tensor) { c := coords[idx] - sa := tensor.Cells1D(tsr[0], c.X) - sb := tensor.Cells1D(tsr[0], c.Y) + sa := tensor.NewRowCellsSliced(tsr[0], c.X) + sb := tensor.NewRowCellsSliced(tsr[0], c.Y) tensor.Call(funcName, sa, sb, mout) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, in, out) diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index ad58bc7d09..8086b4bf40 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -5,7 +5,7 @@ The `stats` package provides standard statistic computations operating on the `t type StatsFunc func(in, out tensor.Tensor) ``` -For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outermost row dimension set to 1. +For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outermost row dimension set to 1. This is All stats are registered in the `tensor.Funcs` global list, and can be called using the `FuncName` method, e.g.,: ```Go diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 9fa28ff20d..6716f3c220 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -17,19 +17,18 @@ import ( // and a single scalar value for a 1D input tensor. // Critically, the stat is always computed over the outer row dimension, // so each cell in a higher-dimensional output reflects the _row-wise_ -// stat for that cell across the different rows. To compute a stat -// on the [tensor.SubSpace] cells themselves, must call on a -// [tensor.New1DViewOf] the sub space. +// stat for that cell across the different rows. Use [tensor.NewRowCellsView], +// [tensor.Cells1D], and [tensor.As1D] to reshape and reslice the data as needed. // All stats functions skip over NaN's, as a missing value. // Stats functions cannot be computed in parallel, // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type StatsFunc func(in, out tensor.Tensor) +type StatsFunc func(in, out tensor.Tensor) error -// CountFuncOut64 computes the count of non-NaN tensor values, +// CountOut64 computes the count of non-NaN tensor values, // and returns the Float64 output values for subsequent use. -func CountFuncOut64(in, out tensor.Tensor) tensor.Tensor { +func CountOut64(in, out tensor.Tensor) (tensor.Tensor, error) { return VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + 1 @@ -39,13 +38,14 @@ func CountFuncOut64(in, out tensor.Tensor) tensor.Tensor { // CountFunc computes the count of non-NaN tensor values. // See [StatsFunc] for general information. -func CountFunc(in, out tensor.Tensor) { - CountFuncOut64(in, out) +func CountFunc(in, out tensor.Tensor) error { + _, err := CountOut64(in, out) + return err } -// SumFuncOut64 computes the sum of tensor values, +// SumOut64 computes the sum of tensor values, // and returns the Float64 output values for subsequent use. -func SumFuncOut64(in, out tensor.Tensor) tensor.Tensor { +func SumOut64(in, out tensor.Tensor) (tensor.Tensor, error) { return VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + val @@ -55,77 +55,87 @@ func SumFuncOut64(in, out tensor.Tensor) tensor.Tensor { // SumFunc computes the sum of tensor values. // See [StatsFunc] for general information. -func SumFunc(in, out tensor.Tensor) { - SumFuncOut64(in, out) +func SumFunc(in, out tensor.Tensor) error { + _, err := SumOut64(in, out) + return err } // SumAbsFunc computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. -func SumAbsFunc(in, out tensor.Tensor) { - VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func SumAbsFunc(in, out tensor.Tensor) error { + _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) }) }, in, out) + return err } // ProdFunc computes the product of tensor values. // See [StatsFunc] for general information. -func ProdFunc(in, out tensor.Tensor) { - VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func ProdFunc(in, out tensor.Tensor) error { + _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { return agg * val }) }, in, out) + return err } // MinFunc computes the min of tensor values. // See [StatsFunc] for general information. -func MinFunc(in, out tensor.Tensor) { - VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func MinFunc(in, out tensor.Tensor) error { + _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, val) }) }, in, out) + return err } // MaxFunc computes the max of tensor values. // See [StatsFunc] for general information. -func MaxFunc(in, out tensor.Tensor) { - VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func MaxFunc(in, out tensor.Tensor) error { + _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, val) }) }, in, out) + return err } // MinAbsFunc computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MinAbsFunc(in, out tensor.Tensor) { - VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func MinAbsFunc(in, out tensor.Tensor) error { + _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, math.Abs(val)) }) }, in, out) + return err } // MaxAbsFunc computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MaxAbsFunc(in, out tensor.Tensor) { - VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func MaxAbsFunc(in, out tensor.Tensor) error { + _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, math.Abs(val)) }) }, in, out) + return err } -// MeanFuncOut64 computes the mean of tensor values, +// MeanOut64 computes the mean of tensor values, // and returns the Float64 output values for subsequent use. -func MeanFuncOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor) { - sum64 := SumFuncOut64(in, out) +func MeanOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor, err error) { + sum64, err := SumOut64(in, out) + if err != nil { + return + } count := out.AsValues().Clone() - count64 = CountFuncOut64(in, count) + count64, _ = CountOut64(in, count) // if sum works, this works nsub := out.Len() for i := range nsub { c := count64.Float1D(i) @@ -135,20 +145,24 @@ func MeanFuncOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor) { sum64.SetFloat1D(sum64.Float1D(i)/c, i) out.SetFloat1D(sum64.Float1D(i), i) } - return sum64, count64 + return sum64, count64, err } // MeanFunc computes the mean of tensor values. // See [StatsFunc] for general information. -func MeanFunc(in, out tensor.Tensor) { - MeanFuncOut64(in, out) +func MeanFunc(in, out tensor.Tensor) error { + _, _, err := MeanOut64(in, out) + return err } -// SumSqDevFuncOut64 computes the sum of squared mean deviates of tensor values, +// SumSqDevOut64 computes the sum of squared mean deviates of tensor values, // and returns the Float64 output values for subsequent use. -func SumSqDevFuncOut64(in, out tensor.Tensor) (ssd64, mean64, count64 tensor.Tensor) { - mean64, count64 = MeanFuncOut64(in, out) - ssd64 = VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func SumSqDevOut64(in, out tensor.Tensor) (ssd64, mean64, count64 tensor.Tensor, err error) { + mean64, count64, err = MeanOut64(in, out) + if err != nil { + return + } + ssd64, err = VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(val1, val2, agg float64) float64 { dv := val1 - val2 return agg + dv*dv @@ -157,10 +171,13 @@ func SumSqDevFuncOut64(in, out tensor.Tensor) (ssd64, mean64, count64 tensor.Ten return } -// VarFuncOut64 computes the sample variance of tensor values, +// VarOut64 computes the sample variance of tensor values, // and returns the Float64 output values for subsequent use. -func VarFuncOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor) { - var64, mean64, count64 = SumSqDevFuncOut64(in, out) +func VarOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, err error) { + var64, mean64, count64, err = SumSqDevOut64(in, out) + if err != nil { + return + } nsub := out.Len() for i := range nsub { c := count64.Float1D(i) @@ -177,14 +194,18 @@ func VarFuncOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor) // VarFunc computes the sample variance of tensor values. // Squared deviations from mean, divided by n-1. See also [VarPopFunc]. // See [StatsFunc] for general information. -func VarFunc(in, out tensor.Tensor) { - VarFuncOut64(in, out) +func VarFunc(in, out tensor.Tensor) error { + _, _, _, err := VarOut64(in, out) + return err } -// StdFuncOut64 computes the sample standard deviation of tensor values. +// StdOut64 computes the sample standard deviation of tensor values. // and returns the Float64 output values for subsequent use. -func StdFuncOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor) { - std64, mean64, count64 = VarFuncOut64(in, out) +func StdOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor, err error) { + std64, mean64, count64, err = VarOut64(in, out) + if err != nil { + return + } nsub := out.Len() for i := range nsub { std := math.Sqrt(std64.Float1D(i)) @@ -197,15 +218,19 @@ func StdFuncOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor) // StdFunc computes the sample standard deviation of tensor values. // Sqrt of variance from [VarFunc]. See also [StdPopFunc]. // See [StatsFunc] for general information. -func StdFunc(in, out tensor.Tensor) { - StdFuncOut64(in, out) +func StdFunc(in, out tensor.Tensor) error { + _, _, _, err := StdOut64(in, out) + return err } // SemFunc computes the sample standard error of the mean of tensor values. // Standard deviation [StdFunc] / sqrt(n). See also [SemPopFunc]. // See [StatsFunc] for general information. -func SemFunc(in, out tensor.Tensor) { - var64, _, count64 := VarFuncOut64(in, out) +func SemFunc(in, out tensor.Tensor) error { + var64, _, count64, err := VarOut64(in, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { c := count64.Float1D(i) @@ -215,12 +240,16 @@ func SemFunc(in, out tensor.Tensor) { out.SetFloat1D(math.Sqrt(var64.Float1D(i))/math.Sqrt(c), i) } } + return nil } -// VarPopFuncOut64 computes the population variance of tensor values. +// VarPopOut64 computes the population variance of tensor values. // and returns the Float64 output values for subsequent use. -func VarPopFuncOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor) { - var64, mean64, count64 = SumSqDevFuncOut64(in, out) +func VarPopOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, err error) { + var64, mean64, count64, err = SumSqDevOut64(in, out) + if err != nil { + return + } nsub := out.Len() for i := range nsub { c := count64.Float1D(i) @@ -236,26 +265,34 @@ func VarPopFuncOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tenso // VarPopFunc computes the population variance of tensor values. // Squared deviations from mean, divided by n. See also [VarFunc]. // See [StatsFunc] for general information. -func VarPopFunc(in, out tensor.Tensor) { - VarPopFuncOut64(in, out) +func VarPopFunc(in, out tensor.Tensor) error { + _, _, _, err := VarPopOut64(in, out) + return err } // StdPopFunc computes the population standard deviation of tensor values. // Sqrt of variance from [VarPopFunc]. See also [StdFunc]. // See [StatsFunc] for general information. -func StdPopFunc(in, out tensor.Tensor) { - var64, _, _ := VarPopFuncOut64(in, out) +func StdPopFunc(in, out tensor.Tensor) error { + var64, _, _, err := VarPopOut64(in, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { out.SetFloat1D(math.Sqrt(var64.Float1D(i)), i) } + return nil } // SemPopFunc computes the population standard error of the mean of tensor values. // Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. // See [StatsFunc] for general information. -func SemPopFunc(in, out tensor.Tensor) { - var64, _, count64 := VarPopFuncOut64(in, out) +func SemPopFunc(in, out tensor.Tensor) error { + var64, _, count64, err := VarPopOut64(in, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { c := count64.Float1D(i) @@ -265,13 +302,14 @@ func SemPopFunc(in, out tensor.Tensor) { out.SetFloat1D(math.Sqrt(var64.Float1D(i))/math.Sqrt(c), i) } } + return nil } -// SumSqScaleFuncOut64 is a helper for sum-of-squares, returning scale and ss +// SumSqScaleOut64 is a helper for sum-of-squares, returning scale and ss // factors aggregated separately for better numerical stability, per BLAS. // Returns the Float64 output values for subsequent use. -func SumSqScaleFuncOut64(in, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { - scale64, ss64 = Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func SumSqScaleOut64(in, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { + scale64, ss64, err = Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { if val == 0 { return scale, ss @@ -289,10 +327,13 @@ func SumSqScaleFuncOut64(in, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { return } -// SumSqFuncOut64 computes the sum of squares of tensor values, +// SumSqOut64 computes the sum of squares of tensor values, // and returns the Float64 output values for subsequent use. -func SumSqFuncOut64(in, out tensor.Tensor) tensor.Tensor { - scale64, ss64 := SumSqScaleFuncOut64(in, out) +func SumSqOut64(in, out tensor.Tensor) (tensor.Tensor, error) { + scale64, ss64, err := SumSqScaleOut64(in, out) + if err != nil { + return nil, err + } nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) @@ -306,20 +347,24 @@ func SumSqFuncOut64(in, out tensor.Tensor) tensor.Tensor { scale64.SetFloat1D(v, i) out.SetFloat1D(v, i) } - return scale64 + return scale64, nil } // SumSqFunc computes the sum of squares of tensor values, // See [StatsFunc] for general information. -func SumSqFunc(in, out tensor.Tensor) { - SumSqFuncOut64(in, out) +func SumSqFunc(in, out tensor.Tensor) error { + _, err := SumSqOut64(in, out) + return err } -// L2NormFuncOut64 computes the square root of the sum of squares of tensor values, +// L2NormOut64 computes the square root of the sum of squares of tensor values, // known as the L2 norm, and returns the Float64 output values for // use in subsequent computations. -func L2NormFuncOut64(in, out tensor.Tensor) tensor.Tensor { - scale64, ss64 := SumSqScaleFuncOut64(in, out) +func L2NormOut64(in, out tensor.Tensor) (tensor.Tensor, error) { + scale64, ss64, err := SumSqScaleOut64(in, out) + if err != nil { + return nil, err + } nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) @@ -333,12 +378,13 @@ func L2NormFuncOut64(in, out tensor.Tensor) tensor.Tensor { scale64.SetFloat1D(v, i) out.SetFloat1D(v, i) } - return scale64 + return scale64, nil } // L2NormFunc computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsFunc] for general information. -func L2NormFunc(in, out tensor.Tensor) { - L2NormFuncOut64(in, out) +func L2NormFunc(in, out tensor.Tensor) error { + _, err := L2NormOut64(in, out) + return err } diff --git a/tensor/stats/stats/norm.go b/tensor/stats/stats/norm.go index 9304daeddb..3e6c6cd2e1 100644 --- a/tensor/stats/stats/norm.go +++ b/tensor/stats/stats/norm.go @@ -12,40 +12,53 @@ import ( // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. -func ZScore(a, out tensor.Tensor) { +func ZScore(a, out tensor.Tensor) error { mout := tensor.NewFloat64() - std, mean, _ := StdFuncOut64(a, mout) + std, mean, _, err := StdOut64(a, mout) + if err != nil { + return err + } tmath.Sub(a, mean, out) tmath.Div(out, std, out) + return nil } // UnitNorm computes unit normalized values into given output tensor, // subtracting the Min value and dividing by the Max of the remaining numbers. -func UnitNorm(a, out tensor.Tensor) { +func UnitNorm(a, out tensor.Tensor) error { mout := tensor.NewFloat64() - MinFunc(a, mout) + err := MinFunc(a, mout) + if err != nil { + return err + } tmath.Sub(a, mout, out) MaxFunc(out, mout) tmath.Div(out, mout, out) + return nil } // Clamp ensures that all values are within min, max limits, clamping // values to those bounds if they exceed them. min and max args are // treated as scalars (first value used). -func Clamp(in, minv, maxv, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) +func Clamp(in, minv, maxv, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err + } mn := minv.Float1D(0) mx := maxv.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math32.Clamp(tsr[0].Float1D(idx), mn, mx), idx) }, in, out) + return nil } // Binarize results in a binary-valued output by setting // values >= the threshold to 1, else 0. threshold is // treated as a scalar (first value used). -func Binarize(in, threshold, out tensor.Tensor) { - tensor.SetShapeFrom(out, in) +func Binarize(in, threshold, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err + } thr := threshold.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { v := tsr[0].Float1D(idx) @@ -56,4 +69,5 @@ func Binarize(in, threshold, out tensor.Tensor) { } tsr[1].SetFloat1D(v, idx) }, in, out) + return nil } diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 5ac66f9c27..d0e181215b 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -26,13 +26,15 @@ func QuantilesFunc(in, qs, out tensor.Tensor) error { if qs.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err + } sin := tensor.AsRows(in.AsValues()) sin.ExcludeMissing() sin.Sort(tensor.Ascending) - tensor.SetShapeFrom(out, qs) sz := len(sin.Indexes) - 1 // length of our own index list if sz <= 0 { - out.SetZeros() + out.(tensor.Values).SetZeros() return nil } fsz := float64(sz) @@ -60,18 +62,18 @@ func QuantilesFunc(in, qs, out tensor.Tensor) error { // MedianFunc computes the median (50% quantile) of tensor values. // See [StatsFunc] for general information. -func MedianFunc(in, out tensor.Tensor) { - QuantilesFunc(in, tensor.NewFloat64Scalar(.5), out) +func MedianFunc(in, out tensor.Tensor) error { + return QuantilesFunc(in, tensor.NewFloat64Scalar(.5), out) } // Q1Func computes the first quantile (25%) of tensor values. // See [StatsFunc] for general information. -func Q1Func(in, out tensor.Tensor) { - QuantilesFunc(in, tensor.NewFloat64Scalar(.25), out) +func Q1Func(in, out tensor.Tensor) error { + return QuantilesFunc(in, tensor.NewFloat64Scalar(.25), out) } // Q3Func computes the third quantile (75%) of tensor values. // See [StatsFunc] for general information. -func Q3Func(in, out tensor.Tensor) { - QuantilesFunc(in, tensor.NewFloat64Scalar(.75), out) +func Q3Func(in, out tensor.Tensor) error { + return QuantilesFunc(in, tensor.NewFloat64Scalar(.75), out) } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index da4c0ed7c7..70684bdec1 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -111,6 +111,18 @@ func (s Stats) FuncName() string { return "stats." + s.String() } +// Call calls this statistic function on given tensors. +// Output results are in the out tensor. +func (s Stats) Call(in, out tensor.Tensor) { + tensor.Call(s.FuncName(), in, out) +} + +// StatOut calls a standard Stats enum function on given tensor, +// returning output as a newly created tensor. +func (s Stats) CallOut(in tensor.Tensor) tensor.Tensor { + return tensor.CallOut(s.FuncName(), in) +} + // StripPackage removes any package name from given string, // used for naming based on FuncName() which could be custom // or have a package prefix. @@ -121,15 +133,3 @@ func StripPackage(name string) string { } return name } - -// Stat calls a standard Stats enum function on given tensors. -// Output results are in the out tensor. -func Stat(stat Stats, in, out tensor.Tensor) { - tensor.Call(stat.FuncName(), in, out) -} - -// StatOut calls a standard Stats enum function on given tensor, -// returning output as a newly created tensor. -func StatOut(stat Stats, in tensor.Tensor) tensor.Tensor { - return tensor.CallOut(stat.FuncName(), in) -} diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index ee64b757e0..bacbf45be3 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -82,7 +82,7 @@ func TestFuncs64(t *testing.T) { assert.InDelta(t, results[Q3], out.Values[0], tol) for stat := Count; stat < StatsN; stat++ { - Stat(stat, ix, out) + stat.Call(ix, out) assert.InDelta(t, results[stat], out.Values[0], tol) } } @@ -147,7 +147,7 @@ func TestFuncsInt(t *testing.T) { assert.Equal(t, results[L2Norm], out.Values[0]) for stat := Count; stat <= SemPop; stat++ { - Stat(stat, ix, out) + stat.Call(ix, out) assert.Equal(t, results[stat], out.Values[0]) } } @@ -188,31 +188,32 @@ func TestNorm(t *testing.T) { ZScore(oned, oneout) mout := tensor.NewFloat64() - std, mean, _ := stats.StdFuncOut64(oneout, mout) + std, mean, _, err := StdOut64(oneout, mout) + assert.NoError(t, err) assert.InDelta(t, 1.0, std.Float1D(0), 1.0e-6) assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) UnitNorm(oned, oneout) - stats.MinFunc(oneout, mout) + MinFunc(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - stats.MaxFunc(oneout, mout) + MaxFunc(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) minv := tensor.NewFloat64Scalar(0) maxv := tensor.NewFloat64Scalar(1) Clamp(oned, minv, maxv, oneout) - stats.MinFunc(oneout, mout) + MinFunc(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - stats.MaxFunc(oneout, mout) + MaxFunc(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) thr := tensor.NewFloat64Scalar(0.5) Binarize(oned, thr, oneout) - stats.MinFunc(oneout, mout) + MinFunc(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - stats.MaxFunc(oneout, mout) + MaxFunc(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) } diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 398d2daf74..2a561ce8df 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -19,14 +19,17 @@ import ( // and returns the Float64 output tensor for further processing as needed. // It uses the _last_ tensor as the output, allowing for multiple inputs, // as in the case of VarVecFun. -func VectorizeOut64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) tensor.Tensor { +func VectorizeOut64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (tensor.Tensor, error) { n := nfunc(tsr...) if n <= 0 { - return nil + return nil, nil } nt := len(tsr) + osz := tensor.CellsSize(tsr[0].ShapeSizes()) out := tsr[nt-1] - osz := out.ShapeSizes() + if err := tensor.SetShapeSizesMustBeValues(out, osz...); err != nil { + return nil, err + } o64 := tensor.NewFloat64(osz...) etsr := slices.Clone(tsr) etsr[nt-1] = o64 @@ -37,20 +40,23 @@ func VectorizeOut64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr for i := range nsub { out.SetFloat1D(o64.Float1D(i), i) } - return o64 + return o64, nil } // Vectorize2Out64 is a version of the [tensor.Vectorize] function // for stats, which makes two Float64 output tensors for aggregating // and computing values, returning them for final computation. -func Vectorize2Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (out1, out2 tensor.Tensor) { +func Vectorize2Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (out1, out2 tensor.Tensor, err error) { n := nfunc(tsr...) if n <= 0 { - return nil, nil + return nil, nil, nil } nt := len(tsr) + osz := tensor.CellsSize(tsr[0].ShapeSizes()) out := tsr[nt-1] - osz := out.ShapeSizes() + if err = tensor.SetShapeSizesMustBeValues(out, osz...); err != nil { + return + } out1 = tensor.NewFloat64(osz...) out2 = tensor.NewFloat64(osz...) tsrs := slices.Clone(tsr[:nt-1]) @@ -58,30 +64,17 @@ func Vectorize2Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr for idx := range n { fun(idx, tsrs...) } - return out1, out2 -} - -// OutShape returns the output shape based on given input -// shape ints, with outer row dim = 1. -func OutShape(ish ...int) []int { - osh := slices.Clone(ish) - osh[0] = 1 - return osh + return out1, out2, err } // NFunc is the nfun for stats functions, returning number of rows of the -// first tensor, and initializing the _last_ one to hold the output -// with the first, row dimension set to 1. +// first tensor func NFunc(tsr ...tensor.Tensor) int { nt := len(tsr) if nt < 2 { return 0 } - in, out := tsr[0], tsr[nt-1] - _ = out - n := in.DimSize(0) - out.SetShapeSizes(OutShape(in.ShapeSizes()...)...) - return n + return tsr[0].DimSize(0) } // VecFunc is a helper function for stats functions, dealing with iterating over @@ -89,11 +82,12 @@ func NFunc(tsr ...tensor.Tensor) int { // It also skips over NaN missing values. func VecFunc(idx int, in, out tensor.Tensor, ini float64, fun func(val, agg float64) float64) { nsub := out.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) } - val := in.FloatRowCell(idx, i) + val := in.Float1D(si + i) if math.IsNaN(val) { continue } @@ -108,11 +102,12 @@ func VecFunc(idx int, in, out tensor.Tensor, ini float64, fun func(val, agg floa // It also skips over NaN missing values. func Vec2inFunc(idx int, in1, in2, out tensor.Tensor, ini float64, fun func(val1, val2, agg float64) float64) { nsub := out.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) } - val1 := in1.FloatRowCell(idx, i) + val1 := in1.Float1D(si + i) if math.IsNaN(val1) { continue } @@ -127,12 +122,13 @@ func Vec2inFunc(idx int, in1, in2, out tensor.Tensor, ini float64, fun func(val1 // It also skips over NaN missing values. func Vec2outFunc(idx int, in, out1, out2 tensor.Tensor, ini1, ini2 float64, fun func(val, agg1, agg2 float64) (float64, float64)) { nsub := out2.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out1.SetFloat1D(ini1, i) out2.SetFloat1D(ini2, i) } - val := in.FloatRowCell(idx, i) + val := in.Float1D(si + i) if math.IsNaN(val) { continue } diff --git a/tensor/string.go b/tensor/string.go index 3c060e938d..c0f98f0b3e 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -172,10 +172,6 @@ func (tsr *String) Clone() Values { return csr } -func (tsr *String) View() Values { - return &String{*tsr.view()} -} - // CopyFrom copies all avail values from other tensor into this tensor, with an // optimized implementation if the other tensor is of the same type, and // otherwise it goes through appropriate standard type. diff --git a/tensor/table/io.go b/tensor/table/io.go index 649d92ead6..8a18e6c149 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -375,7 +375,7 @@ func (dt *Table) WriteCSVRow(w io.Writer, row int, delim tensor.Delims) error { // WriteCSVRowWriter uses csv.Writer to write one row func (dt *Table) WriteCSVRowWriter(cw *csv.Writer, row int, ncol int) error { prec := -1 - if ps, err := tensor.GetPrecision(dt.Meta); err == nil { + if ps, err := tensor.Precision(dt.Meta); err == nil { prec = ps } var rec []string diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index c16be732c6..70ab02825f 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -126,6 +126,7 @@ func TestCells(t *testing.T) { // fmt.Println(s) ss := in.Tensor.SubSpace(i).(*tensor.Float32) // fmt.Println(ss.Values[:16]) + cl := tensor.AsFloat32Tensor(tensor.Cells1D(in, i)) // fmt.Println(cl.Values[:16]) assert.Equal(t, vals, ss.Values[:16]) diff --git a/tensor/tensorcore/tensorgrid.go b/tensor/tensorcore/tensorgrid.go index 37ed6df590..10218173c2 100644 --- a/tensor/tensorcore/tensorgrid.go +++ b/tensor/tensorcore/tensorgrid.go @@ -274,7 +274,7 @@ func (tg *TensorGrid) Color(val float64) (norm float64, clr color.Color) { func (tg *TensorGrid) UpdateRange() { if !tg.Display.Range.FixMin || !tg.Display.Range.FixMax { - min, max, _, _ := tg.Tensor.Range() + min, max, _, _ := tensor.Range(tg.Tensor.AsValues()) if !tg.Display.Range.FixMin { nmin := minmax.NiceRoundNumber(min, true) // true = below # tg.Display.Range.Min = nmin diff --git a/tensor/tensormpi/tensor.go b/tensor/tensormpi/tensor.go index f4d95a3d16..3f4eaa9a9b 100644 --- a/tensor/tensormpi/tensor.go +++ b/tensor/tensormpi/tensor.go @@ -119,7 +119,7 @@ func ReduceTensor(dest, src tensor.Tensor, comm *mpi.Comm, op mpi.Op) error { } slen := src.Len() if slen != dest.Len() { - tensor.SetShapeFrom(dest, src) + tensor.SetShapeFromMustBeValues(dest, src) } var err error switch dt { diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index 9525b026e3..b1bf340f27 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -7,7 +7,6 @@ package tmath import ( "math" - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -51,337 +50,374 @@ func init() { tensor.AddFunc("Y1", Y1, 1) } -func Abs(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Abs(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Acos(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Acos(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Acosh(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Acosh(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Asin(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Asin(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Asinh(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Asinh(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Atan(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Atan(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Atanh(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Atanh(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Cbrt(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Cbrt(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Ceil(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Ceil(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Cos(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Cos(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Cosh(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Cosh(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Erf(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Erf(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Erfc(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Erfc(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Erfcinv(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Erfcinv(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Erfinv(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Erfinv(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Exp(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Exp(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Exp2(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Exp2(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Expm1(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Expm1(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Floor(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Floor(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Gamma(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Gamma(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func J0(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func J0(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func J1(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func J1(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Log(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Log(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Log10(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Log10(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Log1p(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Log1p(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Log2(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Log2(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Logb(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Logb(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Round(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Round(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func RoundToEven(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func RoundToEven(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Sin(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Sin(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Sinh(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Sinh(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Sqrt(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Sqrt(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Tan(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Tan(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Tanh(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Tanh(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Trunc(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Trunc(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Y0(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Y0(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } -func Y1(in, out tensor.Tensor) { - if err := tensor.SetShapeFromMustBeValues(out, in); errors.Log(err) != nil { - return +func Y1(in, out tensor.Tensor) error { + if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { + return err } tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(idx)), idx) }, in, out) + return nil } /* diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index cb4c810237..a02047bc02 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -13,7 +13,7 @@ import ( ) type onef func(x float64) float64 -type tonef func(in, out tensor.Tensor) +type tonef func(in, out tensor.Tensor) error // Equal does equal testing taking into account NaN func Equal(t *testing.T, trg, val float64) { @@ -69,7 +69,7 @@ func TestMath(t *testing.T) { } } -func TestOps(t *testing.T) { +func TestOpsCall(t *testing.T) { x := tensor.NewIntScalar(1) y := tensor.NewIntScalar(4) a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2)) diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 7b12fbc7c1..0ea56df491 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -5,7 +5,6 @@ package tmath import ( - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -17,13 +16,13 @@ func init() { } // Add adds two tensors into output. -func Add(a, b, out tensor.Tensor) { +func Add(a, b, out tensor.Tensor) error { as, bs, os, err := tensor.AlignShapes(a, b) - if errors.Log(err) != nil { - return + if err != nil { + return err } - if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { - return + if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { + return err } olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { @@ -35,16 +34,17 @@ func Add(a, b, out tensor.Tensor) { bi := tensor.WrapIndex1D(bs, oi...) out.SetFloat1D(tsr[0].Float1D(ai)+tsr[1].Float1D(bi), idx) }, a, b, out) + return nil } // Sub subtracts two tensors into output. -func Sub(a, b, out tensor.Tensor) { +func Sub(a, b, out tensor.Tensor) error { as, bs, os, err := tensor.AlignShapes(a, b) - if errors.Log(err) != nil { - return + if err != nil { + return err } - if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { - return + if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { + return err } olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { @@ -56,16 +56,17 @@ func Sub(a, b, out tensor.Tensor) { bi := tensor.WrapIndex1D(bs, oi...) out.SetFloat1D(tsr[0].Float1D(ai)-tsr[1].Float1D(bi), idx) }, a, b, out) + return nil } // Mul multiplies two tensors into output. -func Mul(a, b, out tensor.Tensor) { +func Mul(a, b, out tensor.Tensor) error { as, bs, os, err := tensor.AlignShapes(a, b) - if errors.Log(err) != nil { - return + if err != nil { + return err } - if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { - return + if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { + return err } olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { @@ -77,16 +78,17 @@ func Mul(a, b, out tensor.Tensor) { bi := tensor.WrapIndex1D(bs, oi...) out.SetFloat1D(tsr[0].Float1D(ai)*tsr[1].Float1D(bi), idx) }, a, b, out) + return nil } // Div divides two tensors into output. -func Div(a, b, out tensor.Tensor) { +func Div(a, b, out tensor.Tensor) error { as, bs, os, err := tensor.AlignShapes(a, b) - if errors.Log(err) != nil { - return + if err != nil { + return err } - if err := tensor.SetShapeMustBeValues(out, os); errors.Log(err) != nil { - return + if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { + return err } olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { @@ -98,4 +100,5 @@ func Div(a, b, out tensor.Tensor) { bi := tensor.WrapIndex1D(bs, oi...) out.SetFloat1D(tsr[0].Float1D(ai)/tsr[1].Float1D(bi), idx) }, a, b, out) + return nil } diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 585646eaf2..db0543a557 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestAdd(t *testing.T) { +func TestOps(t *testing.T) { scalar := tensor.NewFloat64Scalar(-5.5) scb := scalar.Clone() scb.SetFloat1D(-4.0, 0) diff --git a/tensor/values.go b/tensor/values.go index 5cd6b22472..03701b731b 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -42,12 +42,6 @@ type Values interface { // New tensors always start out with zeros. SetZeros() - // View clones this tensor, *keeping the same underlying Values slice*, - // instead of making a copy like [Values.Clone] does. A major use of this - // is to then change the shape of the view to provide a different way - // of accessing the same data. See [New1DViewOf] for example. - View() Values - // Clone clones this tensor, creating a duplicate copy of itself with its // own separate memory representation of all the values. Clone() Values From ddcb8b8022146a7114857bbcd631589505398863 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 03:36:12 -0700 Subject: [PATCH 102/311] metric building but getting one testing error --- tensor/base.go | 7 -- tensor/bool.go | 23 ++---- tensor/matrix/matrix.go | 10 +-- tensor/number.go | 6 +- tensor/rowmajor.go | 6 -- tensor/rows.go | 4 +- tensor/stats/metric/funcs.go | 141 ++++++++++++++++++++++------------ tensor/stats/metric/matrix.go | 100 +++++++++++++++--------- tensor/stats/metric/misc.go | 11 ++- tensor/stats/metric/vec.go | 47 ++++++------ tensor/string.go | 6 +- tensor/table/indexes.go | 2 +- tensor/table/io.go | 2 +- tensor/tensor_test.go | 4 +- 14 files changed, 209 insertions(+), 160 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index 1d8c488c9f..88ddf2856b 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -50,13 +50,6 @@ func (tsr *Base[T]) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension. func (tsr *Base[T]) DimSize(dim int) int { return tsr.shape.DimSize(dim) } -// RowCellSize returns the size of the outermost Row shape dimension, -// and the size of all the remaining inner dimensions (the "cell" size). -// Used for Tensors that are columns in a data table. -func (tsr *Base[T]) RowCellSize() (rows, cells int) { - return tsr.shape.RowCellSize() -} - // DataType returns the type of the data elements in the tensor. // Bool is returned for the Bool tensor type. func (tsr *Base[T]) DataType() reflect.Kind { diff --git a/tensor/bool.go b/tensor/bool.go index b7bc4cac1f..9a88805876 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -101,13 +101,6 @@ func (tsr *Bool) NumDims() int { return tsr.shape.NumDims() } // DimSize returns size of given dimension func (tsr *Bool) DimSize(dim int) int { return tsr.shape.DimSize(dim) } -// RowCellSize returns the size of the outermost Row shape dimension, -// and the size of all the remaining inner dimensions (the "cell" size). -// Used for Tensors that are columns in a data table. -func (tsr *Bool) RowCellSize() (rows, cells int) { - return tsr.shape.RowCellSize() -} - func (tsr *Bool) SetShapeSizes(sizes ...int) { tsr.shape.SetShapeSizes(sizes...) nln := tsr.Len() @@ -157,13 +150,13 @@ func (tsr *Bool) SetString(val string, i ...int) { } func (tsr *Bool) StringRowCell(row, cell int) string { - _, sz := tsr.RowCellSize() + _, sz := tsr.shape.RowCellSize() return reflectx.ToString(tsr.Values.Index(row*sz + cell)) } func (tsr *Bool) SetStringRowCell(val string, row, cell int) { if bv, err := reflectx.ToBool(val); err == nil { - _, sz := tsr.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values.Set(bv, row*sz+cell) } } @@ -195,12 +188,12 @@ func (tsr *Bool) SetFloat1D(val float64, off int) { } func (tsr *Bool) FloatRowCell(row, cell int) float64 { - _, sz := tsr.RowCellSize() + _, sz := tsr.shape.RowCellSize() return BoolToFloat64(tsr.Values.Index(row*sz + cell)) } func (tsr *Bool) SetFloatRowCell(val float64, row, cell int) { - _, sz := tsr.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values.Set(Float64ToBool(val), row*sz+cell) } @@ -231,12 +224,12 @@ func (tsr *Bool) SetInt1D(val int, off int) { } func (tsr *Bool) IntRowCell(row, cell int) int { - _, sz := tsr.RowCellSize() + _, sz := tsr.shape.RowCellSize() return BoolToInt(tsr.Values.Index(row*sz + cell)) } func (tsr *Bool) SetIntRowCell(val int, row, cell int) { - _, sz := tsr.RowCellSize() + _, sz := tsr.shape.RowCellSize() tsr.Values.Set(IntToBool(val), row*sz+cell) } @@ -311,8 +304,8 @@ func (tsr *Bool) CopyFrom(frm Values) { // is of the same type, and otherwise it goes through // appropriate standard type. func (tsr *Bool) AppendFrom(frm Values) error { - rows, cell := tsr.RowCellSize() - frows, fcell := frm.RowCellSize() + rows, cell := tsr.shape.RowCellSize() + frows, fcell := frm.Shape().RowCellSize() if cell != fcell { return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) } diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go index a5f49d1482..87fda82fc0 100644 --- a/tensor/matrix/matrix.go +++ b/tensor/matrix/matrix.go @@ -29,13 +29,13 @@ func NewMatrix(tsr tensor.Tensor) (*Matrix, error) { err := errors.New("matrix.NewMatrix: tensor is not 2D") return nil, err } - return &Matrix{Tensor: tsr} + return &Matrix{Tensor: tsr}, nil } // Dims is the gonum/mat.Matrix interface method for returning the // dimension sizes of the 2D Matrix. Assumes Row-major ordering. func (mx *Matrix) Dims() (r, c int) { - return tsr.shape.DimSize(0), tsr.shape.DimSize(1) + return mx.Tensor.DimSize(0), mx.Tensor.DimSize(1) } // At is the gonum/mat.Matrix interface method for returning 2D @@ -47,7 +47,7 @@ func (mx *Matrix) At(i, j int) float64 { // T is the gonum/mat.Matrix transpose method. // It performs an implicit transpose by returning the receiver inside a Transpose. func (mx *Matrix) T() mat.Matrix { - return mat.Transpose{tsr} + return mat.Transpose{mx} } ///////////////////////// Symmetric @@ -76,7 +76,7 @@ func NewSymmetric(tsr tensor.Tensor) (*Symmetric, error) { } sy := &Symmetric{} sy.Tensor = tsr - return sy + return sy, nil } // SymmetricDim is the gonum/mat.Matrix interface method for returning the @@ -89,7 +89,7 @@ func (sy *Symmetric) SymmetricDim() (r int) { // using standard Float64 interface func CopyDense(to tensor.Tensor, dm *mat.Dense) { nr, nc := dm.Dims() - to.SetShapeSizes(nr, nc) + tensor.SetShapeSizesMustBeValues(to, nr, nc) idx := 0 for ri := 0; ri < nr; ri++ { for ci := 0; ci < nc; ci++ { diff --git a/tensor/number.go b/tensor/number.go index c773b5a85a..abb62354f4 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -270,8 +270,8 @@ func (tsr *Number[T]) CopyFrom(frm Values) { // is of the same type, and otherwise it goes through // appropriate standard type. func (tsr *Number[T]) AppendFrom(frm Values) error { - rows, cell := tsr.RowCellSize() - frows, fcell := frm.RowCellSize() + rows, cell := tsr.shape.RowCellSize() + frows, fcell := frm.Shape().RowCellSize() if cell != fcell { return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) } @@ -324,7 +324,7 @@ func (tsr *Number[T]) RowTensor(row int) Values { // SetRowTensor sets the values of the SubSpace at given row to given values. func (tsr *Number[T]) SetRowTensor(val Values, row int) { - _, cells := tsr.RowCellSize() + _, cells := tsr.shape.RowCellSize() st := row * cells mx := min(val.Len(), cells) tsr.CopyCellsFrom(val, st, 0, mx) diff --git a/tensor/rowmajor.go b/tensor/rowmajor.go index 2e331fdaef..d700d39999 100644 --- a/tensor/rowmajor.go +++ b/tensor/rowmajor.go @@ -14,12 +14,6 @@ package tensor type RowMajor interface { Tensor - // RowCellSize returns the size of the outermost Row shape dimension, - // and the size of all the remaining inner dimensions (the "cell" size). - // Commonly used to organize multiple instances (rows) of higher-dimensional - // patterns (cells), and the [Rows] type operates on the outer row dimension. - RowCellSize() (rows, cells int) - // SubSpace returns a new tensor with innermost subspace at given // offset(s) in outermost dimension(s) (len(offs) < [NumDims]). // The new tensor points to the values of the this tensor (i.e., modifications diff --git a/tensor/rows.go b/tensor/rows.go index 411c2b75ea..c07dcdbf4c 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -117,7 +117,7 @@ func (rw *Rows) Shape() *Shape { // as NumRows() * cell size. func (rw *Rows) Len() int { rows := rw.NumRows() - _, cells := rw.Tensor.RowCellSize() + _, cells := rw.Tensor.Shape().RowCellSize() return cells * rows } @@ -134,7 +134,7 @@ func (rw *Rows) DimSize(dim int) int { // (via [Rows.NumRows] method), and the size of all the remaining // inner dimensions (the "cell" size). func (rw *Rows) RowCellSize() (rows, cells int) { - _, cells = rw.Tensor.RowCellSize() + _, cells = rw.Tensor.Shape().RowCellSize() rows = rw.NumRows() return } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 4842d25a5d..4f504dbc21 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -28,12 +28,12 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type MetricFunc func(a, b, out tensor.Tensor) +type MetricFunc func(a, b, out tensor.Tensor) error -// SumSquaresScaleFuncOut64 computes the sum of squares differences between tensor values, +// SumSquaresScaleOut64 computes the sum of squares differences between tensor values, // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. -func SumSquaresScaleFuncOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { - scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func SumSquaresScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { + scale64, ss64, err = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { return a - b }) @@ -41,10 +41,13 @@ func SumSquaresScaleFuncOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Ten return } -// SumSquaresFuncOut64 computes the sum of squares differences between tensor values, +// SumSquaresOut64 computes the sum of squares differences between tensor values, // and returns the Float64 output values for use in subsequent computations. -func SumSquaresFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { - scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) +func SumSquaresOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { + scale64, ss64, err := SumSquaresScaleOut64(a, b, out) + if err != nil { + return nil, err + } nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) @@ -58,19 +61,23 @@ func SumSquaresFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { scale64.SetFloat1D(v, i) out.SetFloat1D(v, i) } - return scale64 + return scale64, err } // SumSquaresFunc computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. -func SumSquaresFunc(a, b, out tensor.Tensor) { - SumSquaresFuncOut64(a, b, out) +func SumSquaresFunc(a, b, out tensor.Tensor) error { + _, err := SumSquaresOut64(a, b, out) + return err } // EuclideanFunc computes the Euclidean square root of the sum of squares // differences between tensor values, aka the L2 Norm. -func EuclideanFunc(a, b, out tensor.Tensor) { - scale64, ss64 := SumSquaresScaleFuncOut64(a, b, out) +func EuclideanFunc(a, b, out tensor.Tensor) error { + scale64, ss64, err := SumSquaresScaleOut64(a, b, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) @@ -84,24 +91,26 @@ func EuclideanFunc(a, b, out tensor.Tensor) { scale64.SetFloat1D(v, i) out.SetFloat1D(v, i) } + return nil } // AbsFunc computes the sum of the absolute value of differences between the // tensor values, aka the L1 Norm. // See [MetricFunc] for general information. -func AbsFunc(a, b, out tensor.Tensor) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func AbsFunc(a, b, out tensor.Tensor) error { + _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + math.Abs(a-b) }) }, a, b, out) + return err } // HammingFunc computes the sum of 1s for every element that is different, // i.e., "city block" distance. // See [MetricFunc] for general information. -func HammingFunc(a, b, out tensor.Tensor) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func HammingFunc(a, b, out tensor.Tensor) error { + _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { if a != b { agg += 1 @@ -109,13 +118,14 @@ func HammingFunc(a, b, out tensor.Tensor) { return agg }) }, a, b, out) + return err } -// SumSquaresBinTolScaleFuncOut64 computes the sum of squares differences between tensor values, +// SumSquaresBinTolScaleOut64 computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. -func SumSquaresBinTolScaleFuncOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor) { - scale64, ss64 = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func SumSquaresBinTolScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { + scale64, ss64, err = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { d := a - b if math.Abs(d) < 0.5 { @@ -130,8 +140,11 @@ func SumSquaresBinTolScaleFuncOut64(a, b, out tensor.Tensor) (scale64, ss64 tens // EuclideanBinTolFunc computes the Euclidean square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. -func EuclideanBinTolFunc(a, b, out tensor.Tensor) { - scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) +func EuclideanBinTolFunc(a, b, out tensor.Tensor) error { + scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) @@ -145,12 +158,16 @@ func EuclideanBinTolFunc(a, b, out tensor.Tensor) { scale64.SetFloat1D(v, i) out.SetFloat1D(v, i) } + return nil } // SumSquaresBinTolFunc computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTolFunc(a, b, out tensor.Tensor) { - scale64, ss64 := SumSquaresBinTolScaleFuncOut64(a, b, out) +func SumSquaresBinTolFunc(a, b, out tensor.Tensor) error { + scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { scale := scale64.Float1D(i) @@ -164,6 +181,7 @@ func SumSquaresBinTolFunc(a, b, out tensor.Tensor) { scale64.SetFloat1D(v, i) out.SetFloat1D(v, i) } + return nil } // CrossEntropyFunc is a standard measure of the difference between two @@ -173,8 +191,8 @@ func SumSquaresBinTolFunc(a, b, out tensor.Tensor) { // using Kullback-Leibler (KL) divergence. It is computed as: // a * log(a/b) + (1-a) * log(1-a/1-b). // See [MetricFunc] for general information. -func CrossEntropyFunc(a, b, out tensor.Tensor) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func CrossEntropyFunc(a, b, out tensor.Tensor) error { + _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { b = math32.Clamp(b, 0.000001, 0.999999) if a >= 1.0 { @@ -187,25 +205,30 @@ func CrossEntropyFunc(a, b, out tensor.Tensor) { return agg }) }, a, b, out) + return err } // InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. // See [MetricFunc] for general information. -func InnerProductFunc(a, b, out tensor.Tensor) { - stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func InnerProductFunc(a, b, out tensor.Tensor) error { + _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + a*b }) }, a, b, out) + return err } // CovarianceFunc computes the co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. -func CovarianceFunc(a, b, out tensor.Tensor) { - amean, acount := stats.MeanFuncOut64(a, out) - bmean, _ := stats.MeanFuncOut64(b, out) - cov64 := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func CovarianceFunc(a, b, out tensor.Tensor) error { + amean, acount, err := stats.MeanOut64(a, out) + if err != nil { + return err + } + bmean, _, _ := stats.MeanOut64(b, out) + cov64, _ := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2inFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, am, bm, agg float64) float64 { return agg + (a-am)*(b-bm) }) @@ -219,19 +242,23 @@ func CovarianceFunc(a, b, out tensor.Tensor) { cov64.SetFloat1D(cov64.Float1D(i)/c, i) out.SetFloat1D(cov64.Float1D(i), i) } + return nil } -// CorrelationFuncOut64 computes the correlation between two vectors, +// CorrelationOut64 computes the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized covariance). // Equivalent to the cosine of mean-normalized vectors. // Returns the Float64 output values for subsequent use. -func CorrelationFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { - amean, _ := stats.MeanFuncOut64(a, out) - bmean, _ := stats.MeanFuncOut64(b, out) - ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func CorrelationOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { + amean, _, err := stats.MeanOut64(a, out) + if err != nil { + return nil, err + } + bmean, _, _ := stats.MeanOut64(b, out) + ss64, avar64, bvar64, err := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2in3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], tsr[5], tsr[6], 0, func(a, b, am, bm, ss, avar, bvar float64) (float64, float64, float64) { ad := a - am bd := b - bm @@ -241,6 +268,9 @@ func CorrelationFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { return ss, avar, bvar }) }, a, b, amean, bmean, out) + if err != nil { + return nil, err + } nsub := out.Len() for i := range nsub { @@ -252,7 +282,7 @@ func CorrelationFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { ss64.SetFloat1D(ss, i) out.SetFloat1D(ss, i) } - return ss64 + return ss64, nil } // CorrelationFunc computes the correlation between two vectors, @@ -261,8 +291,9 @@ func CorrelationFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized [CovarianceFunc]). // Equivalent to the [CosineFunc] of mean-normalized vectors. -func CorrelationFunc(a, b, out tensor.Tensor) { - CorrelationFuncOut64(a, b, out) +func CorrelationFunc(a, b, out tensor.Tensor) error { + _, err := CorrelationOut64(a, b, out) + return err } // InvCorrelationFunc computes 1 minus the correlation between two vectors, @@ -273,20 +304,24 @@ func CorrelationFunc(a, b, out tensor.Tensor) { // Equivalent to the [CosineFunc] of mean-normalized vectors. // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCorrelationFunc(a, b, out tensor.Tensor) { - cor64 := CorrelationFuncOut64(a, b, out) +func InvCorrelationFunc(a, b, out tensor.Tensor) error { + cor64, err := CorrelationOut64(a, b, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { cor := cor64.Float1D(i) out.SetFloat1D(1-cor, i) } + return nil } -// CosineFuncOut64 computes the high-dimensional angle between two vectors, +// CosineOut64 computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized [InnerProductFunc]: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc]. -func CosineFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { - ss64, avar64, bvar64 := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { +func CosineOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { + ss64, avar64, bvar64, err := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { ss += a * b avar += a * a @@ -294,6 +329,9 @@ func CosineFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { return ss, avar, bvar }) }, a, b, out) + if err != nil { + return nil, err + } nsub := out.Len() for i := range nsub { ss := ss64.Float1D(i) @@ -304,14 +342,15 @@ func CosineFuncOut64(a, b, out tensor.Tensor) tensor.Tensor { ss64.SetFloat1D(ss, i) out.SetFloat1D(ss, i) } - return ss64 + return ss64, nil } // CosineFunc computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc] -func CosineFunc(a, b, out tensor.Tensor) { - CosineFuncOut64(a, b, out) +func CosineFunc(a, b, out tensor.Tensor) error { + _, err := CosineOut64(a, b, out) + return err } // InvCosineFunc computes 1 minus the cosine between two vectors, @@ -319,11 +358,15 @@ func CosineFunc(a, b, out tensor.Tensor) { // inner product / sqrt(ssA * ssB). // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCosineFunc(a, b, out tensor.Tensor) { - cos64 := CosineFuncOut64(a, b, out) +func InvCosineFunc(a, b, out tensor.Tensor) error { + cos64, err := CosineOut64(a, b, out) + if err != nil { + return err + } nsub := out.Len() for i := range nsub { cos := cos64.Float1D(i) out.SetFloat1D(1-cos, i) } + return nil } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index a78cefc17c..f4235454ad 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -5,7 +5,7 @@ package metric import ( - "log/slog" + "errors" "cogentcore.org/core/math32/vecint" "cogentcore.org/core/tensor" @@ -31,12 +31,14 @@ func init() { // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied // to the upper triangular region, for maximum efficiency. -func Matrix(funcName string, in, out tensor.Tensor) { - rows, cells := in.RowCellSize() +func Matrix(funcName string, in, out tensor.Tensor) error { + rows, cells := in.Shape().RowCellSize() if rows == 0 || cells == 0 { - return + return nil + } + if err := tensor.SetShapeSizesMustBeValues(out, rows, rows); err != nil { + return err } - out.SetShapeSizes(rows, rows) mout := tensor.NewFloat64Scalar(0.0) coords := TriangularLIndicies(rows) nc := len(coords) @@ -44,8 +46,8 @@ func Matrix(funcName string, in, out tensor.Tensor) { tensor.VectorizeThreaded(cells*3, func(tsr ...tensor.Tensor) int { return nc }, func(idx int, tsr ...tensor.Tensor) { c := coords[idx] - sa := tensor.NewRowCellsSliced(tsr[0], c.X) - sb := tensor.NewRowCellsSliced(tsr[0], c.Y) + sa := tensor.Cells1D(tsr[0], c.X) + sb := tensor.Cells1D(tsr[0], c.Y) tensor.Call(funcName, sa, sb, mout) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, in, out) @@ -55,6 +57,7 @@ func Matrix(funcName string, in, out tensor.Tensor) { } out.SetFloat(out.Float(c.X, c.Y), c.Y, c.X) } + return nil } // CrossMatrix computes the distance / similarity matrix between @@ -66,16 +69,18 @@ func Matrix(funcName string, in, out tensor.Tensor) { // The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. -func CrossMatrix(funcName string, a, b, out tensor.Tensor) { - arows, acells := a.RowCellSize() +func CrossMatrix(funcName string, a, b, out tensor.Tensor) error { + arows, acells := a.Shape().RowCellSize() if arows == 0 || acells == 0 { - return + return nil } - brows, bcells := b.RowCellSize() + brows, bcells := b.Shape().RowCellSize() if brows == 0 || bcells == 0 { - return + return nil + } + if err := tensor.SetShapeSizesMustBeValues(out, arows, brows); err != nil { + return err } - out.SetShapeSizes(arows, brows) mout := tensor.NewFloat64Scalar(0.0) // note: flops estimating 3 per item on average -- different for different metrics. flops := min(acells, bcells) * 3 @@ -89,6 +94,7 @@ func CrossMatrix(funcName string, a, b, out tensor.Tensor) { tensor.Call(funcName, sa, sb, mout) tsr[2].SetFloat(mout.Float1D(0), ar, br) }, a, b, out) + return nil } // CovarianceMatrix generates the cells x cells square covariance matrix @@ -104,17 +110,17 @@ func CrossMatrix(funcName string, a, b, out tensor.Tensor) { // which is typical in neural network models, and use // Correlation if they are on very different scales, because it effectively rescales). // The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. -func CovarianceMatrix(funcName string, in, out tensor.Tensor) { - rows, cells := in.RowCellSize() +func CovarianceMatrix(funcName string, in, out tensor.Tensor) error { + rows, cells := in.Shape().RowCellSize() if rows == 0 || cells == 0 { - return + return nil } - - flatvw := in.View() - flatvw.SetShapeSizes(in.DimSize(0), cells) - mout := tensor.NewFloat64Scalar(0.0) - out.SetShapeSizes(cells, cells) + if err := tensor.SetShapeSizesMustBeValues(out, cells, cells); err != nil { + return err + } + flatvw := tensor.NewReshaped(in, rows, cells) + var av, bv tensor.Tensor curCoords := vecint.Vector2i{-1, -1} @@ -141,6 +147,7 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { } out.SetFloat(out.Float(c.X, c.Y), c.Y, c.X) } + return nil } // PCA performs the eigen decomposition of the given CovarianceMatrix, @@ -150,16 +157,22 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) { // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *lowest* to *highest*. // Note that PCA produces results in the *opposite* order of [SVD]. -func PCA(covar, eigenvecs, vals tensor.Tensor) { +func PCA(covar, eigenvecs, vals tensor.Tensor) error { n := covar.DimSize(0) - cv := tensor.AsFloat64Tensor(covar) - eigenvecs.SetShapeSizes(n, n) - vals.SetShapeSizes(n) + cv, err := matrix.NewSymmetric(tensor.AsFloat64Tensor(covar)) + if err != nil { + return err + } + if err := tensor.SetShapeSizesMustBeValues(eigenvecs, n, n); err != nil { + return err + } + if err := tensor.SetShapeSizesMustBeValues(vals, n); err != nil { + return err + } var eig mat.EigenSym ok := eig.Factorize(cv, true) if !ok { - slog.Error("gonum mat.EigenSym Factorize failed") - return + return errors.New("gonum mat.EigenSym Factorize failed") } var ev mat.Dense eig.VectorsTo(&ev) @@ -167,8 +180,9 @@ func PCA(covar, eigenvecs, vals tensor.Tensor) { fv := tensor.AsFloat64Tensor(vals) eig.Values(fv.Values) if fv != vals { - vals.CopyFrom(fv) + vals.(tensor.Values).CopyFrom(fv) } + return nil } // SVD performs the eigen decomposition of the given CovarianceMatrix, @@ -178,16 +192,22 @@ func PCA(covar, eigenvecs, vals tensor.Tensor) { // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *highest* to *lowest*. // Note that SVD produces results in the *opposite* order of [PCA]. -func SVD(covar, eigenvecs, vals tensor.Tensor) { +func SVD(covar, eigenvecs, vals tensor.Tensor) error { n := covar.DimSize(0) - cv := tensor.AsFloat64Tensor(covar) - eigenvecs.SetShapeSizes(n, n) - vals.SetShapeSizes(n) + cv, err := matrix.NewSymmetric(tensor.AsFloat64Tensor(covar)) + if err != nil { + return err + } + if err := tensor.SetShapeSizesMustBeValues(eigenvecs, n, n); err != nil { + return err + } + if err := tensor.SetShapeSizesMustBeValues(vals, n); err != nil { + return err + } var eig mat.SVD ok := eig.Factorize(cv, mat.SVDFull) // todo: benchmark different versions if !ok { - slog.Error("gonum mat.SVD Factorize failed") - return + return errors.New("gonum mat.SVD Factorize failed") } var ev mat.Dense eig.UTo(&ev) @@ -195,8 +215,9 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) { fv := tensor.AsFloat64Tensor(vals) eig.Values(fv.Values) if fv != vals { - vals.CopyFrom(fv) + vals.(tensor.Values).CopyFrom(fv) } + return nil } // ProjectOnMatrixColumn is a convenience function for projecting given vector @@ -206,15 +227,17 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) { // and each row of cells is projected through the matrix column, producing a // 1D output with the number of rows. Otherwise a single number is produced. // This is typically done with results from SVD or PCA. -func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) { +func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) error { ci := int(colindex.Float1D(0)) col := tensor.NewSliced(mtx, tensor.Slice{}, tensor.Slice{Start: ci, Stop: ci + 1}) // fmt.Println(mtx.String(), col.String()) - rows, cells := vec.RowCellSize() + rows, cells := vec.Shape().RowCellSize() mout := tensor.NewFloat64() if rows > 0 && cells > 0 { msum := tensor.NewFloat64Scalar(0) - out.SetShapeSizes(rows) + if err := tensor.SetShapeSizesMustBeValues(out, rows); err != nil { + return err + } for i := range rows { tmath.Mul(tensor.Cells1D(vec, i), col, mout) stats.SumFunc(mout, msum) @@ -225,6 +248,7 @@ func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) { tmath.Mul(vec, col, mout) stats.SumFunc(mout, out) } + return nil } //////////////////////////////////////////// diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index d15de0482f..65bcd44058 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -22,12 +22,15 @@ func init() { // Note: this does _not_ use any existing Indexes for the probe, // but does for the vocab, and the returned index is the logical index // into any existing Indexes. -func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) { - rows, _ := vocab.RowCellSize() +func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) error { + if err := tensor.SetShapeSizesMustBeValues(out, 2); err != nil { + return err + } + rows, _ := vocab.Shape().RowCellSize() mi := -1 mout := tensor.NewFloat64Scalar(0.0) mind := math.MaxFloat64 - pr1d := tensor.New1DViewOf(probe) + pr1d := tensor.As1D(probe) for ri := range rows { sub := tensor.Cells1D(vocab, ri) tensor.Call(funcName, pr1d, sub, mout) @@ -37,7 +40,7 @@ func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) { mind = d } } - out.SetShapeSizes(2) out.SetFloat1D(float64(mi), 0) out.SetFloat1D(mind, 1) + return nil } diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 42f20e6d69..9ac113f18f 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -14,14 +14,17 @@ import ( // Vectorize3Out64 is a version of the [tensor.Vectorize] function // for metrics, which makes three Float64 output tensors for aggregating // and computing values, returning them for final computation. -func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (out1, out2, out3 tensor.Tensor) { +func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr ...tensor.Tensor), tsr ...tensor.Tensor) (out1, out2, out3 tensor.Tensor, err error) { n := nfunc(tsr...) if n <= 0 { - return nil, nil, nil + return nil, nil, nil, nil } nt := len(tsr) + osz := tensor.CellsSize(tsr[0].ShapeSizes()) out := tsr[nt-1] - osz := out.ShapeSizes() + if err := tensor.SetShapeSizesMustBeValues(out, osz...); err != nil { + return nil, nil, nil, err + } out1 = tensor.NewFloat64(osz...) out2 = tensor.NewFloat64(osz...) out3 = tensor.NewFloat64(osz...) @@ -30,15 +33,7 @@ func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr for idx := range n { fun(idx, tsrs...) } - return out1, out2, out3 -} - -// OutShape returns the output shape based on given input -// shape ints, with outer row dim = 1. -func OutShape(ish ...int) []int { - osh := slices.Clone(ish) - osh[0] = 1 - return osh + return out1, out2, out3, nil } // NFunc is the nfun for metrics functions, returning the min number of rows across the @@ -49,9 +44,8 @@ func NFunc(tsr ...tensor.Tensor) int { if nt < 3 { return 0 } - a, b, out := tsr[0], tsr[1], tsr[nt-1] + a, b := tsr[0], tsr[1] na, nb := a.DimSize(0), b.DimSize(0) - out.SetShapeSizes(OutShape(a.ShapeSizes()...)...) return min(na, nb) } @@ -60,15 +54,16 @@ func NFunc(tsr ...tensor.Tensor) int { // It also skips over NaN missing values. func VecFunc(idx int, a, b, out tensor.Tensor, ini float64, fun func(a, b, agg float64) float64) { nsub := out.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) } - av := a.FloatRowCell(idx, i) + av := a.Float1D(si + i) if math.IsNaN(av) { continue } - bv := b.FloatRowCell(idx, i) + bv := b.Float1D(si + i) if math.IsNaN(bv) { continue } @@ -82,16 +77,17 @@ func VecFunc(idx int, a, b, out tensor.Tensor, ini float64, fun func(a, b, agg f // It also skips over NaN missing values. func VecSSFunc(idx int, a, b, out1, out2 tensor.Tensor, ini1, ini2 float64, fun func(a, b float64) float64) { nsub := out2.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out1.SetFloat1D(ini1, i) out2.SetFloat1D(ini2, i) } - av := a.FloatRowCell(idx, i) + av := a.Float1D(si + i) if math.IsNaN(av) { continue } - bv := b.FloatRowCell(idx, i) + bv := b.Float1D(si + i) if math.IsNaN(bv) { continue } @@ -118,15 +114,16 @@ func VecSSFunc(idx int, a, b, out1, out2 tensor.Tensor, ini1, ini2 float64, fun // e.g., the mean. It also skips over NaN missing values. func Vec2inFunc(idx int, a, b, a2, b2, out tensor.Tensor, ini float64, fun func(a, b, a2, b2, agg float64) float64) { nsub := out.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out.SetFloat1D(ini, i) } - av := a.FloatRowCell(idx, i) + av := a.Float1D(si + i) if math.IsNaN(av) { continue } - bv := b.FloatRowCell(idx, i) + bv := b.Float1D(si + i) if math.IsNaN(bv) { continue } @@ -142,17 +139,18 @@ func Vec2inFunc(idx int, a, b, a2, b2, out tensor.Tensor, ini float64, fun func( // e.g., the mean. It also skips over NaN missing values. func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 tensor.Tensor, ini float64, fun func(a, b, a2, b2, out1, out2, out3 float64) (float64, float64, float64)) { nsub := out1.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out1.SetFloat1D(ini, i) out2.SetFloat1D(ini, i) out3.SetFloat1D(ini, i) } - av := a.FloatRowCell(idx, i) + av := a.Float1D(si + i) if math.IsNaN(av) { continue } - bv := b.FloatRowCell(idx, i) + bv := b.Float1D(si + i) if math.IsNaN(bv) { continue } @@ -173,17 +171,18 @@ func Vec2in3outFunc(idx int, a, b, a2, b2, out1, out2, out3 tensor.Tensor, ini f // This version has 3 output vectors. It also skips over NaN missing values. func Vec3outFunc(idx int, a, b, out1, out2, out3 tensor.Tensor, ini float64, fun func(a, b, out1, out2, out3 float64) (float64, float64, float64)) { nsub := out1.Len() + si := idx * nsub // 1D start of sub for i := range nsub { if idx == 0 { out1.SetFloat1D(ini, i) out2.SetFloat1D(ini, i) out3.SetFloat1D(ini, i) } - av := a.FloatRowCell(idx, i) + av := a.Float1D(si + i) if math.IsNaN(av) { continue } - bv := b.FloatRowCell(idx, i) + bv := b.Float1D(si + i) if math.IsNaN(bv) { continue } diff --git a/tensor/string.go b/tensor/string.go index c0f98f0b3e..8c49b84e54 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -192,8 +192,8 @@ func (tsr *String) CopyFrom(frm Values) { // is of the same type, and otherwise it goes through // appropriate standard type. func (tsr *String) AppendFrom(frm Values) error { - rows, cell := tsr.RowCellSize() - frows, fcell := frm.RowCellSize() + rows, cell := tsr.shape.RowCellSize() + frows, fcell := frm.Shape().RowCellSize() if cell != fcell { return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) } @@ -248,7 +248,7 @@ func (tsr *String) RowTensor(row int) Values { // SetRowTensor sets the values of the SubSpace at given row to given values. func (tsr *String) SetRowTensor(val Values, row int) { - _, cells := tsr.RowCellSize() + _, cells := tsr.shape.RowCellSize() st := row * cells mx := min(val.Len(), cells) tsr.CopyCellsFrom(val, st, 0, mx) diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 87697be915..3fce2e8fff 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -229,7 +229,7 @@ func (dt *Table) New() *Table { } for ci, cl := range nt.Columns.Values { scl := dt.Columns.Values[ci] - _, csz := cl.RowCellSize() + _, csz := cl.Shape().RowCellSize() for i, srw := range dt.Indexes { cl.CopyCellsFrom(scl, i*csz, srw*csz, csz) } diff --git a/tensor/table/io.go b/tensor/table/io.go index 8a18e6c149..2de6619889 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -122,7 +122,7 @@ func (dt *Table) ReadCSVRow(rec []string, row int) { } nan := math.NaN() for _, tsr := range dt.Columns.Values { - _, csz := tsr.RowCellSize() + _, csz := tsr.Shape().RowCellSize() stoff := row * csz for cc := 0; cc < csz; cc++ { str := rec[ci] diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 36db0a7860..ab02c6c6bb 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -20,7 +20,7 @@ func TestTensorString(t *testing.T) { assert.Equal(t, true, tsr.IsString()) assert.Equal(t, reflect.String, tsr.DataType()) assert.Equal(t, 2, tsr.SubSpace(0).Len()) - r, c := tsr.RowCellSize() + r, c := tsr.Shape().RowCellSize() assert.Equal(t, 4, r) assert.Equal(t, 2, c) @@ -79,7 +79,7 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, false, tsr.IsString()) assert.Equal(t, reflect.Float64, tsr.DataType()) assert.Equal(t, 2, tsr.SubSpace(0).Len()) - r, c := tsr.RowCellSize() + r, c := tsr.Shape().RowCellSize() assert.Equal(t, 4, r) assert.Equal(t, 2, c) From ac9ad1718699486137fd00a8cc9e700620e1c746 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 03:38:45 -0700 Subject: [PATCH 103/311] use Call etc methods on metrics enum --- tensor/stats/metric/metric_test.go | 2 +- tensor/stats/metric/metrics.go | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 567109ab5c..727b890fd3 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -65,7 +65,7 @@ func TestFuncs(t *testing.T) { assert.InDelta(t, results[InvCosine], out.Values[0], tol) for met := Euclidean; met < MetricsN; met++ { - Metric(met, atsr, btsr, out) + met.Call(atsr, btsr, out) assert.InDelta(t, results[met], out.Values[0], tol) } } diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index 81a5969fcd..a353bf41d6 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -26,18 +26,6 @@ func init() { tensor.AddFunc(Cosine.FuncName(), CosineFunc, 1) } -// Metric calls a standard Metrics enum function on given tensors. -// Output results are in the out tensor. -func Metric(metric Metrics, a, b, out tensor.Tensor) { - tensor.Call(metric.FuncName(), a, b, out) -} - -// MetricOut calls a standard Metrics enum function on given tensors, -// returning output as a newly created tensor. -func MetricOut(metric Metrics, a, b tensor.Tensor) tensor.Tensor { - return tensor.CallOut(metric.FuncName(), a, b) -} - // Metrics are standard metric functions type Metrics int32 //enums:enum @@ -112,6 +100,18 @@ func (m Metrics) FuncName() string { return "metric." + m.String() } +// Call calls a standard Metrics enum function on given tensors. +// Output results are in the out tensor. +func (m Metrics) Call(a, b, out tensor.Tensor) { + tensor.Call(m.FuncName(), a, b, out) +} + +// CallOut calls a standard Metrics enum function on given tensors, +// returning output as a newly created tensor. +func (m Metrics) CallOut(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut(m.FuncName(), a, b) +} + // Increasing returns true if the distance metric is such that metric // values increase as a function of distance (e.g., Euclidean) // and false if metric values decrease as a function of distance From ec0666b2ee3c13907fcc3f68556fbe0bd1045c38 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 10:35:51 -0700 Subject: [PATCH 104/311] everything building and testing in tensor. --- tensor/databrowser/datatab.go | 4 ++-- tensor/databrowser/filetree.go | 8 ++++---- tensor/examples/datafs-sim/sim.go | 12 ++++++------ tensor/stats/cluster/cluster.go | 2 +- tensor/stats/glm/glm.go | 12 ++++++------ tensor/stats/metric/matrix.go | 11 +++++++---- tensor/tensormpi/tensor.go | 14 +++++++------- tensor/tmath/math_test.go | 2 +- tensor/tmath/ops_test.go | 2 +- 9 files changed, 35 insertions(+), 32 deletions(-) diff --git a/tensor/databrowser/datatab.go b/tensor/databrowser/datatab.go index 5d78f1adb1..53b430b9b8 100644 --- a/tensor/databrowser/datatab.go +++ b/tensor/databrowser/datatab.go @@ -34,7 +34,7 @@ func NewTab[T any](br *Browser, label string, mkfun func(tab *core.Frame) T) T { func (br *Browser) NewTabTensorTable(label string, dt *table.Table) *tensorcore.Table { tv := NewTab[*tensorcore.Table](br, label, func(tab *core.Frame) *tensorcore.Table { tb := core.NewToolbar(tab) - tv := tensorcore.New(tab) + tv := tensorcore.NewTable(tab) tb.Maker(tv.MakeToolbar) return tv }) @@ -85,7 +85,7 @@ func (br *Browser) NewTabPlot(label string, dt *table.Table) *plotcore.PlotEdito // to view the given slice of structs. func (br *Browser) NewTabSliceTable(label string, slc any) *core.Table { tv := NewTab[*core.Table](br, label, func(tab *core.Frame) *core.Table { - return core.New(tab) + return core.NewTable(tab) }) tv.SetSlice(slc) br.Update() diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index a1c0455767..7c74d78250 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -140,7 +140,7 @@ func (fn *FileNode) OpenFile() error { d.RunDialog(br) default: - dt := table.NewTable() + dt := table.New() err := dt.OpenCSV(fn.Filepath, tensor.Tab) // todo: need more flexible data handling mode if err != nil { core.ErrorSnackbar(br, err) @@ -221,7 +221,7 @@ func (fn *FileNode) PlotFile() { switch fn.Info.Known { case fileinfo.Tensor: tsr := d.Data - dt = table.NewTable(df) + dt = table.New(df) dt.Columns.Rows = tsr.DimSize(0) if ix, ok := tsr.(*tensor.Rows); ok { dt.Indexes = ix.Indexes @@ -230,11 +230,11 @@ func (fn *FileNode) PlotFile() { for r := range dt.Columns.Rows { rc.Values[r] = r } - dt.AddColumn(fn.Name, tsr) + dt.AddColumn(fn.Name, tsr.AsValues()) // case fileinfo.Table: // dt = d.AsTable() default: - dt = table.NewTable(df) + dt = table.New(df) err := dt.OpenCSV(fn.Filepath, tensor.Tab) // todo: need more flexible data handling mode if err != nil { core.ErrorSnackbar(br, err) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 5aef740d7e..c490497c66 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -71,9 +71,9 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { tensor.SetCalcFunc(lt, func() error { trl := ss.Stats.Item("Trial").AsInt() if st.IsString() { - lt.SetStringRow(st.StringRow(0), trl) + lt.SetStringRow(st.String1D(0), trl) } else { - lt.SetFloatRow(st.FloatRow(0), trl) + lt.SetFloatRow(st.Float1D(0), trl) } return nil }) @@ -89,9 +89,9 @@ func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { row := lt.DimSize(0) lt.SetShapeSizes(row + 1) if st.IsString() { - lt.SetStringRow(st.StringRow(0), row) + lt.SetStringRow(st.String1D(0), row) } else { - lt.SetFloatRow(st.FloatRow(0), row) + lt.SetFloatRow(st.Float1D(0), row) } return nil }) @@ -118,7 +118,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a lt := dd.NewOfType(ag.String(), st.DataType(), nctr) lt.Metadata().Copy(*st.Metadata()) tensor.SetCalcFunc(lt, func() error { - stats.Stat(ag, src, stout) + ag.Call(src, stout) ctr := ss.Stats.Item(level).AsInt() lt.SetFloatRow(stout.FloatRow(0), ctr) return nil @@ -128,7 +128,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a lt := logd.NewOfType(nm, st.DataType(), nctr) lt.Metadata().Copy(*st.Metadata()) tensor.SetCalcFunc(lt, func() error { - v := st.FloatRow(0) + v := st.Float1D(0) ctr := ss.Stats.Item(level).AsInt() lt.SetFloatRow(v, ctr) return nil diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index 313bd7ead3..c61797f429 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -114,7 +114,7 @@ func InitAllLeaves(ntot int) *Node { func Glom(root *Node, funcName string, dmat tensor.Tensor) *Node { ntot := dmat.DimSize(0) // number of leaves mout := tensor.NewFloat64Scalar(0) - stats.MaxFunc(tensor.New1DViewOf(dmat), mout) + stats.MaxFunc(tensor.As1D(dmat), mout) maxd := mout.Float1D(0) // indexes in each group aidx := make([]int, ntot) diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index bd7b736ed0..6530864047 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -88,7 +88,7 @@ type GLM struct { Table *table.Table // tensor columns from table with the respective variables - IndepVars, DepVars, PredVars, ErrVars tensor.Tensor + IndepVars, DepVars, PredVars, ErrVars tensor.RowMajor // Number of independent and dependent variables NIndepVars, NDepVars int @@ -126,7 +126,7 @@ func (glm *GLM) init(nIv, nDv int) { func (glm *GLM) SetTable(dt *table.Table, indepVars, depVars, predVars, errVars string) error { iv := dt.Column(indepVars) dv := dt.Column(depVars) - var pv, ev tensor.Tensor + var pv, ev *tensor.Rows if predVars != "" { pv = dt.Column(predVars) } @@ -139,8 +139,8 @@ func (glm *GLM) SetTable(dt *table.Table, indepVars, depVars, predVars, errVars if ev != nil && !ev.Shape().IsEqual(dv.Shape()) { return fmt.Errorf("errVars must have same shape as depVars") } - _, nIv := iv.RowCellSize() - _, nDv := dv.RowCellSize() + _, nIv := iv.Shape().RowCellSize() + _, nDv := dv.Shape().RowCellSize() glm.init(nIv, nDv) glm.Table = dt glm.IndepVars = iv @@ -163,10 +163,10 @@ func (glm *GLM) Run() { ev := glm.ErrVars if pv == nil { - pv = dv.Clone() + pv = tensor.Clone(dv) } if ev == nil { - ev = dv.Clone() + ev = tensor.Clone(dv) } nDv := glm.NDepVars diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index f4235454ad..f5cb89edcd 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -229,22 +229,25 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) error { // This is typically done with results from SVD or PCA. func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) error { ci := int(colindex.Float1D(0)) - col := tensor.NewSliced(mtx, tensor.Slice{}, tensor.Slice{Start: ci, Stop: ci + 1}) + col := tensor.As1D(tensor.NewSliced(mtx, tensor.Slice{}, tensor.Slice{Start: ci, Stop: ci + 1})) // fmt.Println(mtx.String(), col.String()) rows, cells := vec.Shape().RowCellSize() - mout := tensor.NewFloat64() if rows > 0 && cells > 0 { msum := tensor.NewFloat64Scalar(0) if err := tensor.SetShapeSizesMustBeValues(out, rows); err != nil { return err } + mout := tensor.NewFloat64(cells) for i := range rows { - tmath.Mul(tensor.Cells1D(vec, i), col, mout) + err := tmath.Mul(tensor.Cells1D(vec, i), col, mout) + if err != nil { + return err + } stats.SumFunc(mout, msum) - // fmt.Println(tensor.Cells1D(vec, i).String(), mout.String(), msum.String()) out.SetFloat1D(msum.Float1D(0), i) } } else { + mout := tensor.NewFloat64(1) tmath.Mul(vec, col, mout) stats.SumFunc(mout, out) } diff --git a/tensor/tensormpi/tensor.go b/tensor/tensormpi/tensor.go index 3f4eaa9a9b..a2e59425a2 100644 --- a/tensor/tensormpi/tensor.go +++ b/tensor/tensormpi/tensor.go @@ -15,13 +15,13 @@ import ( // using a row-based tensor organization (as in an table.Table). // dest will have np * src.Rows Rows, filled with each processor's data, in order. // dest must have same overall shape as src at start, but rows will be enforced. -func GatherTensorRows(dest, src tensor.Tensor, comm *mpi.Comm) error { +func GatherTensorRows(dest, src tensor.Values, comm *mpi.Comm) error { dt := src.DataType() if dt == reflect.String { return GatherTensorRowsString(dest.(*tensor.String), src.(*tensor.String), comm) } - sr, _ := src.RowCellSize() - dr, _ := dest.RowCellSize() + sr, _ := src.Shape().RowCellSize() + dr, _ := dest.Shape().RowCellSize() np := mpi.WorldSize() dl := np * sr if dr != dl { @@ -62,8 +62,8 @@ func GatherTensorRows(dest, src tensor.Tensor, comm *mpi.Comm) error { // dest will have np * src.Rows Rows, filled with each processor's data, in order. // dest must have same overall shape as src at start, but rows will be enforced. func GatherTensorRowsString(dest, src *tensor.String, comm *mpi.Comm) error { - sr, _ := src.RowCellSize() - dr, _ := dest.RowCellSize() + sr, _ := src.Shape().RowCellSize() + dr, _ := dest.Shape().RowCellSize() np := mpi.WorldSize() dl := np * sr if dr != dl { @@ -112,14 +112,14 @@ func GatherTensorRowsString(dest, src *tensor.String, comm *mpi.Comm) error { // IMPORTANT: src and dest must be different slices! // each processor must have the same shape and organization for this to make sense. // does nothing for strings. -func ReduceTensor(dest, src tensor.Tensor, comm *mpi.Comm, op mpi.Op) error { +func ReduceTensor(dest, src tensor.Values, comm *mpi.Comm, op mpi.Op) error { dt := src.DataType() if dt == reflect.String { return nil } slen := src.Len() if slen != dest.Len() { - tensor.SetShapeFromMustBeValues(dest, src) + tensor.SetShapeFrom(dest, src) } var err error switch dt { diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index a02047bc02..f423198d36 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -36,7 +36,7 @@ func TestMath(t *testing.T) { oneout := oned.Clone() cell2d := tensor.NewFloat32(5, 2, 6) - _, cells := cell2d.RowCellSize() + _, cells := cell2d.Shape().RowCellSize() assert.Equal(t, cells, 12) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { ci := idx % cells diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index db0543a557..f858ff23b6 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -23,7 +23,7 @@ func TestOps(t *testing.T) { oneout := oned.Clone() cell2d := tensor.NewFloat32(5, 12) - _, cells := cell2d.RowCellSize() + _, cells := cell2d.Shape().RowCellSize() assert.Equal(t, cells, 12) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { ci := idx % cells From 1654dff4dfe73aa6fefb8b6d912400a98d053f1d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 10:41:36 -0700 Subject: [PATCH 105/311] everything everywhere should now be building and passing tests.. --- base/randx/dists_test.go | 10 +- enums/enumgen/testdata/enumgen.go | 2 +- examples/plot/plot.go | 2 +- goal/transpile/transpile_test.go | 14 +- types/typegen/testdata/typegen.go | 2 +- .../cogentcore_org-core-base-reflectx.go | 2 + ...cogentcore_org-core-tensor-stats-metric.go | 101 ++-- .../cogentcore_org-core-tensor-stats-stats.go | 151 ++--- .../cogentcore_org-core-tensor-table.go | 2 +- .../cogentcore_org-core-tensor-tmath.go | 96 ++-- .../symbols/cogentcore_org-core-tensor.go | 529 ++++++++++++------ 11 files changed, 552 insertions(+), 359 deletions(-) diff --git a/base/randx/dists_test.go b/base/randx/dists_test.go index 936e26a78b..33da38e3d6 100644 --- a/base/randx/dists_test.go +++ b/base/randx/dists_test.go @@ -15,7 +15,7 @@ import ( func TestGaussianGen(t *testing.T) { nsamp := int(1e6) - dt := table.NewTable() + dt := table.New() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -45,7 +45,7 @@ func TestGaussianGen(t *testing.T) { func TestBinomialGen(t *testing.T) { nsamp := int(1e6) - dt := table.NewTable() + dt := table.New() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -82,7 +82,7 @@ func TestBinomialGen(t *testing.T) { func TestPoissonGen(t *testing.T) { nsamp := int(1e6) - dt := table.NewTable() + dt := table.New() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -119,7 +119,7 @@ func TestPoissonGen(t *testing.T) { func TestGammaGen(t *testing.T) { nsamp := int(1e6) - dt := table.NewTable() + dt := table.New() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) @@ -150,7 +150,7 @@ func TestGammaGen(t *testing.T) { func TestBetaGen(t *testing.T) { nsamp := int(1e6) - dt := table.NewTable() + dt := table.New() dt.AddFloat32Column("Val") dt.SetNumRows(nsamp) diff --git a/enums/enumgen/testdata/enumgen.go b/enums/enumgen/testdata/enumgen.go index 48f2da15e6..33366ee1a4 100644 --- a/enums/enumgen/testdata/enumgen.go +++ b/enums/enumgen/testdata/enumgen.go @@ -1,4 +1,4 @@ -// Code generated by "enumgen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build16270102/b655/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. +// Code generated by "enumgen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build1497819794/b657/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. package testdata diff --git a/examples/plot/plot.go b/examples/plot/plot.go index b84b97d419..32e1c00954 100644 --- a/examples/plot/plot.go +++ b/examples/plot/plot.go @@ -19,7 +19,7 @@ var tsv embed.FS func main() { b := core.NewBody("Plot Example") - epc := table.NewTable("epc") + epc := table.New("epc") epc.OpenFS(tsv, "ra25epoch.tsv", tensor.Tab) pl := plotcore.NewPlotEditor(b) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index c866a98c3e..839d13ff19 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -200,13 +200,13 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - // {"# x := 1", `x := tensor.NewIntScalar(1)`}, - // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64Tensor(3, 4)`}, + {"# x := 1", `x := tensor.NewIntScalar(1)`}, + {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, } st := NewState() diff --git a/types/typegen/testdata/typegen.go b/types/typegen/testdata/typegen.go index 117616b8ec..e14a692b4a 100644 --- a/types/typegen/testdata/typegen.go +++ b/types/typegen/testdata/typegen.go @@ -1,4 +1,4 @@ -// Code generated by "typegen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build16270102/b976/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. +// Code generated by "typegen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build1497819794/b982/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. package testdata diff --git a/yaegicore/symbols/cogentcore_org-core-base-reflectx.go b/yaegicore/symbols/cogentcore_org-core-base-reflectx.go index aef279b5af..3d9d493c96 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-reflectx.go +++ b/yaegicore/symbols/cogentcore_org-core-base-reflectx.go @@ -17,6 +17,8 @@ func init() { "CopySliceRobust": reflect.ValueOf(reflectx.CopySliceRobust), "FormatDefault": reflect.ValueOf(reflectx.FormatDefault), "KindIsBasic": reflect.ValueOf(reflectx.KindIsBasic), + "KindIsFloat": reflect.ValueOf(reflectx.KindIsFloat), + "KindIsInt": reflect.ValueOf(reflectx.KindIsInt), "KindIsNumber": reflect.ValueOf(reflectx.KindIsNumber), "LongTypeName": reflect.ValueOf(reflectx.LongTypeName), "MapAdd": reflect.ValueOf(reflectx.MapAdd), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go index 0c7bcf73e9..96b439de4a 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -10,58 +10,55 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/stats/metric/metric"] = map[string]reflect.Value{ // function, constant and variable definitions - "Abs": reflect.ValueOf(metric.Abs), - "AbsFunc": reflect.ValueOf(metric.AbsFunc), - "ClosestRow": reflect.ValueOf(metric.ClosestRow), - "Correlation": reflect.ValueOf(metric.Correlation), - "CorrelationFunc": reflect.ValueOf(metric.CorrelationFunc), - "CorrelationFuncOut64": reflect.ValueOf(metric.CorrelationFuncOut64), - "Cosine": reflect.ValueOf(metric.Cosine), - "CosineFunc": reflect.ValueOf(metric.CosineFunc), - "CosineFuncOut64": reflect.ValueOf(metric.CosineFuncOut64), - "Covariance": reflect.ValueOf(metric.Covariance), - "CovarianceFunc": reflect.ValueOf(metric.CovarianceFunc), - "CovarianceMatrix": reflect.ValueOf(metric.CovarianceMatrix), - "CrossEntropy": reflect.ValueOf(metric.CrossEntropy), - "CrossEntropyFunc": reflect.ValueOf(metric.CrossEntropyFunc), - "CrossMatrix": reflect.ValueOf(metric.CrossMatrix), - "Euclidean": reflect.ValueOf(metric.Euclidean), - "EuclideanBinTol": reflect.ValueOf(metric.EuclideanBinTol), - "EuclideanBinTolFunc": reflect.ValueOf(metric.EuclideanBinTolFunc), - "EuclideanFunc": reflect.ValueOf(metric.EuclideanFunc), - "Hamming": reflect.ValueOf(metric.Hamming), - "HammingFunc": reflect.ValueOf(metric.HammingFunc), - "InnerProduct": reflect.ValueOf(metric.InnerProduct), - "InnerProductFunc": reflect.ValueOf(metric.InnerProductFunc), - "InvCorrelation": reflect.ValueOf(metric.InvCorrelation), - "InvCorrelationFunc": reflect.ValueOf(metric.InvCorrelationFunc), - "InvCosine": reflect.ValueOf(metric.InvCosine), - "InvCosineFunc": reflect.ValueOf(metric.InvCosineFunc), - "Matrix": reflect.ValueOf(metric.Matrix), - "Metric": reflect.ValueOf(metric.Metric), - "MetricOut": reflect.ValueOf(metric.MetricOut), - "MetricsN": reflect.ValueOf(metric.MetricsN), - "MetricsValues": reflect.ValueOf(metric.MetricsValues), - "NFunc": reflect.ValueOf(metric.NFunc), - "OutShape": reflect.ValueOf(metric.OutShape), - "PCA": reflect.ValueOf(metric.PCA), - "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), - "SVD": reflect.ValueOf(metric.SVD), - "SumSquares": reflect.ValueOf(metric.SumSquares), - "SumSquaresBinTol": reflect.ValueOf(metric.SumSquaresBinTol), - "SumSquaresBinTolFunc": reflect.ValueOf(metric.SumSquaresBinTolFunc), - "SumSquaresBinTolScaleFuncOut64": reflect.ValueOf(metric.SumSquaresBinTolScaleFuncOut64), - "SumSquaresFunc": reflect.ValueOf(metric.SumSquaresFunc), - "SumSquaresFuncOut64": reflect.ValueOf(metric.SumSquaresFuncOut64), - "SumSquaresScaleFuncOut64": reflect.ValueOf(metric.SumSquaresScaleFuncOut64), - "TriangularLIndicies": reflect.ValueOf(metric.TriangularLIndicies), - "TriangularN": reflect.ValueOf(metric.TriangularN), - "Vec2in3outFunc": reflect.ValueOf(metric.Vec2in3outFunc), - "Vec2inFunc": reflect.ValueOf(metric.Vec2inFunc), - "Vec3outFunc": reflect.ValueOf(metric.Vec3outFunc), - "VecFunc": reflect.ValueOf(metric.VecFunc), - "VecSSFunc": reflect.ValueOf(metric.VecSSFunc), - "Vectorize3Out64": reflect.ValueOf(metric.Vectorize3Out64), + "Abs": reflect.ValueOf(metric.Abs), + "AbsFunc": reflect.ValueOf(metric.AbsFunc), + "ClosestRow": reflect.ValueOf(metric.ClosestRow), + "Correlation": reflect.ValueOf(metric.Correlation), + "CorrelationFunc": reflect.ValueOf(metric.CorrelationFunc), + "CorrelationOut64": reflect.ValueOf(metric.CorrelationOut64), + "Cosine": reflect.ValueOf(metric.Cosine), + "CosineFunc": reflect.ValueOf(metric.CosineFunc), + "CosineOut64": reflect.ValueOf(metric.CosineOut64), + "Covariance": reflect.ValueOf(metric.Covariance), + "CovarianceFunc": reflect.ValueOf(metric.CovarianceFunc), + "CovarianceMatrix": reflect.ValueOf(metric.CovarianceMatrix), + "CrossEntropy": reflect.ValueOf(metric.CrossEntropy), + "CrossEntropyFunc": reflect.ValueOf(metric.CrossEntropyFunc), + "CrossMatrix": reflect.ValueOf(metric.CrossMatrix), + "Euclidean": reflect.ValueOf(metric.Euclidean), + "EuclideanBinTol": reflect.ValueOf(metric.EuclideanBinTol), + "EuclideanBinTolFunc": reflect.ValueOf(metric.EuclideanBinTolFunc), + "EuclideanFunc": reflect.ValueOf(metric.EuclideanFunc), + "Hamming": reflect.ValueOf(metric.Hamming), + "HammingFunc": reflect.ValueOf(metric.HammingFunc), + "InnerProduct": reflect.ValueOf(metric.InnerProduct), + "InnerProductFunc": reflect.ValueOf(metric.InnerProductFunc), + "InvCorrelation": reflect.ValueOf(metric.InvCorrelation), + "InvCorrelationFunc": reflect.ValueOf(metric.InvCorrelationFunc), + "InvCosine": reflect.ValueOf(metric.InvCosine), + "InvCosineFunc": reflect.ValueOf(metric.InvCosineFunc), + "Matrix": reflect.ValueOf(metric.Matrix), + "MetricsN": reflect.ValueOf(metric.MetricsN), + "MetricsValues": reflect.ValueOf(metric.MetricsValues), + "NFunc": reflect.ValueOf(metric.NFunc), + "PCA": reflect.ValueOf(metric.PCA), + "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), + "SVD": reflect.ValueOf(metric.SVD), + "SumSquares": reflect.ValueOf(metric.SumSquares), + "SumSquaresBinTol": reflect.ValueOf(metric.SumSquaresBinTol), + "SumSquaresBinTolFunc": reflect.ValueOf(metric.SumSquaresBinTolFunc), + "SumSquaresBinTolScaleOut64": reflect.ValueOf(metric.SumSquaresBinTolScaleOut64), + "SumSquaresFunc": reflect.ValueOf(metric.SumSquaresFunc), + "SumSquaresOut64": reflect.ValueOf(metric.SumSquaresOut64), + "SumSquaresScaleOut64": reflect.ValueOf(metric.SumSquaresScaleOut64), + "TriangularLIndicies": reflect.ValueOf(metric.TriangularLIndicies), + "TriangularN": reflect.ValueOf(metric.TriangularN), + "Vec2in3outFunc": reflect.ValueOf(metric.Vec2in3outFunc), + "Vec2inFunc": reflect.ValueOf(metric.Vec2inFunc), + "Vec3outFunc": reflect.ValueOf(metric.Vec3outFunc), + "VecFunc": reflect.ValueOf(metric.VecFunc), + "VecSSFunc": reflect.ValueOf(metric.VecSSFunc), + "Vectorize3Out64": reflect.ValueOf(metric.Vectorize3Out64), // type definitions "MetricFunc": reflect.ValueOf((*metric.MetricFunc)(nil)), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index 8df9b6bfd9..22fec6eb93 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -10,81 +10,82 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/stats/stats/stats"] = map[string]reflect.Value{ // function, constant and variable definitions - "Count": reflect.ValueOf(stats.Count), - "CountFunc": reflect.ValueOf(stats.CountFunc), - "CountFuncOut64": reflect.ValueOf(stats.CountFuncOut64), - "Describe": reflect.ValueOf(stats.Describe), - "DescribeTable": reflect.ValueOf(stats.DescribeTable), - "DescribeTableAll": reflect.ValueOf(stats.DescribeTableAll), - "DescriptiveStats": reflect.ValueOf(&stats.DescriptiveStats).Elem(), - "GroupAll": reflect.ValueOf(stats.GroupAll), - "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), - "GroupStats": reflect.ValueOf(stats.GroupStats), - "Groups": reflect.ValueOf(stats.Groups), - "L1Norm": reflect.ValueOf(stats.L1Norm), - "L2Norm": reflect.ValueOf(stats.L2Norm), - "L2NormFunc": reflect.ValueOf(stats.L2NormFunc), - "L2NormFuncOut64": reflect.ValueOf(stats.L2NormFuncOut64), - "Max": reflect.ValueOf(stats.Max), - "MaxAbs": reflect.ValueOf(stats.MaxAbs), - "MaxAbsFunc": reflect.ValueOf(stats.MaxAbsFunc), - "MaxFunc": reflect.ValueOf(stats.MaxFunc), - "Mean": reflect.ValueOf(stats.Mean), - "MeanFunc": reflect.ValueOf(stats.MeanFunc), - "MeanFuncOut64": reflect.ValueOf(stats.MeanFuncOut64), - "Median": reflect.ValueOf(stats.Median), - "MedianFunc": reflect.ValueOf(stats.MedianFunc), - "Min": reflect.ValueOf(stats.Min), - "MinAbs": reflect.ValueOf(stats.MinAbs), - "MinAbsFunc": reflect.ValueOf(stats.MinAbsFunc), - "MinFunc": reflect.ValueOf(stats.MinFunc), - "NFunc": reflect.ValueOf(stats.NFunc), - "OutShape": reflect.ValueOf(stats.OutShape), - "Prod": reflect.ValueOf(stats.Prod), - "ProdFunc": reflect.ValueOf(stats.ProdFunc), - "Q1": reflect.ValueOf(stats.Q1), - "Q1Func": reflect.ValueOf(stats.Q1Func), - "Q3": reflect.ValueOf(stats.Q3), - "Q3Func": reflect.ValueOf(stats.Q3Func), - "QuantilesFunc": reflect.ValueOf(stats.QuantilesFunc), - "Sem": reflect.ValueOf(stats.Sem), - "SemFunc": reflect.ValueOf(stats.SemFunc), - "SemPop": reflect.ValueOf(stats.SemPop), - "SemPopFunc": reflect.ValueOf(stats.SemPopFunc), - "Stat": reflect.ValueOf(stats.Stat), - "StatOut": reflect.ValueOf(stats.StatOut), - "StatsN": reflect.ValueOf(stats.StatsN), - "StatsValues": reflect.ValueOf(stats.StatsValues), - "Std": reflect.ValueOf(stats.Std), - "StdFunc": reflect.ValueOf(stats.StdFunc), - "StdFuncOut64": reflect.ValueOf(stats.StdFuncOut64), - "StdPop": reflect.ValueOf(stats.StdPop), - "StdPopFunc": reflect.ValueOf(stats.StdPopFunc), - "StripPackage": reflect.ValueOf(stats.StripPackage), - "Sum": reflect.ValueOf(stats.Sum), - "SumAbs": reflect.ValueOf(stats.SumAbs), - "SumAbsFunc": reflect.ValueOf(stats.SumAbsFunc), - "SumFunc": reflect.ValueOf(stats.SumFunc), - "SumFuncOut64": reflect.ValueOf(stats.SumFuncOut64), - "SumSq": reflect.ValueOf(stats.SumSq), - "SumSqDevFuncOut64": reflect.ValueOf(stats.SumSqDevFuncOut64), - "SumSqFunc": reflect.ValueOf(stats.SumSqFunc), - "SumSqFuncOut64": reflect.ValueOf(stats.SumSqFuncOut64), - "SumSqScaleFuncOut64": reflect.ValueOf(stats.SumSqScaleFuncOut64), - "TableGroupDescribe": reflect.ValueOf(stats.TableGroupDescribe), - "TableGroupStats": reflect.ValueOf(stats.TableGroupStats), - "TableGroups": reflect.ValueOf(stats.TableGroups), - "Var": reflect.ValueOf(stats.Var), - "VarFunc": reflect.ValueOf(stats.VarFunc), - "VarFuncOut64": reflect.ValueOf(stats.VarFuncOut64), - "VarPop": reflect.ValueOf(stats.VarPop), - "VarPopFunc": reflect.ValueOf(stats.VarPopFunc), - "VarPopFuncOut64": reflect.ValueOf(stats.VarPopFuncOut64), - "Vec2inFunc": reflect.ValueOf(stats.Vec2inFunc), - "Vec2outFunc": reflect.ValueOf(stats.Vec2outFunc), - "VecFunc": reflect.ValueOf(stats.VecFunc), - "Vectorize2Out64": reflect.ValueOf(stats.Vectorize2Out64), - "VectorizeOut64": reflect.ValueOf(stats.VectorizeOut64), + "Binarize": reflect.ValueOf(stats.Binarize), + "Clamp": reflect.ValueOf(stats.Clamp), + "Count": reflect.ValueOf(stats.Count), + "CountFunc": reflect.ValueOf(stats.CountFunc), + "CountOut64": reflect.ValueOf(stats.CountOut64), + "Describe": reflect.ValueOf(stats.Describe), + "DescribeTable": reflect.ValueOf(stats.DescribeTable), + "DescribeTableAll": reflect.ValueOf(stats.DescribeTableAll), + "DescriptiveStats": reflect.ValueOf(&stats.DescriptiveStats).Elem(), + "GroupAll": reflect.ValueOf(stats.GroupAll), + "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), + "GroupStats": reflect.ValueOf(stats.GroupStats), + "Groups": reflect.ValueOf(stats.Groups), + "L1Norm": reflect.ValueOf(stats.L1Norm), + "L2Norm": reflect.ValueOf(stats.L2Norm), + "L2NormFunc": reflect.ValueOf(stats.L2NormFunc), + "L2NormOut64": reflect.ValueOf(stats.L2NormOut64), + "Max": reflect.ValueOf(stats.Max), + "MaxAbs": reflect.ValueOf(stats.MaxAbs), + "MaxAbsFunc": reflect.ValueOf(stats.MaxAbsFunc), + "MaxFunc": reflect.ValueOf(stats.MaxFunc), + "Mean": reflect.ValueOf(stats.Mean), + "MeanFunc": reflect.ValueOf(stats.MeanFunc), + "MeanOut64": reflect.ValueOf(stats.MeanOut64), + "Median": reflect.ValueOf(stats.Median), + "MedianFunc": reflect.ValueOf(stats.MedianFunc), + "Min": reflect.ValueOf(stats.Min), + "MinAbs": reflect.ValueOf(stats.MinAbs), + "MinAbsFunc": reflect.ValueOf(stats.MinAbsFunc), + "MinFunc": reflect.ValueOf(stats.MinFunc), + "NFunc": reflect.ValueOf(stats.NFunc), + "Prod": reflect.ValueOf(stats.Prod), + "ProdFunc": reflect.ValueOf(stats.ProdFunc), + "Q1": reflect.ValueOf(stats.Q1), + "Q1Func": reflect.ValueOf(stats.Q1Func), + "Q3": reflect.ValueOf(stats.Q3), + "Q3Func": reflect.ValueOf(stats.Q3Func), + "QuantilesFunc": reflect.ValueOf(stats.QuantilesFunc), + "Sem": reflect.ValueOf(stats.Sem), + "SemFunc": reflect.ValueOf(stats.SemFunc), + "SemPop": reflect.ValueOf(stats.SemPop), + "SemPopFunc": reflect.ValueOf(stats.SemPopFunc), + "StatsN": reflect.ValueOf(stats.StatsN), + "StatsValues": reflect.ValueOf(stats.StatsValues), + "Std": reflect.ValueOf(stats.Std), + "StdFunc": reflect.ValueOf(stats.StdFunc), + "StdOut64": reflect.ValueOf(stats.StdOut64), + "StdPop": reflect.ValueOf(stats.StdPop), + "StdPopFunc": reflect.ValueOf(stats.StdPopFunc), + "StripPackage": reflect.ValueOf(stats.StripPackage), + "Sum": reflect.ValueOf(stats.Sum), + "SumAbs": reflect.ValueOf(stats.SumAbs), + "SumAbsFunc": reflect.ValueOf(stats.SumAbsFunc), + "SumFunc": reflect.ValueOf(stats.SumFunc), + "SumOut64": reflect.ValueOf(stats.SumOut64), + "SumSq": reflect.ValueOf(stats.SumSq), + "SumSqDevOut64": reflect.ValueOf(stats.SumSqDevOut64), + "SumSqFunc": reflect.ValueOf(stats.SumSqFunc), + "SumSqOut64": reflect.ValueOf(stats.SumSqOut64), + "SumSqScaleOut64": reflect.ValueOf(stats.SumSqScaleOut64), + "TableGroupDescribe": reflect.ValueOf(stats.TableGroupDescribe), + "TableGroupStats": reflect.ValueOf(stats.TableGroupStats), + "TableGroups": reflect.ValueOf(stats.TableGroups), + "UnitNorm": reflect.ValueOf(stats.UnitNorm), + "Var": reflect.ValueOf(stats.Var), + "VarFunc": reflect.ValueOf(stats.VarFunc), + "VarOut64": reflect.ValueOf(stats.VarOut64), + "VarPop": reflect.ValueOf(stats.VarPop), + "VarPopFunc": reflect.ValueOf(stats.VarPopFunc), + "VarPopOut64": reflect.ValueOf(stats.VarPopOut64), + "Vec2inFunc": reflect.ValueOf(stats.Vec2inFunc), + "Vec2outFunc": reflect.ValueOf(stats.Vec2outFunc), + "VecFunc": reflect.ValueOf(stats.VecFunc), + "Vectorize2Out64": reflect.ValueOf(stats.Vectorize2Out64), + "VectorizeOut64": reflect.ValueOf(stats.VectorizeOut64), + "ZScore": reflect.ValueOf(stats.ZScore), // type definitions "Stats": reflect.ValueOf((*stats.Stats)(nil)), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-table.go b/yaegicore/symbols/cogentcore_org-core-tensor-table.go index 47c808865c..566f186d42 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-table.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-table.go @@ -16,9 +16,9 @@ func init() { "DetectTableHeaders": reflect.ValueOf(table.DetectTableHeaders), "Headers": reflect.ValueOf(table.Headers), "InferDataType": reflect.ValueOf(table.InferDataType), + "New": reflect.ValueOf(table.New), "NewColumns": reflect.ValueOf(table.NewColumns), "NewSliceTable": reflect.ValueOf(table.NewSliceTable), - "NewTable": reflect.ValueOf(table.NewTable), "NewView": reflect.ValueOf(table.NewView), "NoHeaders": reflect.ValueOf(table.NoHeaders), "ShapeFromString": reflect.ValueOf(table.ShapeFromString), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go index 4513200599..3f442d0e68 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -10,60 +10,46 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/tmath/tmath"] = map[string]reflect.Value{ // function, constant and variable definitions - "Abs": reflect.ValueOf(tmath.Abs), - "Acos": reflect.ValueOf(tmath.Acos), - "Acosh": reflect.ValueOf(tmath.Acosh), - "Add": reflect.ValueOf(tmath.Add), - "AddScalar": reflect.ValueOf(tmath.AddScalar), - "AddSubSpace": reflect.ValueOf(tmath.AddSubSpace), - "Asin": reflect.ValueOf(tmath.Asin), - "Asinh": reflect.ValueOf(tmath.Asinh), - "Atan": reflect.ValueOf(tmath.Atan), - "Atanh": reflect.ValueOf(tmath.Atanh), - "Binarize": reflect.ValueOf(tmath.Binarize), - "Cbrt": reflect.ValueOf(tmath.Cbrt), - "Ceil": reflect.ValueOf(tmath.Ceil), - "Clamp": reflect.ValueOf(tmath.Clamp), - "Cos": reflect.ValueOf(tmath.Cos), - "Cosh": reflect.ValueOf(tmath.Cosh), - "Div": reflect.ValueOf(tmath.Div), - "DivScalar": reflect.ValueOf(tmath.DivScalar), - "DivScalarInv": reflect.ValueOf(tmath.DivScalarInv), - "DivSubSpace": reflect.ValueOf(tmath.DivSubSpace), - "DivSubSpaceInv": reflect.ValueOf(tmath.DivSubSpaceInv), - "Erf": reflect.ValueOf(tmath.Erf), - "Erfc": reflect.ValueOf(tmath.Erfc), - "Erfcinv": reflect.ValueOf(tmath.Erfcinv), - "Erfinv": reflect.ValueOf(tmath.Erfinv), - "Exp": reflect.ValueOf(tmath.Exp), - "Exp2": reflect.ValueOf(tmath.Exp2), - "Expm1": reflect.ValueOf(tmath.Expm1), - "Floor": reflect.ValueOf(tmath.Floor), - "Gamma": reflect.ValueOf(tmath.Gamma), - "J0": reflect.ValueOf(tmath.J0), - "J1": reflect.ValueOf(tmath.J1), - "Log": reflect.ValueOf(tmath.Log), - "Log10": reflect.ValueOf(tmath.Log10), - "Log1p": reflect.ValueOf(tmath.Log1p), - "Log2": reflect.ValueOf(tmath.Log2), - "Logb": reflect.ValueOf(tmath.Logb), - "Mul": reflect.ValueOf(tmath.Mul), - "MulScalar": reflect.ValueOf(tmath.MulScalar), - "MulSubSpace": reflect.ValueOf(tmath.MulSubSpace), - "Round": reflect.ValueOf(tmath.Round), - "RoundToEven": reflect.ValueOf(tmath.RoundToEven), - "Sin": reflect.ValueOf(tmath.Sin), - "Sinh": reflect.ValueOf(tmath.Sinh), - "Sqrt": reflect.ValueOf(tmath.Sqrt), - "Sub": reflect.ValueOf(tmath.Sub), - "SubScalar": reflect.ValueOf(tmath.SubScalar), - "SubSubSpace": reflect.ValueOf(tmath.SubSubSpace), - "Tan": reflect.ValueOf(tmath.Tan), - "Tanh": reflect.ValueOf(tmath.Tanh), - "Trunc": reflect.ValueOf(tmath.Trunc), - "UnitNorm": reflect.ValueOf(tmath.UnitNorm), - "Y0": reflect.ValueOf(tmath.Y0), - "Y1": reflect.ValueOf(tmath.Y1), - "ZScore": reflect.ValueOf(tmath.ZScore), + "Abs": reflect.ValueOf(tmath.Abs), + "Acos": reflect.ValueOf(tmath.Acos), + "Acosh": reflect.ValueOf(tmath.Acosh), + "Add": reflect.ValueOf(tmath.Add), + "Asin": reflect.ValueOf(tmath.Asin), + "Asinh": reflect.ValueOf(tmath.Asinh), + "Atan": reflect.ValueOf(tmath.Atan), + "Atanh": reflect.ValueOf(tmath.Atanh), + "Cbrt": reflect.ValueOf(tmath.Cbrt), + "Ceil": reflect.ValueOf(tmath.Ceil), + "Cos": reflect.ValueOf(tmath.Cos), + "Cosh": reflect.ValueOf(tmath.Cosh), + "Div": reflect.ValueOf(tmath.Div), + "Erf": reflect.ValueOf(tmath.Erf), + "Erfc": reflect.ValueOf(tmath.Erfc), + "Erfcinv": reflect.ValueOf(tmath.Erfcinv), + "Erfinv": reflect.ValueOf(tmath.Erfinv), + "Exp": reflect.ValueOf(tmath.Exp), + "Exp2": reflect.ValueOf(tmath.Exp2), + "Expm1": reflect.ValueOf(tmath.Expm1), + "Floor": reflect.ValueOf(tmath.Floor), + "Gamma": reflect.ValueOf(tmath.Gamma), + "J0": reflect.ValueOf(tmath.J0), + "J1": reflect.ValueOf(tmath.J1), + "Log": reflect.ValueOf(tmath.Log), + "Log10": reflect.ValueOf(tmath.Log10), + "Log1p": reflect.ValueOf(tmath.Log1p), + "Log2": reflect.ValueOf(tmath.Log2), + "Logb": reflect.ValueOf(tmath.Logb), + "Mul": reflect.ValueOf(tmath.Mul), + "Round": reflect.ValueOf(tmath.Round), + "RoundToEven": reflect.ValueOf(tmath.RoundToEven), + "Sin": reflect.ValueOf(tmath.Sin), + "Sinh": reflect.ValueOf(tmath.Sinh), + "Sqrt": reflect.ValueOf(tmath.Sqrt), + "Sub": reflect.ValueOf(tmath.Sub), + "Tan": reflect.ValueOf(tmath.Tan), + "Tanh": reflect.ValueOf(tmath.Tanh), + "Trunc": reflect.ValueOf(tmath.Trunc), + "Y0": reflect.ValueOf(tmath.Y0), + "Y1": reflect.ValueOf(tmath.Y1), } } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 4771e5d312..5049614e28 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -5,247 +5,454 @@ package symbols import ( "cogentcore.org/core/base/metadata" "cogentcore.org/core/tensor" - "gonum.org/v1/gonum/mat" + "go/constant" + "go/token" "reflect" ) func init() { Symbols["cogentcore.org/core/tensor/tensor"] = map[string]reflect.Value{ // function, constant and variable definitions - "AddFunc": reflect.ValueOf(tensor.AddFunc), - "AddShapes": reflect.ValueOf(tensor.AddShapes), - "AsFloat32": reflect.ValueOf(tensor.AsFloat32), - "AsFloat64": reflect.ValueOf(tensor.AsFloat64), - "Ascending": reflect.ValueOf(tensor.Ascending), - "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), - "BoolToInt": reflect.ValueOf(tensor.BoolToInt), - "Calc": reflect.ValueOf(tensor.Calc), - "Call": reflect.ValueOf(tensor.Call), - "CallOut": reflect.ValueOf(tensor.CallOut), - "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), - "CallString": reflect.ValueOf(tensor.CallString), - "ColMajorStrides": reflect.ValueOf(tensor.ColMajorStrides), - "Comma": reflect.ValueOf(tensor.Comma), - "Contains": reflect.ValueOf(tensor.Contains), - "CopyDense": reflect.ValueOf(tensor.CopyDense), - "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), - "DelimsN": reflect.ValueOf(tensor.DelimsN), - "DelimsValues": reflect.ValueOf(tensor.DelimsValues), - "Descending": reflect.ValueOf(tensor.Descending), - "Detect": reflect.ValueOf(tensor.Detect), - "Equals": reflect.ValueOf(tensor.Equals), - "Exclude": reflect.ValueOf(tensor.Exclude), - "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), - "Float64ToString": reflect.ValueOf(tensor.Float64ToString), - "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), - "GetPrecision": reflect.ValueOf(tensor.GetPrecision), - "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), - "Include": reflect.ValueOf(tensor.Include), - "IntToBool": reflect.ValueOf(tensor.IntToBool), - "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), - "NFirstLen": reflect.ValueOf(tensor.NFirstLen), - "NFirstRows": reflect.ValueOf(tensor.NFirstRows), - "NMinLen": reflect.ValueOf(tensor.NMinLen), - "New1DViewOf": reflect.ValueOf(tensor.New1DViewOf), - "NewBits": reflect.ValueOf(tensor.NewBits), - "NewBitsShape": reflect.ValueOf(tensor.NewBitsShape), - "NewByte": reflect.ValueOf(tensor.NewByte), - "NewFloat32": reflect.ValueOf(tensor.NewFloat32), - "NewFloat64": reflect.ValueOf(tensor.NewFloat64), - "NewFloat64FromSlice": reflect.ValueOf(tensor.NewFloat64FromSlice), - "NewFloat64Indexed": reflect.ValueOf(tensor.NewFloat64Indexed), - "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), - "NewFunc": reflect.ValueOf(tensor.NewFunc), - "NewIndexed": reflect.ValueOf(tensor.NewIndexed), - "NewInt": reflect.ValueOf(tensor.NewInt), - "NewInt32": reflect.ValueOf(tensor.NewInt32), - "NewIntFromSlice": reflect.ValueOf(tensor.NewIntFromSlice), - "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), - "NewOfType": reflect.ValueOf(tensor.NewOfType), - "NewShape": reflect.ValueOf(tensor.NewShape), - "NewString": reflect.ValueOf(tensor.NewString), - "NewStringFromSlice": reflect.ValueOf(tensor.NewStringFromSlice), - "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), - "NewStringShape": reflect.ValueOf(tensor.NewStringShape), - "NewStringTensorFromSlice": reflect.ValueOf(tensor.NewStringTensorFromSlice), - "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), - "OddColumn": reflect.ValueOf(tensor.OddColumn), - "OddRow": reflect.ValueOf(tensor.OddRow), - "OpenCSV": reflect.ValueOf(tensor.OpenCSV), - "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), - "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), - "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), - "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), - "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), - "Projection2DString": reflect.ValueOf(tensor.Projection2DString), - "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), - "ReadCSV": reflect.ValueOf(tensor.ReadCSV), - "RowCellSplit": reflect.ValueOf(tensor.RowCellSplit), - "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), - "SaveCSV": reflect.ValueOf(tensor.SaveCSV), - "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), - "SetPrecision": reflect.ValueOf(tensor.SetPrecision), - "Slice": reflect.ValueOf(tensor.Slice), - "SliceSet": reflect.ValueOf(tensor.SliceSet), - "SliceSize": reflect.ValueOf(tensor.SliceSize), - "Space": reflect.ValueOf(tensor.Space), - "Sprint": reflect.ValueOf(tensor.Sprint), - "SprintIndexed": reflect.ValueOf(tensor.SprintIndexed), - "Stable": reflect.ValueOf(tensor.Stable), - "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), - "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), - "Tab": reflect.ValueOf(tensor.Tab), - "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), - "Unstable": reflect.ValueOf(tensor.Unstable), - "UseCase": reflect.ValueOf(tensor.UseCase), - "Vectorize": reflect.ValueOf(tensor.Vectorize), - "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), - "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), - "WriteCSV": reflect.ValueOf(tensor.WriteCSV), + "AddFunc": reflect.ValueOf(tensor.AddFunc), + "AddShapes": reflect.ValueOf(tensor.AddShapes), + "AlignShapes": reflect.ValueOf(tensor.AlignShapes), + "As1D": reflect.ValueOf(tensor.As1D), + "AsFloat32Tensor": reflect.ValueOf(tensor.AsFloat32Tensor), + "AsFloat64Scalar": reflect.ValueOf(tensor.AsFloat64Scalar), + "AsFloat64Slice": reflect.ValueOf(tensor.AsFloat64Slice), + "AsFloat64Tensor": reflect.ValueOf(tensor.AsFloat64Tensor), + "AsIndexed": reflect.ValueOf(tensor.AsIndexed), + "AsIntScalar": reflect.ValueOf(tensor.AsIntScalar), + "AsIntSlice": reflect.ValueOf(tensor.AsIntSlice), + "AsIntTensor": reflect.ValueOf(tensor.AsIntTensor), + "AsMasked": reflect.ValueOf(tensor.AsMasked), + "AsReshaped": reflect.ValueOf(tensor.AsReshaped), + "AsRows": reflect.ValueOf(tensor.AsRows), + "AsSliced": reflect.ValueOf(tensor.AsSliced), + "AsStringScalar": reflect.ValueOf(tensor.AsStringScalar), + "AsStringSlice": reflect.ValueOf(tensor.AsStringSlice), + "AsStringTensor": reflect.ValueOf(tensor.AsStringTensor), + "Ascending": reflect.ValueOf(tensor.Ascending), + "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), + "BoolToInt": reflect.ValueOf(tensor.BoolToInt), + "Calc": reflect.ValueOf(tensor.Calc), + "Call": reflect.ValueOf(tensor.Call), + "CallOut": reflect.ValueOf(tensor.CallOut), + "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), + "CallString": reflect.ValueOf(tensor.CallString), + "Cells1D": reflect.ValueOf(tensor.Cells1D), + "CellsSize": reflect.ValueOf(tensor.CellsSize), + "Clone": reflect.ValueOf(tensor.Clone), + "ColumnMajorStrides": reflect.ValueOf(tensor.ColumnMajorStrides), + "Comma": reflect.ValueOf(tensor.Comma), + "Contains": reflect.ValueOf(tensor.Contains), + "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), + "DelimsN": reflect.ValueOf(tensor.DelimsN), + "DelimsValues": reflect.ValueOf(tensor.DelimsValues), + "Descending": reflect.ValueOf(tensor.Descending), + "Detect": reflect.ValueOf(tensor.Detect), + "Equals": reflect.ValueOf(tensor.Equals), + "Exclude": reflect.ValueOf(tensor.Exclude), + "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), + "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), + "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), + "Include": reflect.ValueOf(tensor.Include), + "IntToBool": reflect.ValueOf(tensor.IntToBool), + "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), + "MustBeValues": reflect.ValueOf(tensor.MustBeValues), + "NFirstLen": reflect.ValueOf(tensor.NFirstLen), + "NFirstRows": reflect.ValueOf(tensor.NFirstRows), + "NMinLen": reflect.ValueOf(tensor.NMinLen), + "NewAxis": reflect.ValueOf(constant.MakeFromLiteral("1", token.INT, 0)), + "NewBool": reflect.ValueOf(tensor.NewBool), + "NewBoolShape": reflect.ValueOf(tensor.NewBoolShape), + "NewByte": reflect.ValueOf(tensor.NewByte), + "NewFloat32": reflect.ValueOf(tensor.NewFloat32), + "NewFloat64": reflect.ValueOf(tensor.NewFloat64), + "NewFloat64FromValues": reflect.ValueOf(tensor.NewFloat64FromValues), + "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), + "NewFunc": reflect.ValueOf(tensor.NewFunc), + "NewIndexed": reflect.ValueOf(tensor.NewIndexed), + "NewInt": reflect.ValueOf(tensor.NewInt), + "NewInt32": reflect.ValueOf(tensor.NewInt32), + "NewIntFromValues": reflect.ValueOf(tensor.NewIntFromValues), + "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), + "NewMasked": reflect.ValueOf(tensor.NewMasked), + "NewOfType": reflect.ValueOf(tensor.NewOfType), + "NewReshaped": reflect.ValueOf(tensor.NewReshaped), + "NewRowCellsView": reflect.ValueOf(tensor.NewRowCellsView), + "NewRows": reflect.ValueOf(tensor.NewRows), + "NewShape": reflect.ValueOf(tensor.NewShape), + "NewSlice": reflect.ValueOf(tensor.NewSlice), + "NewSliceInts": reflect.ValueOf(tensor.NewSliceInts), + "NewSliced": reflect.ValueOf(tensor.NewSliced), + "NewSlicedIndexes": reflect.ValueOf(tensor.NewSlicedIndexes), + "NewString": reflect.ValueOf(tensor.NewString), + "NewStringFromValues": reflect.ValueOf(tensor.NewStringFromValues), + "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), + "NewStringShape": reflect.ValueOf(tensor.NewStringShape), + "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), + "OddColumn": reflect.ValueOf(tensor.OddColumn), + "OddRow": reflect.ValueOf(tensor.OddRow), + "OpenCSV": reflect.ValueOf(tensor.OpenCSV), + "Precision": reflect.ValueOf(tensor.Precision), + "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), + "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), + "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), + "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), + "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), + "Projection2DString": reflect.ValueOf(tensor.Projection2DString), + "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), + "Range": reflect.ValueOf(tensor.Range), + "ReadCSV": reflect.ValueOf(tensor.ReadCSV), + "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), + "SaveCSV": reflect.ValueOf(tensor.SaveCSV), + "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), + "SetPrecision": reflect.ValueOf(tensor.SetPrecision), + "SetShape": reflect.ValueOf(tensor.SetShape), + "SetShapeFrom": reflect.ValueOf(tensor.SetShapeFrom), + "SetShapeFromMustBeValues": reflect.ValueOf(tensor.SetShapeFromMustBeValues), + "SetShapeNames": reflect.ValueOf(tensor.SetShapeNames), + "SetShapeSizesFromTensor": reflect.ValueOf(tensor.SetShapeSizesFromTensor), + "SetShapeSizesMustBeValues": reflect.ValueOf(tensor.SetShapeSizesMustBeValues), + "ShapeNames": reflect.ValueOf(tensor.ShapeNames), + "Space": reflect.ValueOf(tensor.Space), + "Sprint": reflect.ValueOf(tensor.Sprint), + "Stable": reflect.ValueOf(tensor.Stable), + "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), + "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), + "Tab": reflect.ValueOf(tensor.Tab), + "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), + "Unstable": reflect.ValueOf(tensor.Unstable), + "UseCase": reflect.ValueOf(tensor.UseCase), + "Vectorize": reflect.ValueOf(tensor.Vectorize), + "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), + "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), + "WrapIndex1D": reflect.ValueOf(tensor.WrapIndex1D), + "WriteCSV": reflect.ValueOf(tensor.WriteCSV), // type definitions - "Bits": reflect.ValueOf((*tensor.Bits)(nil)), + "Bool": reflect.ValueOf((*tensor.Bool)(nil)), "Delims": reflect.ValueOf((*tensor.Delims)(nil)), "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), "Func": reflect.ValueOf((*tensor.Func)(nil)), "Indexed": reflect.ValueOf((*tensor.Indexed)(nil)), - "Range": reflect.ValueOf((*tensor.Range)(nil)), + "Masked": reflect.ValueOf((*tensor.Masked)(nil)), + "Reshaped": reflect.ValueOf((*tensor.Reshaped)(nil)), + "RowMajor": reflect.ValueOf((*tensor.RowMajor)(nil)), + "Rows": reflect.ValueOf((*tensor.Rows)(nil)), "Shape": reflect.ValueOf((*tensor.Shape)(nil)), + "Slice": reflect.ValueOf((*tensor.Slice)(nil)), + "Sliced": reflect.ValueOf((*tensor.Sliced)(nil)), "String": reflect.ValueOf((*tensor.String)(nil)), "Tensor": reflect.ValueOf((*tensor.Tensor)(nil)), + "Values": reflect.ValueOf((*tensor.Values)(nil)), // interface wrapper definitions - "_Tensor": reflect.ValueOf((*_cogentcore_org_core_tensor_Tensor)(nil)), + "_RowMajor": reflect.ValueOf((*_cogentcore_org_core_tensor_RowMajor)(nil)), + "_Tensor": reflect.ValueOf((*_cogentcore_org_core_tensor_Tensor)(nil)), + "_Values": reflect.ValueOf((*_cogentcore_org_core_tensor_Values)(nil)), } } +// _cogentcore_org_core_tensor_RowMajor is an interface wrapper for RowMajor type +type _cogentcore_org_core_tensor_RowMajor struct { + IValue interface{} + WAsValues func() tensor.Values + WDataType func() reflect.Kind + WDimSize func(dim int) int + WFloat func(i ...int) float64 + WFloat1D func(i int) float64 + WFloatRow func(row int) float64 + WFloatRowCell func(row int, cell int) float64 + WInt func(i ...int) int + WInt1D func(i int) int + WIntRow func(row int) int + WIntRowCell func(row int, cell int) int + WIsString func() bool + WLabel func() string + WLen func() int + WMetadata func() *metadata.Data + WNumDims func() int + WRowTensor func(row int) tensor.Values + WSetFloat func(val float64, i ...int) + WSetFloat1D func(val float64, i int) + WSetFloatRow func(val float64, row int) + WSetFloatRowCell func(val float64, row int, cell int) + WSetInt func(val int, i ...int) + WSetInt1D func(val int, i int) + WSetIntRow func(val int, row int) + WSetIntRowCell func(val int, row int, cell int) + WSetRowTensor func(val tensor.Values, row int) + WSetString func(val string, i ...int) + WSetString1D func(val string, i int) + WSetStringRow func(val string, row int) + WSetStringRowCell func(val string, row int, cell int) + WShape func() *tensor.Shape + WShapeSizes func() []int + WString func() string + WString1D func(i int) string + WStringRow func(row int) string + WStringRowCell func(row int, cell int) string + WStringValue func(i ...int) string + WSubSpace func(offs ...int) tensor.Values +} + +func (W _cogentcore_org_core_tensor_RowMajor) AsValues() tensor.Values { return W.WAsValues() } +func (W _cogentcore_org_core_tensor_RowMajor) DataType() reflect.Kind { return W.WDataType() } +func (W _cogentcore_org_core_tensor_RowMajor) DimSize(dim int) int { return W.WDimSize(dim) } +func (W _cogentcore_org_core_tensor_RowMajor) Float(i ...int) float64 { return W.WFloat(i...) } +func (W _cogentcore_org_core_tensor_RowMajor) Float1D(i int) float64 { return W.WFloat1D(i) } +func (W _cogentcore_org_core_tensor_RowMajor) FloatRow(row int) float64 { return W.WFloatRow(row) } +func (W _cogentcore_org_core_tensor_RowMajor) FloatRowCell(row int, cell int) float64 { + return W.WFloatRowCell(row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) Int(i ...int) int { return W.WInt(i...) } +func (W _cogentcore_org_core_tensor_RowMajor) Int1D(i int) int { return W.WInt1D(i) } +func (W _cogentcore_org_core_tensor_RowMajor) IntRow(row int) int { return W.WIntRow(row) } +func (W _cogentcore_org_core_tensor_RowMajor) IntRowCell(row int, cell int) int { + return W.WIntRowCell(row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) IsString() bool { return W.WIsString() } +func (W _cogentcore_org_core_tensor_RowMajor) Label() string { return W.WLabel() } +func (W _cogentcore_org_core_tensor_RowMajor) Len() int { return W.WLen() } +func (W _cogentcore_org_core_tensor_RowMajor) Metadata() *metadata.Data { return W.WMetadata() } +func (W _cogentcore_org_core_tensor_RowMajor) NumDims() int { return W.WNumDims() } +func (W _cogentcore_org_core_tensor_RowMajor) RowTensor(row int) tensor.Values { + return W.WRowTensor(row) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } +func (W _cogentcore_org_core_tensor_RowMajor) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } +func (W _cogentcore_org_core_tensor_RowMajor) SetFloatRow(val float64, row int) { + W.WSetFloatRow(val, row) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetFloatRowCell(val float64, row int, cell int) { + W.WSetFloatRowCell(val, row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetInt(val int, i ...int) { W.WSetInt(val, i...) } +func (W _cogentcore_org_core_tensor_RowMajor) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } +func (W _cogentcore_org_core_tensor_RowMajor) SetIntRow(val int, row int) { W.WSetIntRow(val, row) } +func (W _cogentcore_org_core_tensor_RowMajor) SetIntRowCell(val int, row int, cell int) { + W.WSetIntRowCell(val, row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetRowTensor(val tensor.Values, row int) { + W.WSetRowTensor(val, row) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetString(val string, i ...int) { + W.WSetString(val, i...) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetString1D(val string, i int) { W.WSetString1D(val, i) } +func (W _cogentcore_org_core_tensor_RowMajor) SetStringRow(val string, row int) { + W.WSetStringRow(val, row) +} +func (W _cogentcore_org_core_tensor_RowMajor) SetStringRowCell(val string, row int, cell int) { + W.WSetStringRowCell(val, row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) Shape() *tensor.Shape { return W.WShape() } +func (W _cogentcore_org_core_tensor_RowMajor) ShapeSizes() []int { return W.WShapeSizes() } +func (W _cogentcore_org_core_tensor_RowMajor) String() string { + if W.WString == nil { + return "" + } + return W.WString() +} +func (W _cogentcore_org_core_tensor_RowMajor) String1D(i int) string { return W.WString1D(i) } +func (W _cogentcore_org_core_tensor_RowMajor) StringRow(row int) string { return W.WStringRow(row) } +func (W _cogentcore_org_core_tensor_RowMajor) StringRowCell(row int, cell int) string { + return W.WStringRowCell(row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) StringValue(i ...int) string { + return W.WStringValue(i...) +} +func (W _cogentcore_org_core_tensor_RowMajor) SubSpace(offs ...int) tensor.Values { + return W.WSubSpace(offs...) +} + // _cogentcore_org_core_tensor_Tensor is an interface wrapper for Tensor type type _cogentcore_org_core_tensor_Tensor struct { + IValue interface{} + WAsValues func() tensor.Values + WDataType func() reflect.Kind + WDimSize func(dim int) int + WFloat func(i ...int) float64 + WFloat1D func(i int) float64 + WInt func(i ...int) int + WInt1D func(i int) int + WIsString func() bool + WLabel func() string + WLen func() int + WMetadata func() *metadata.Data + WNumDims func() int + WSetFloat func(val float64, i ...int) + WSetFloat1D func(val float64, i int) + WSetInt func(val int, i ...int) + WSetInt1D func(val int, i int) + WSetString func(val string, i ...int) + WSetString1D func(val string, i int) + WShape func() *tensor.Shape + WShapeSizes func() []int + WString func() string + WString1D func(i int) string + WStringValue func(i ...int) string +} + +func (W _cogentcore_org_core_tensor_Tensor) AsValues() tensor.Values { return W.WAsValues() } +func (W _cogentcore_org_core_tensor_Tensor) DataType() reflect.Kind { return W.WDataType() } +func (W _cogentcore_org_core_tensor_Tensor) DimSize(dim int) int { return W.WDimSize(dim) } +func (W _cogentcore_org_core_tensor_Tensor) Float(i ...int) float64 { return W.WFloat(i...) } +func (W _cogentcore_org_core_tensor_Tensor) Float1D(i int) float64 { return W.WFloat1D(i) } +func (W _cogentcore_org_core_tensor_Tensor) Int(i ...int) int { return W.WInt(i...) } +func (W _cogentcore_org_core_tensor_Tensor) Int1D(i int) int { return W.WInt1D(i) } +func (W _cogentcore_org_core_tensor_Tensor) IsString() bool { return W.WIsString() } +func (W _cogentcore_org_core_tensor_Tensor) Label() string { return W.WLabel() } +func (W _cogentcore_org_core_tensor_Tensor) Len() int { return W.WLen() } +func (W _cogentcore_org_core_tensor_Tensor) Metadata() *metadata.Data { return W.WMetadata() } +func (W _cogentcore_org_core_tensor_Tensor) NumDims() int { return W.WNumDims() } +func (W _cogentcore_org_core_tensor_Tensor) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } +func (W _cogentcore_org_core_tensor_Tensor) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } +func (W _cogentcore_org_core_tensor_Tensor) SetInt(val int, i ...int) { W.WSetInt(val, i...) } +func (W _cogentcore_org_core_tensor_Tensor) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } +func (W _cogentcore_org_core_tensor_Tensor) SetString(val string, i ...int) { W.WSetString(val, i...) } +func (W _cogentcore_org_core_tensor_Tensor) SetString1D(val string, i int) { W.WSetString1D(val, i) } +func (W _cogentcore_org_core_tensor_Tensor) Shape() *tensor.Shape { return W.WShape() } +func (W _cogentcore_org_core_tensor_Tensor) ShapeSizes() []int { return W.WShapeSizes() } +func (W _cogentcore_org_core_tensor_Tensor) String() string { + if W.WString == nil { + return "" + } + return W.WString() +} +func (W _cogentcore_org_core_tensor_Tensor) String1D(i int) string { return W.WString1D(i) } +func (W _cogentcore_org_core_tensor_Tensor) StringValue(i ...int) string { return W.WStringValue(i...) } + +// _cogentcore_org_core_tensor_Values is an interface wrapper for Values type +type _cogentcore_org_core_tensor_Values struct { IValue interface{} - WAppendFrom func(from tensor.Tensor) error - WAt func(i int, j int) float64 + WAppendFrom func(from tensor.Values) error + WAsValues func() tensor.Values WBytes func() []byte - WClone func() tensor.Tensor - WCopyCellsFrom func(from tensor.Tensor, to int, start int, n int) - WCopyFrom func(from tensor.Tensor) + WClone func() tensor.Values + WCopyCellsFrom func(from tensor.Values, to int, start int, n int) + WCopyFrom func(from tensor.Values) WDataType func() reflect.Kind WDimSize func(dim int) int - WDims func() (r int, c int) WFloat func(i ...int) float64 WFloat1D func(i int) float64 + WFloatRow func(row int) float64 WFloatRowCell func(row int, cell int) float64 WInt func(i ...int) int WInt1D func(i int) int + WIntRow func(row int) int WIntRowCell func(row int, cell int) int WIsString func() bool WLabel func() string WLen func() int WMetadata func() *metadata.Data WNumDims func() int - WRange func() (min float64, max float64, minIndex int, maxIndex int) - WRowCellSize func() (rows int, cells int) - WRowTensor func(row int) tensor.Tensor + WRowTensor func(row int) tensor.Values WSetFloat func(val float64, i ...int) WSetFloat1D func(val float64, i int) + WSetFloatRow func(val float64, row int) WSetFloatRowCell func(val float64, row int, cell int) WSetInt func(val int, i ...int) WSetInt1D func(val int, i int) + WSetIntRow func(val int, row int) WSetIntRowCell func(val int, row int, cell int) - WSetNames func(names ...string) WSetNumRows func(rows int) - WSetRowTensor func(val tensor.Tensor, row int) - WSetShape func(sizes ...int) - WSetShapeFrom func(from tensor.Tensor) + WSetRowTensor func(val tensor.Values, row int) + WSetShapeSizes func(sizes ...int) WSetString func(val string, i ...int) WSetString1D func(val string, i int) + WSetStringRow func(val string, row int) WSetStringRowCell func(val string, row int, cell int) WSetZeros func() WShape func() *tensor.Shape + WShapeSizes func() []int WSizeof func() int64 WString func() string WString1D func(i int) string + WStringRow func(row int) string WStringRowCell func(row int, cell int) string WStringValue func(i ...int) string - WSubSpace func(offs ...int) tensor.Tensor - WT func() mat.Matrix - WView func() tensor.Tensor + WSubSpace func(offs ...int) tensor.Values } -func (W _cogentcore_org_core_tensor_Tensor) AppendFrom(from tensor.Tensor) error { +func (W _cogentcore_org_core_tensor_Values) AppendFrom(from tensor.Values) error { return W.WAppendFrom(from) } -func (W _cogentcore_org_core_tensor_Tensor) At(i int, j int) float64 { return W.WAt(i, j) } -func (W _cogentcore_org_core_tensor_Tensor) Bytes() []byte { return W.WBytes() } -func (W _cogentcore_org_core_tensor_Tensor) Clone() tensor.Tensor { return W.WClone() } -func (W _cogentcore_org_core_tensor_Tensor) CopyCellsFrom(from tensor.Tensor, to int, start int, n int) { +func (W _cogentcore_org_core_tensor_Values) AsValues() tensor.Values { return W.WAsValues() } +func (W _cogentcore_org_core_tensor_Values) Bytes() []byte { return W.WBytes() } +func (W _cogentcore_org_core_tensor_Values) Clone() tensor.Values { return W.WClone() } +func (W _cogentcore_org_core_tensor_Values) CopyCellsFrom(from tensor.Values, to int, start int, n int) { W.WCopyCellsFrom(from, to, start, n) } -func (W _cogentcore_org_core_tensor_Tensor) CopyFrom(from tensor.Tensor) { W.WCopyFrom(from) } -func (W _cogentcore_org_core_tensor_Tensor) DataType() reflect.Kind { return W.WDataType() } -func (W _cogentcore_org_core_tensor_Tensor) DimSize(dim int) int { return W.WDimSize(dim) } -func (W _cogentcore_org_core_tensor_Tensor) Dims() (r int, c int) { return W.WDims() } -func (W _cogentcore_org_core_tensor_Tensor) Float(i ...int) float64 { return W.WFloat(i...) } -func (W _cogentcore_org_core_tensor_Tensor) Float1D(i int) float64 { return W.WFloat1D(i) } -func (W _cogentcore_org_core_tensor_Tensor) FloatRowCell(row int, cell int) float64 { +func (W _cogentcore_org_core_tensor_Values) CopyFrom(from tensor.Values) { W.WCopyFrom(from) } +func (W _cogentcore_org_core_tensor_Values) DataType() reflect.Kind { return W.WDataType() } +func (W _cogentcore_org_core_tensor_Values) DimSize(dim int) int { return W.WDimSize(dim) } +func (W _cogentcore_org_core_tensor_Values) Float(i ...int) float64 { return W.WFloat(i...) } +func (W _cogentcore_org_core_tensor_Values) Float1D(i int) float64 { return W.WFloat1D(i) } +func (W _cogentcore_org_core_tensor_Values) FloatRow(row int) float64 { return W.WFloatRow(row) } +func (W _cogentcore_org_core_tensor_Values) FloatRowCell(row int, cell int) float64 { return W.WFloatRowCell(row, cell) } -func (W _cogentcore_org_core_tensor_Tensor) Int(i ...int) int { return W.WInt(i...) } -func (W _cogentcore_org_core_tensor_Tensor) Int1D(i int) int { return W.WInt1D(i) } -func (W _cogentcore_org_core_tensor_Tensor) IntRowCell(row int, cell int) int { +func (W _cogentcore_org_core_tensor_Values) Int(i ...int) int { return W.WInt(i...) } +func (W _cogentcore_org_core_tensor_Values) Int1D(i int) int { return W.WInt1D(i) } +func (W _cogentcore_org_core_tensor_Values) IntRow(row int) int { return W.WIntRow(row) } +func (W _cogentcore_org_core_tensor_Values) IntRowCell(row int, cell int) int { return W.WIntRowCell(row, cell) } -func (W _cogentcore_org_core_tensor_Tensor) IsString() bool { return W.WIsString() } -func (W _cogentcore_org_core_tensor_Tensor) Label() string { return W.WLabel() } -func (W _cogentcore_org_core_tensor_Tensor) Len() int { return W.WLen() } -func (W _cogentcore_org_core_tensor_Tensor) Metadata() *metadata.Data { return W.WMetadata() } -func (W _cogentcore_org_core_tensor_Tensor) NumDims() int { return W.WNumDims() } -func (W _cogentcore_org_core_tensor_Tensor) Range() (min float64, max float64, minIndex int, maxIndex int) { - return W.WRange() -} -func (W _cogentcore_org_core_tensor_Tensor) RowCellSize() (rows int, cells int) { - return W.WRowCellSize() -} -func (W _cogentcore_org_core_tensor_Tensor) RowTensor(row int) tensor.Tensor { +func (W _cogentcore_org_core_tensor_Values) IsString() bool { return W.WIsString() } +func (W _cogentcore_org_core_tensor_Values) Label() string { return W.WLabel() } +func (W _cogentcore_org_core_tensor_Values) Len() int { return W.WLen() } +func (W _cogentcore_org_core_tensor_Values) Metadata() *metadata.Data { return W.WMetadata() } +func (W _cogentcore_org_core_tensor_Values) NumDims() int { return W.WNumDims() } +func (W _cogentcore_org_core_tensor_Values) RowTensor(row int) tensor.Values { return W.WRowTensor(row) } -func (W _cogentcore_org_core_tensor_Tensor) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } -func (W _cogentcore_org_core_tensor_Tensor) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } -func (W _cogentcore_org_core_tensor_Tensor) SetFloatRowCell(val float64, row int, cell int) { +func (W _cogentcore_org_core_tensor_Values) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } +func (W _cogentcore_org_core_tensor_Values) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } +func (W _cogentcore_org_core_tensor_Values) SetFloatRow(val float64, row int) { + W.WSetFloatRow(val, row) +} +func (W _cogentcore_org_core_tensor_Values) SetFloatRowCell(val float64, row int, cell int) { W.WSetFloatRowCell(val, row, cell) } -func (W _cogentcore_org_core_tensor_Tensor) SetInt(val int, i ...int) { W.WSetInt(val, i...) } -func (W _cogentcore_org_core_tensor_Tensor) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } -func (W _cogentcore_org_core_tensor_Tensor) SetIntRowCell(val int, row int, cell int) { +func (W _cogentcore_org_core_tensor_Values) SetInt(val int, i ...int) { W.WSetInt(val, i...) } +func (W _cogentcore_org_core_tensor_Values) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } +func (W _cogentcore_org_core_tensor_Values) SetIntRow(val int, row int) { W.WSetIntRow(val, row) } +func (W _cogentcore_org_core_tensor_Values) SetIntRowCell(val int, row int, cell int) { W.WSetIntRowCell(val, row, cell) } -func (W _cogentcore_org_core_tensor_Tensor) SetNames(names ...string) { W.WSetNames(names...) } -func (W _cogentcore_org_core_tensor_Tensor) SetNumRows(rows int) { W.WSetNumRows(rows) } -func (W _cogentcore_org_core_tensor_Tensor) SetRowTensor(val tensor.Tensor, row int) { +func (W _cogentcore_org_core_tensor_Values) SetNumRows(rows int) { W.WSetNumRows(rows) } +func (W _cogentcore_org_core_tensor_Values) SetRowTensor(val tensor.Values, row int) { W.WSetRowTensor(val, row) } -func (W _cogentcore_org_core_tensor_Tensor) SetShape(sizes ...int) { W.WSetShape(sizes...) } -func (W _cogentcore_org_core_tensor_Tensor) SetShapeFrom(from tensor.Tensor) { W.WSetShapeFrom(from) } -func (W _cogentcore_org_core_tensor_Tensor) SetString(val string, i ...int) { W.WSetString(val, i...) } -func (W _cogentcore_org_core_tensor_Tensor) SetString1D(val string, i int) { W.WSetString1D(val, i) } -func (W _cogentcore_org_core_tensor_Tensor) SetStringRowCell(val string, row int, cell int) { +func (W _cogentcore_org_core_tensor_Values) SetShapeSizes(sizes ...int) { W.WSetShapeSizes(sizes...) } +func (W _cogentcore_org_core_tensor_Values) SetString(val string, i ...int) { W.WSetString(val, i...) } +func (W _cogentcore_org_core_tensor_Values) SetString1D(val string, i int) { W.WSetString1D(val, i) } +func (W _cogentcore_org_core_tensor_Values) SetStringRow(val string, row int) { + W.WSetStringRow(val, row) +} +func (W _cogentcore_org_core_tensor_Values) SetStringRowCell(val string, row int, cell int) { W.WSetStringRowCell(val, row, cell) } -func (W _cogentcore_org_core_tensor_Tensor) SetZeros() { W.WSetZeros() } -func (W _cogentcore_org_core_tensor_Tensor) Shape() *tensor.Shape { return W.WShape() } -func (W _cogentcore_org_core_tensor_Tensor) Sizeof() int64 { return W.WSizeof() } -func (W _cogentcore_org_core_tensor_Tensor) String() string { +func (W _cogentcore_org_core_tensor_Values) SetZeros() { W.WSetZeros() } +func (W _cogentcore_org_core_tensor_Values) Shape() *tensor.Shape { return W.WShape() } +func (W _cogentcore_org_core_tensor_Values) ShapeSizes() []int { return W.WShapeSizes() } +func (W _cogentcore_org_core_tensor_Values) Sizeof() int64 { return W.WSizeof() } +func (W _cogentcore_org_core_tensor_Values) String() string { if W.WString == nil { return "" } return W.WString() } -func (W _cogentcore_org_core_tensor_Tensor) String1D(i int) string { return W.WString1D(i) } -func (W _cogentcore_org_core_tensor_Tensor) StringRowCell(row int, cell int) string { +func (W _cogentcore_org_core_tensor_Values) String1D(i int) string { return W.WString1D(i) } +func (W _cogentcore_org_core_tensor_Values) StringRow(row int) string { return W.WStringRow(row) } +func (W _cogentcore_org_core_tensor_Values) StringRowCell(row int, cell int) string { return W.WStringRowCell(row, cell) } -func (W _cogentcore_org_core_tensor_Tensor) StringValue(i ...int) string { return W.WStringValue(i...) } -func (W _cogentcore_org_core_tensor_Tensor) SubSpace(offs ...int) tensor.Tensor { +func (W _cogentcore_org_core_tensor_Values) StringValue(i ...int) string { return W.WStringValue(i...) } +func (W _cogentcore_org_core_tensor_Values) SubSpace(offs ...int) tensor.Values { return W.WSubSpace(offs...) } -func (W _cogentcore_org_core_tensor_Tensor) T() mat.Matrix { return W.WT() } -func (W _cogentcore_org_core_tensor_Tensor) View() tensor.Tensor { return W.WView() } From d89abc26c4820b17b844ab0ca9e48d989b79a8b9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 12:08:41 -0700 Subject: [PATCH 106/311] SubSpace properly deals with 1D tensors (cells are a scalar), and docs, api cleanup for stats. --- tensor/README.md | 2 +- tensor/base.go | 10 ++++-- tensor/cmd/tablecat/tablecat.go | 2 +- tensor/examples/datafs-sim/sim.go | 4 +-- tensor/reshaped.go | 2 +- tensor/stats/metric/metric_test.go | 15 ++++++--- tensor/stats/stats/README.md | 30 +++++++++++------- tensor/stats/stats/describe.go | 2 +- tensor/stats/stats/group.go | 51 +++++++++++++++++------------- tensor/stats/stats/group_test.go | 6 ++-- 10 files changed, 76 insertions(+), 48 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index 717f84857b..2e412e8d7e 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -56,7 +56,7 @@ All tensor package functions are registered using a global name-to-function map In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: -* **Flat, 1D**: this is the simplest data shape, and any tensor can be turned into a flat 1D list using the `New1DView` function, which returns a `Reshaped` view. The [stats](stats) functions for example report summary statistics across the outermost row dimension, so converting data to this 1D view gives stats across all the data. +* **Flat, 1D**: this is the simplest data shape, and any tensor can be turned into a flat 1D list using the `As1D` function, which either returns the tensor itself it is already 1D, or a `Reshaped` 1D view. The [stats](stats) functions for example report summary statistics across the outermost row dimension, so converting data to this 1D view gives stats across all the data. * **Row, Cell 2D**: This is the natural shape for tabular data, and the `RowMajor` type and `Rows` view provide methods for efficiently accessing data in this way. In addition, the [stats](stats) and [metric](metric) packages automatically compute statistics across the outermost row dimension, aggregating results across rows for each cell. Thus, you end up with the "average cell-wise pattern" when you do `stats.Mean` for example. The `NewRowCellsView` function returns a `Reshaped` view of any tensor organized into this 2D shape, with the row vs. cell split specified at any point in the list of dimensions, which can be useful in obtaining the desired results. diff --git a/tensor/base.go b/tensor/base.go index 88ddf2856b..15f1fda04a 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -98,11 +98,17 @@ func (tsr *Base[T]) SetNumRows(rows int) { func (tsr *Base[T]) subSpaceImpl(offs ...int) *Base[T] { nd := tsr.NumDims() od := len(offs) - if od >= nd { + if od > nd { return nil } + var ssz []int + if od == nd { // scalar subspace + ssz = []int{1} + } else { + ssz = tsr.shape.Sizes[od:] + } stsr := &Base[T]{} - stsr.SetShapeSizes(tsr.shape.Sizes[od:]...) + stsr.SetShapeSizes(ssz...) sti := make([]int, nd) copy(sti, offs) stoff := tsr.shape.IndexTo1D(sti...) diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index 2e6b214c23..710eef58ba 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -159,7 +159,7 @@ func AvgByColumn(files []string, column string) { } cols = append(cols, dt.Columns.Keys[ci]) } - stats.TableGroupStats(dir, stats.Mean.FuncName(), dt, cols...) + stats.TableGroupStats(dir, stats.Mean, dt, cols...) std := dir.Item("Stats") avgdt := std.GetDirTable(nil) // todo: has stat name slash tensor.SetPrecision(avgdt.Meta, LogPrec) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index c490497c66..1ef817f11e 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -157,8 +157,8 @@ func (ss *Sim) Run() { dir, _ := ss.Logs.Mkdir("Stats") stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") sts := []string{"SSE", "AvgSSE", "TrlErr"} - stats.TableGroupStats(dir, stats.Mean.FuncName(), alldt, sts...) - stats.TableGroupStats(dir, stats.Sem.FuncName(), alldt, sts...) + stats.TableGroupStats(dir, stats.Mean, alldt, sts...) + stats.TableGroupStats(dir, stats.Sem, alldt, sts...) } diff --git a/tensor/reshaped.go b/tensor/reshaped.go index 0e57601863..38e40d69b6 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -48,7 +48,7 @@ func NewReshaped(tsr Tensor, sizes ...int) *Reshaped { // and the remainder are collapsed to form the 1D cells dimension. // This is useful for stats, metrics and other packages that operate // on data in this shape. -func NewRowCellsView(tsr Values, split int) *Reshaped { +func NewRowCellsView(tsr Tensor, split int) *Reshaped { sizes := tsr.ShapeSizes() rows := sizes[:split] cells := sizes[split:] diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 727b890fd3..1d5a3443ae 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -90,7 +90,8 @@ func TestMatrix(t *testing.T) { assert.NoError(t, err) in := dt.Column("Input") out := tensor.NewFloat64() - Matrix(Euclidean.FuncName(), in, out) + err = Matrix(Euclidean.FuncName(), in, out) + assert.NoError(t, err) // fmt.Println(out.Tensor) assert.Equal(t, simres, out.String()) } @@ -108,7 +109,8 @@ func TestPCAIris(t *testing.T) { } data := dt.Column("data") covar := tensor.NewFloat64() - CovarianceMatrix(Correlation.FuncName(), data, covar) + err = CovarianceMatrix(Correlation.FuncName(), data, covar) + assert.NoError(t, err) // fmt.Printf("correl: %s\n", covar.String()) vecs := tensor.NewFloat64() @@ -125,7 +127,8 @@ func TestPCAIris(t *testing.T) { colidx := tensor.NewFloat64Scalar(3) // strongest at end prjns := tensor.NewFloat64() - ProjectOnMatrixColumn(vecs, data, colidx, prjns) + err = ProjectOnMatrixColumn(vecs, data, colidx, prjns) + assert.NoError(t, err) // tensor.SaveCSV(prjns, "testdata/pca_projection.csv", tensor.Comma) trgprjns := []float64{ 2.6692308782935146, @@ -146,7 +149,8 @@ func TestPCAIris(t *testing.T) { //////////////////////////////////////////////////////////// // SVD - SVD(covar, vecs, vals) + err = SVD(covar, vecs, vals) + assert.NoError(t, err) // fmt.Printf("correl vec: %v\n", vecs) // fmt.Printf("correl val: %v\n", vals) for i, v := range vals.Values { @@ -154,7 +158,8 @@ func TestPCAIris(t *testing.T) { } colidx.SetFloat1D(0, 0) // strongest at start - ProjectOnMatrixColumn(vecs, data, colidx, prjns) + err = ProjectOnMatrixColumn(vecs, data, colidx, prjns) + assert.NoError(t, err) // tensor.SaveCSV(prjns, "testdata/svd_projection.csv", tensor.Comma) trgprjns = []float64{ -2.6692308782935172, diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index 8086b4bf40..41f86b08ab 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -2,16 +2,26 @@ The `stats` package provides standard statistic computations operating on the `tensor.Tensor` standard data representation, using this standard function: ```Go -type StatsFunc func(in, out tensor.Tensor) +type StatsFunc func(in, out tensor.Tensor) error ``` -For 1D data, the output is a scalar value in the out tensor, and otherwise it is an n-dimensional "cell" with outermost row dimension set to 1. This is +The stats functions always operate on the outermost _row_ dimension, and it is up to the caller to reshape the tensor to accomplish the desired results. -All stats are registered in the `tensor.Funcs` global list, and can be called using the `FuncName` method, e.g.,: +* To obtain a single summary statistic across all values, use `tensor.As1D` + +* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the statistic for each such cell computed across the outer rows dimension (e.g., the "average" cell pattern for the `Mean` statistic). + +* Use `tensor.NewRowCellsView` to reshape any tensor into a 2D rows x cells shape, with the cells starting at a given dimension. Thus, any number of outer dimensions can be collapsed into the outer row dimension, and the remaining dimensions become the cells. + +By contrast, the [NumPy Statistics](https://numpy.org/doc/stable/reference/generated/numpy.mean.html#numpy.mean) functions take an `axis` dimension to compute over, but passing such arguments via the universal function calling api for tensors introduces complications, so it is simpler to just have a single designated behavior and reshape the data to achieve the desired results. + +All stats are registered in the `tensor.Funcs` global list (for use in Goal), and can be called through the `Stats` enum: ```Go -tensor.Call(Mean.FuncName(), in, out) +stats.Mean.Call(in, out) ``` +All stats functions (and all tensor functions more generally) skip over NaN's as a missing value, so they are equivalent to the `nanmean` etc versions in NumPy. + ## Stats The following statistics are supported (per the `Stats` enum in `stats.go`): @@ -40,15 +50,13 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): Here is the general info associated with these function calls: -`StatsFunc` is the function signature for a stats function, where the output has the same shape as the input but with the outermost row dimension size of 1, and contains the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. - -Critically, the stat is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ stat for that cell across the different rows. To compute a stat on the `tensor.SubSpace` cells themselves, must call on a `tensor.New1DViewOf` the `RowTensor`. - -All stats functions skip over NaN's, as a missing value. +`StatsFunc` is the function signature for a stats function, where the output must be a `tensor.Values` tensor, and is automatically shaped to hold the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. Stats functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. -## norm functions +## Normalization functions + +The stats package also has the following standard normalization functions for transforming data into standard ranges in various ways: * `UnitNorm` subtracts `min` and divides by resulting `max` to normalize to 0..1 unit range. * `ZScore` subtracts the mean and divides by the standard deviation. @@ -95,5 +103,5 @@ See the [examples/planets](../examples/planets) example for an interactive explo ## Vectorize functions -See [vecfuncs.go](vecfuncs.go) for corresponding `tensor.Vectorize` functions that are used in performing the computations. These cannot be parallelized directly due to shared writing to output accumulators, and other ordering constraints. If needed, special atomic-locking or other such techniques would be required. +See [vec.go](vec.go) for corresponding `tensor.Vectorize` functions that are used in performing the computations. These cannot be parallelized directly due to shared writing to output accumulators, and other ordering constraints. If needed, special atomic-locking or other such techniques would be required. diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go index 83175da21d..c6be23e990 100644 --- a/tensor/stats/stats/describe.go +++ b/tensor/stats/stats/describe.go @@ -41,7 +41,7 @@ func Describe(dir *datafs.Data, tsrs ...tensor.Tensor) { for _, st := range DescriptiveStats { stnm := st.String() sv := datafs.NewValue[float64](td, stnm, 1) - tensor.Call(st.FuncName(), tsr, sv) + st.Call(tsr, sv) } } } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 9607c4dc28..4e3752470e 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" @@ -31,10 +30,10 @@ import ( // rows, indirected through any existing indexes on the inputs, so that // the results can be used directly as Indexes into the corresponding tensor data. // Uses a stable sort on columns, so ordering of other dimensions is preserved. -func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) { +func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) error { gd, err := dir.RecycleDir("Groups") - if errors.Log(err) != nil { - return + if err != nil { + return err } makeIdxs := func(dir *datafs.Data, srt *tensor.Rows, val string, start, r int) { n := r - start @@ -85,35 +84,37 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) { } } } + return nil } // TableGroups runs [Groups] on the given columns from given [table.Table]. -func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) { +func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) error { dv := table.NewView(dt) // important for consistency across columns, to do full outer product sort first. dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) - Groups(dir, dv.ColumnList(columns...)...) + return Groups(dir, dv.ColumnList(columns...)...) } // GroupAll copies all indexes from the first given tensor, // into an "All/All" tensor in the given [datafs], which can then // be used with [GroupStats] to generate summary statistics across // all the data. See [Groups] for more general documentation. -func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) { +func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { gd, err := dir.RecycleDir("Groups") - if errors.Log(err) != nil { - return + if err != nil { + return err } tsr := tensor.AsRows(tsrs[0]) nr := tsr.NumRows() if nr == 0 { - return + return nil } td, _ := gd.Mkdir("All") it := datafs.NewValue[int](td, "All", nr) for j := range nr { it.SetIntRow(tsr.RowIndex(j), j) // key to indirect through any existing indexes } + return nil } // todo: GroupCombined @@ -128,14 +129,14 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) { // a String tensor with the unique values of each source [Groups] tensor, // and a aligned Float64 tensor with the statistics results for each such // unique group value. See the README.md file for a diagram of the results. -func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) { +func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) error { gd, err := dir.RecycleDir("Groups") - if errors.Log(err) != nil { - return + if err != nil { + return err } sd, err := dir.RecycleDir("Stats") - if errors.Log(err) != nil { - return + if err != nil { + return err } stnm := StripPackage(stat) spl := strings.Split(stat, ".") @@ -171,22 +172,28 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) { } } } + return nil } -// TableGroupStats runs [GroupStats] on the given columns from given [table.Table]. -func TableGroupStats(dir *datafs.Data, stat string, dt *table.Table, columns ...string) { - GroupStats(dir, stat, dt.ColumnList(columns...)...) +// TableGroupStats runs [GroupStats] using standard [Stats] +// on the given columns from given [table.Table]. +func TableGroupStats(dir *datafs.Data, stat Stats, dt *table.Table, columns ...string) error { + return GroupStats(dir, stat.FuncName(), dt.ColumnList(columns...)...) } // GroupDescribe runs standard descriptive statistics on given tensor data // using [GroupStats] function, with [DescriptiveStats] list of stats. -func GroupDescribe(dir *datafs.Data, tsrs ...tensor.Tensor) { +func GroupDescribe(dir *datafs.Data, tsrs ...tensor.Tensor) error { for _, st := range DescriptiveStats { - GroupStats(dir, st.FuncName(), tsrs...) + err := GroupStats(dir, st.FuncName(), tsrs...) + if err != nil { + return err + } } + return nil } // TableGroupDescribe runs [GroupDescribe] on the given columns from given [table.Table]. -func TableGroupDescribe(dir *datafs.Data, dt *table.Table, columns ...string) { - GroupDescribe(dir, dt.ColumnList(columns...)...) +func TableGroupDescribe(dir *datafs.Data, dt *table.Table, columns ...string) error { + return GroupDescribe(dir, dt.ColumnList(columns...)...) } diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index 01adfb0fd1..7dcf922e3b 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -26,13 +26,15 @@ func TestGroup(t *testing.T) { dt.Column("Value").SetFloatRow(float64(i), i) } dir, _ := datafs.NewDir("Group") - TableGroups(dir, dt, "Name") + err := TableGroups(dir, dt, "Name") + assert.NoError(t, err) ixs := dir.FlatValuesFunc(nil) assert.Equal(t, []int{0, 1}, tensor.AsIntTensor(ixs[0]).Values) assert.Equal(t, []int{2, 3}, tensor.AsIntTensor(ixs[1]).Values) - TableGroupStats(dir, Mean.FuncName(), dt, "Value") + err = TableGroupStats(dir, Mean, dt, "Value") + assert.NoError(t, err) // AggColumn(spl, "Value", stats.Mean) // st := spl.AggsToTable(table.ColumnNameOnly) From 579b53adccdb76c7e8b7618aaa34a9b9422b9286 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 13:05:51 -0700 Subject: [PATCH 107/311] matrix readme update, and error checking for same tensor shape --- tensor/convert.go | 8 ++++++++ tensor/stats/metric/README.md | 21 +++++++++++++------- tensor/stats/metric/funcs.go | 37 ++++++++++++++++++++++++++--------- tensor/stats/metric/vec.go | 3 +++ tensor/stats/stats/README.md | 10 +++++----- tensor/stats/stats/funcs.go | 12 ++++-------- 6 files changed, 62 insertions(+), 29 deletions(-) diff --git a/tensor/convert.go b/tensor/convert.go index 353498d63c..368b0a7b1b 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -31,6 +31,14 @@ func MustBeValues(tsr Tensor) (Values, error) { return vl, nil } +// MustBeSameShape returns an error if the two tensors do not have the same shape. +func MustBeSameShape(a, b Tensor) error { + if !a.Shape().IsEqual(b.Shape()) { + return errors.New("tensor.MustBeSameShape: tensors must have the same shape") + } + return nil +} + // SetShape sets the dimension sizes from given Shape func SetShape(vals Values, sh *Shape) { vals.SetShapeSizes(sh.Sizes...) diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 0877f8b8bf..8f0f3133ca 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -1,8 +1,17 @@ # metric -`metric` provides various similarity / distance metrics for comparing tensors, operating on the `tensor.Tensor` standard data representation. +`metric` provides various similarity / distance metrics for comparing two tensors, operating on the `tensor.Tensor` standard data representation, using this standard function: +```Go +type MetricFunc func(a, b, out tensor.Tensor) error +``` -The `Matrix` function returns a distance / similarity matrix computed from the n-dimensional "cells" of row-organized tensor data, and the `LabeledMatrix` type provides labels for displaying such matricies. +The metric functions always operate on the outermost _row_ dimension, and it is up to the caller to reshape the tensors to accomplish the desired results. The two tensors must have the same shape. + +* To obtain a single summary metric across all values, use `tensor.As1D`. + +* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the metric for each such cell computed across the outer rows dimension. For the `Euclidean` metric for example, each cell has the difference for that cell value across all the rows between the two tensors. See [Matrix functions](#matrix-functions) below for a function that computes the distances _between each cell pattern and all the others_, as a distance or similarity matrix. + +* Use `tensor.NewRowCellsView` to reshape any tensor into a 2D rows x cells shape, with the cells starting at a given dimension. Thus, any number of outer dimensions can be collapsed into the outer row dimension, and the remaining dimensions become the cells. ## Metrics @@ -27,15 +36,13 @@ The `Matrix` function returns a distance / similarity matrix computed from the n Here is general info about these functions: -`MetricFunc` is the function signature for a metric function, where the output has the same shape as the inputs but with the outermost row dimension size of 1, and contains the metric value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. - -Critically, the metric is always computed over the outer row dimension, so each cell in a higher-dimensional output reflects the _row-wise_ metric for that cell across the different rows. To compute a metric on the `tensor.SubSpace` cells themselves, must call on a `tensor.New1DViewOf` the sub space. See [simat](../simat) package. +The output must be a `tensor.Values` tensor, and it is automatically shaped to hold the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. -All metric functions skip over NaN's, as a missing value, and use the min of the length of the two tensors. +All metric functions skip over NaN's, as a missing value. Metric functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. -## Matrix functions +# Matrix functions * `Matrix` computes a distance / similarity matrix using a metric function, operating on the n-dimensional sub-space patterns on a given tensor (i.e., a row-wise list of patterns). The result is a square rows x rows matrix where each cell is the metric value for the pattern at the given row. The diagonal contains the self-similarity metric. diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 4f504dbc21..8d27511ae5 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -13,15 +13,10 @@ import ( ) // MetricFunc is the function signature for a metric function, -// where the output has the same shape as the inputs but with -// the outermost row dimension size of 1, and contains -// the metric value(s) for the "cells" in higher-dimensional tensors, -// and a single scalar value for a 1D input tensor. -// Critically, the metric is always computed over the outer row dimension, -// so each cell in a higher-dimensional output reflects the _row-wise_ -// metric for that cell across the different rows. To compute a metric -// on the [tensor.SubSpace] cells themselves, must call on a -// [tensor.New1DViewOf] the sub space. See [simat] package. +// which is computed over the outermost row dimension and the +// output is the shape of the remaining inner cells (a scalar for 1D inputs). +// Use [tensor.As1D], [tensor.NewRowCellsView], [tensor.Cells1D] etc +// to reshape and reslice the data as needed. // All metric functions skip over NaN's, as a missing value, // and use the min of the length of the two tensors. // Metric functions cannot be computed in parallel, @@ -33,6 +28,9 @@ type MetricFunc func(a, b, out tensor.Tensor) error // SumSquaresScaleOut64 computes the sum of squares differences between tensor values, // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. func SumSquaresScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { + if err = tensor.MustBeSameShape(a, b); err != nil { + return + } scale64, ss64, err = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { return a - b @@ -98,6 +96,9 @@ func EuclideanFunc(a, b, out tensor.Tensor) error { // tensor values, aka the L1 Norm. // See [MetricFunc] for general information. func AbsFunc(a, b, out tensor.Tensor) error { + if err := tensor.MustBeSameShape(a, b); err != nil { + return err + } _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + math.Abs(a-b) @@ -110,6 +111,9 @@ func AbsFunc(a, b, out tensor.Tensor) error { // i.e., "city block" distance. // See [MetricFunc] for general information. func HammingFunc(a, b, out tensor.Tensor) error { + if err := tensor.MustBeSameShape(a, b); err != nil { + return err + } _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { if a != b { @@ -125,6 +129,9 @@ func HammingFunc(a, b, out tensor.Tensor) error { // with binary tolerance: differences < 0.5 are thresholded to 0. // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. func SumSquaresBinTolScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { + if err = tensor.MustBeSameShape(a, b); err != nil { + return + } scale64, ss64, err = stats.Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecSSFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], 0, 1, func(a, b float64) float64 { d := a - b @@ -192,6 +199,9 @@ func SumSquaresBinTolFunc(a, b, out tensor.Tensor) error { // a * log(a/b) + (1-a) * log(1-a/1-b). // See [MetricFunc] for general information. func CrossEntropyFunc(a, b, out tensor.Tensor) error { + if err := tensor.MustBeSameShape(a, b); err != nil { + return err + } _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { b = math32.Clamp(b, 0.000001, 0.999999) @@ -211,6 +221,9 @@ func CrossEntropyFunc(a, b, out tensor.Tensor) error { // InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. // See [MetricFunc] for general information. func InnerProductFunc(a, b, out tensor.Tensor) error { + if err := tensor.MustBeSameShape(a, b); err != nil { + return err + } _, err := stats.VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], tsr[2], 0, func(a, b, agg float64) float64 { return agg + a*b @@ -223,6 +236,9 @@ func InnerProductFunc(a, b, out tensor.Tensor) error { // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. func CovarianceFunc(a, b, out tensor.Tensor) error { + if err := tensor.MustBeSameShape(a, b); err != nil { + return err + } amean, acount, err := stats.MeanOut64(a, out) if err != nil { return err @@ -253,6 +269,9 @@ func CovarianceFunc(a, b, out tensor.Tensor) error { // Equivalent to the cosine of mean-normalized vectors. // Returns the Float64 output values for subsequent use. func CorrelationOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { + if err := tensor.MustBeSameShape(a, b); err != nil { + return nil, err + } amean, _, err := stats.MeanOut64(a, out) if err != nil { return nil, err diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 9ac113f18f..004cca9bf6 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -19,6 +19,9 @@ func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr if n <= 0 { return nil, nil, nil, nil } + if err = tensor.MustBeSameShape(tsr[0], tsr[1]); err != nil { + return + } nt := len(tsr) osz := tensor.CellsSize(tsr[0].ShapeSizes()) out := tsr[nt-1] diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index 41f86b08ab..bcd092b1a8 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -4,18 +4,18 @@ The `stats` package provides standard statistic computations operating on the `t ```Go type StatsFunc func(in, out tensor.Tensor) error ``` - +n The stats functions always operate on the outermost _row_ dimension, and it is up to the caller to reshape the tensor to accomplish the desired results. -* To obtain a single summary statistic across all values, use `tensor.As1D` +* To obtain a single summary statistic across all values, use `tensor.As1D`. -* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the statistic for each such cell computed across the outer rows dimension (e.g., the "average" cell pattern for the `Mean` statistic). +* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the statistic for each such cell computed across the outer rows dimension. For the `Mean` statistic for example, each cell contains the average of that cell across all the rows. * Use `tensor.NewRowCellsView` to reshape any tensor into a 2D rows x cells shape, with the cells starting at a given dimension. Thus, any number of outer dimensions can be collapsed into the outer row dimension, and the remaining dimensions become the cells. By contrast, the [NumPy Statistics](https://numpy.org/doc/stable/reference/generated/numpy.mean.html#numpy.mean) functions take an `axis` dimension to compute over, but passing such arguments via the universal function calling api for tensors introduces complications, so it is simpler to just have a single designated behavior and reshape the data to achieve the desired results. -All stats are registered in the `tensor.Funcs` global list (for use in Goal), and can be called through the `Stats` enum: +All stats are registered in the `tensor.Funcs` global list (for use in Goal), and can be called through the `Stats` enum e.g.: ```Go stats.Mean.Call(in, out) ``` @@ -50,7 +50,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): Here is the general info associated with these function calls: -`StatsFunc` is the function signature for a stats function, where the output must be a `tensor.Values` tensor, and is automatically shaped to hold the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. +The output must be a `tensor.Values` tensor, and it is automatically shaped to hold the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. Stats functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 6716f3c220..9b4c8a00fd 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -11,14 +11,10 @@ import ( ) // StatsFunc is the function signature for a stats function, -// where the output has the same shape as the input but with -// the outermost row dimension size of 1, and contains -// the stat value(s) for the "cells" in higher-dimensional tensors, -// and a single scalar value for a 1D input tensor. -// Critically, the stat is always computed over the outer row dimension, -// so each cell in a higher-dimensional output reflects the _row-wise_ -// stat for that cell across the different rows. Use [tensor.NewRowCellsView], -// [tensor.Cells1D], and [tensor.As1D] to reshape and reslice the data as needed. +// which is computed over the outermost row dimension and the +// output is the shape of the remaining inner cells (a scalar for 1D inputs). +// Use [tensor.As1D], [tensor.NewRowCellsView], [tensor.Cells1D] etc +// to reshape and reslice the data as needed. // All stats functions skip over NaN's, as a missing value. // Stats functions cannot be computed in parallel, // e.g., using VectorizeThreaded or GPU, due to shared writing From 213043a867fc061aa4d63c1bb0c8bf3b15bd71cc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 13:17:25 -0700 Subject: [PATCH 108/311] add goal/testdata --- goal/testdata/test.goal | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 goal/testdata/test.goal diff --git a/goal/testdata/test.goal b/goal/testdata/test.goal new file mode 100644 index 0000000000..fdd2a23b3e --- /dev/null +++ b/goal/testdata/test.goal @@ -0,0 +1,11 @@ + +// # x := [3, 5, 4] +# x := zeros(3, 4) +# nd := x.ndim +# sz := x.size +# sh := x.shape + +fmt.Println(x) +fmt.Println(nd) +fmt.Println(sz) +fmt.Println(sh) From 21155c69d8232a6374513102170f2b3011c88437 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 15:24:13 -0700 Subject: [PATCH 109/311] switched stats and metric func / enum so the enum has the prefix and the func is plain name. --- tensor/funcs.go | 262 ++++++++++-------- tensor/funcs_test.go | 10 +- tensor/stats/metric/funcs.go | 54 ++-- tensor/stats/metric/matrix.go | 43 ++- tensor/stats/metric/metric_test.go | 58 ++-- tensor/stats/metric/metrics.go | 77 +++-- tensor/stats/metric/misc.go | 12 +- tensor/stats/stats/describe.go | 2 +- tensor/stats/stats/funcs.go | 70 ++--- tensor/stats/stats/group.go | 11 + tensor/stats/stats/group_test.go | 2 +- tensor/stats/stats/norm.go | 4 +- tensor/stats/stats/quantiles.go | 22 +- tensor/stats/stats/stats.go | 109 +++++--- tensor/stats/stats/stats_test.go | 176 ++++++------ tensor/tmath/math_test.go | 10 - tensor/tmath/ops_test.go | 11 + ...cogentcore_org-core-tensor-stats-metric.go | 27 +- .../cogentcore_org-core-tensor-stats-stats.go | 45 +-- .../symbols/cogentcore_org-core-tensor.go | 6 +- 20 files changed, 559 insertions(+), 452 deletions(-) diff --git a/tensor/funcs.go b/tensor/funcs.go index 359e0aa6c8..b029197b70 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -7,25 +7,27 @@ package tensor import ( "fmt" "reflect" + "slices" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" ) -// StringFirstArg should be used to set StringFirst functions -const StringFirstArg = true +// AnyFirstArg should be used to set AnyFirst functions +const AnyFirstArg = true // Func represents a registered tensor function, which has // In number of input Tensor arguments, and Out number of output -// arguments (typically 1). There can also be an optional -// string first argument, which is used to specify the name of -// another function in some cases (e.g., a stat or metric function). +// arguments (typically 1). There can also be an 'any' first +// argument to support other kinds of parameters. +// This is used to make tensor functions available to the Goal +// language. type Func struct { // Name is the original CamelCase Go name for function Name string // Fun is the function, which must _only_ take some number of Tensor - // args, with an optional first string arg per [StringFirst]. + // args, with an optional any first arg. Fun any // In is number of input args @@ -34,21 +36,25 @@ type Func struct { // Out is number of output args Out int - // StringFirst indicates if there is a string first argument, which can be - // used for many purposes including passing the name of another function to use - // in computation. - StringFirst bool + // AnyFirst indicates if there is an 'any' first argument. + AnyFirst bool } // NewFunc creates a new Func desciption of the given -// function, with specified number of output arguments, -// and an optional first string argument. +// function, which must have a signature like this: +// func([opt any,] a, b, out tensor.Tensor) error +// i.e., taking some specific number of Tensor arguments (up to 5). +// Functions can also take an 'any' first argument to handle other +// non-tensor inputs (e.g., function pointer, dirfs directory, etc). +// The name should be a standard 'package.FuncName' qualified, exported +// CamelCase name, with 'out' indicating the number of output arguments, +// and an optional arg indicating an 'any' first argument. // The remaining arguments in the function (automatically // determined) are classified as input arguments. -func NewFunc(name string, fun any, out int, stringFirst ...bool) (*Func, error) { +func NewFunc(name string, fun any, out int, anyFirst ...bool) (*Func, error) { fn := &Func{Name: name, Fun: fun, Out: out} - if len(stringFirst) == 1 && stringFirst[0] { - fn.StringFirst = true + if len(anyFirst) == 1 && anyFirst[0] { + fn.AnyFirst = true } nargs := fn.ArgCount() if out > nargs { @@ -59,19 +65,20 @@ func NewFunc(name string, fun any, out int, stringFirst ...bool) (*Func, error) } // Funcs is the global tensor named function registry. -// All functions must be of the form: func(a, b tensor.Tensor) -// i.e., taking some specific number of Tensor arguments, +// All functions must have a signature like this: +// func([opt any,] a, b, out tensor.Tensor) error +// i.e., taking some specific number of Tensor arguments (up to 5), // with the number of output vs. input arguments registered. +// Functions can also take an 'any' first argument to handle other +// non-tensor inputs (e.g., function pointer, dirfs directory, etc). +// This is used to make tensor functions available to the Goal +// language. var Funcs map[string]*Func // AddFunc adds given named function to the global tensor named function -// registry, which is used in goal to call functions by name, and -// in specific packages to call functions by enum String() names. -// Use the standard Go CamelCase name. -// The number of output arguments must be provided here, -// along with an optional first string argument if present; -// the number of input arguments is automatically set from that. -func AddFunc(name string, fun any, out int, stringFirst ...bool) error { +// registry, which is used by Goal to call functions by name. +// See [NewFunc] for more information. +func AddFunc(name string, fun any, out int, anyFirst ...bool) error { if Funcs == nil { Funcs = make(map[string]*Func) } @@ -79,7 +86,7 @@ func AddFunc(name string, fun any, out int, stringFirst ...bool) error { if ok { return errors.Log(fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name)) } - fn, err := NewFunc(name, fun, out, stringFirst...) + fn, err := NewFunc(name, fun, out, anyFirst...) if errors.Log(err) != nil { return err } @@ -88,61 +95,14 @@ func AddFunc(name string, fun any, out int, stringFirst ...bool) error { return nil } -// Call calls function of given name, with given set of arguments -// (input and output) appropriate for the given function. -// An error is returned if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match. -func Call(name string, tsr ...Tensor) error { - fn, ok := Funcs[name] - if !ok { - return errors.Log(fmt.Errorf("tensor.Call: function of name %q not registered", name)) - } - return fn.Call(tsr...) -} - -// CallString calls function of given name, with given set of arguments -// (string, input and output) appropriate for the given function. -// An error is returned if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match. This version of [Call] is for functions that -// have an initial string argument -func CallString(name, first string, tsr ...Tensor) error { - fn, ok := Funcs[name] - if !ok { - return errors.Log(fmt.Errorf("tensor.Call: function of name %q not registered", name)) - } - return fn.CallString(first, tsr...) -} - -// CallOut calls function of given name, with given set of _input_ -// arguments appropriate for the given function, returning a created -// output tensor, for the common case with just one return value. -// An error is logged if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match, or an error is returned by the function. -func CallOut(name string, tsr ...Tensor) Tensor { - fn, ok := Funcs[name] - if !ok { - errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) - return nil - } - return errors.Log1(fn.CallOut(tsr...))[0] -} - -// CallOutMulti calls function of given name, with given set of _input_ -// arguments appropriate for the given function, returning newly created -// output tensors, for the rare case of multiple return values. -// An error is logged if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match, or an error is returned by the function. -func CallOutMulti(name string, tsr ...Tensor) []Tensor { +// FuncByName finds function of given name in the registry, +// returning an error if the function name has not been registered. +func FuncByName(name string) (*Func, error) { fn, ok := Funcs[name] if !ok { - errors.Log(fmt.Errorf("tensor.CallOut: function of name %q not registered", name)) - return nil + return nil, errors.Log(fmt.Errorf("tensor.FuncByName: function of name %q not registered", name)) } - return errors.Log1(fn.CallOut(tsr...)) + return fn, nil } // ArgCount returns the number of tensor arguments the function takes, @@ -160,24 +120,24 @@ func (fn *Func) ArgCount() int { nargs = 4 case func(a, b, c, d, e Tensor) error: nargs = 5 - // string cases: - case func(s string, a Tensor) error: + // any cases: + case func(first any, a Tensor) error: nargs = 1 - case func(s string, a, b Tensor) error: + case func(first any, a, b Tensor) error: nargs = 2 - case func(s string, a, b, c Tensor) error: + case func(first any, a, b, c Tensor) error: nargs = 3 - case func(s string, a, b, c, d Tensor) error: + case func(first any, a, b, c, d Tensor) error: nargs = 4 - case func(s string, a, b, c, d, e Tensor) error: + case func(first any, a, b, c, d, e Tensor) error: nargs = 5 } return nargs } -// argCheck returns an error if the number of args in list does not +// ArgCheck returns an error if the number of args in list does not // match the number required as specified. -func (fn *Func) argCheck(n int, tsr ...Tensor) error { +func (fn *Func) ArgCheck(n int, tsr ...Tensor) error { if len(tsr) != n { return fmt.Errorf("tensor.Call: args passed to %q: %d does not match required: %d", fn.Name, len(tsr), n) } @@ -187,32 +147,32 @@ func (fn *Func) argCheck(n int, tsr ...Tensor) error { // Call calls function with given set of input & output arguments // appropriate for the given function (error if not). func (fn *Func) Call(tsr ...Tensor) error { - if fn.StringFirst { + if fn.AnyFirst { return fmt.Errorf("tensor.Call: function %q: requires a first string argument", fn.Name) } switch f := fn.Fun.(type) { case func(a Tensor) error: - if err := fn.argCheck(1, tsr...); err != nil { + if err := fn.ArgCheck(1, tsr...); err != nil { return err } return f(tsr[0]) case func(a, b Tensor) error: - if err := fn.argCheck(2, tsr...); err != nil { + if err := fn.ArgCheck(2, tsr...); err != nil { return err } return f(tsr[0], tsr[1]) case func(a, b, c Tensor) error: - if err := fn.argCheck(3, tsr...); err != nil { + if err := fn.ArgCheck(3, tsr...); err != nil { return err } return f(tsr[0], tsr[1], tsr[2]) case func(a, b, c, d Tensor) error: - if err := fn.argCheck(4, tsr...); err != nil { + if err := fn.ArgCheck(4, tsr...); err != nil { return err } return f(tsr[0], tsr[1], tsr[2], tsr[3]) case func(a, b, c, d, e Tensor) error: - if err := fn.argCheck(5, tsr...); err != nil { + if err := fn.ArgCheck(5, tsr...); err != nil { return err } return f(tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) @@ -220,47 +180,64 @@ func (fn *Func) Call(tsr ...Tensor) error { return nil } -// CallString calls function with given set of input & output arguments +// CallAny calls function with given set of input & output arguments // appropriate for the given function (error if not), -// with an initial string argument. -func (fn *Func) CallString(s string, tsr ...Tensor) error { - if !fn.StringFirst { - return fmt.Errorf("tensor.CallString: function %q: does not take a first string argument", fn.Name) +// with a first 'any' argument. +func (fn *Func) CallAny(first any, tsr ...Tensor) error { + if !fn.AnyFirst { + return fmt.Errorf("tensor.CallAny: function %q: does not take a first 'any' argument", fn.Name) } switch f := fn.Fun.(type) { - case func(s string, a Tensor) error: - if err := fn.argCheck(1, tsr...); err != nil { + case func(first any, a Tensor) error: + if err := fn.ArgCheck(1, tsr...); err != nil { return err } - return f(s, tsr[0]) - case func(s string, a, b Tensor) error: - if err := fn.argCheck(2, tsr...); err != nil { + return f(first, tsr[0]) + case func(first any, a, b Tensor) error: + if err := fn.ArgCheck(2, tsr...); err != nil { return err } - return f(s, tsr[0], tsr[1]) - case func(s string, a, b, c Tensor) error: - if err := fn.argCheck(3, tsr...); err != nil { + return f(first, tsr[0], tsr[1]) + case func(first any, a, b, c Tensor) error: + if err := fn.ArgCheck(3, tsr...); err != nil { return err } - return f(s, tsr[0], tsr[1], tsr[2]) - case func(s string, a, b, c, d Tensor) error: - if err := fn.argCheck(4, tsr...); err != nil { + return f(first, tsr[0], tsr[1], tsr[2]) + case func(first any, a, b, c, d Tensor) error: + if err := fn.ArgCheck(4, tsr...); err != nil { return err } - return f(s, tsr[0], tsr[1], tsr[2], tsr[3]) - case func(s string, a, b, c, d, e Tensor) error: - if err := fn.argCheck(5, tsr...); err != nil { + return f(first, tsr[0], tsr[1], tsr[2], tsr[3]) + case func(first any, a, b, c, d, e Tensor) error: + if err := fn.ArgCheck(5, tsr...); err != nil { return err } - return f(s, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) + return f(first, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) } return nil } -// CallOut calls function with given set of _input_ arguments -// appropriate for the given function (error if not). -// Newly-created output values are returned. -func (fn *Func) CallOut(tsr ...Tensor) ([]Tensor, error) { +// CallOut is like [Call] but it automatically creates an output +// tensor of the same type as the first input tensor passed, +// and returns the output as return values, along with any error. +func (fn *Func) CallOut(tsr ...Tensor) (Tensor, error) { + if fn.Out == 0 { + err := fn.Call(tsr...) + return nil, err + } + typ := reflect.Float64 + if fn.In > 0 { + typ = tsr[0].DataType() + } + out := NewOfType(typ) + tlist := slices.Clone(tsr) + tlist = append(tlist, out) + err := fn.Call(tlist...) + return out, err +} + +// CallOutMulti is like [CallOut] but deals with multiple output tensors. +func (fn *Func) CallOutMulti(tsr ...Tensor) ([]Tensor, error) { if fn.Out == 0 { err := fn.Call(tsr...) return nil, err @@ -278,6 +255,61 @@ func (fn *Func) CallOut(tsr ...Tensor) ([]Tensor, error) { return outs, err } +// Call calls function of given name, with given set of _input_ +// and output arguments appropriate for the given function. +// An error is logged if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match, or an error is returned by the function. +func Call(name string, tsr ...Tensor) error { + fn, err := FuncByName(name) + if err != nil { + return err + } + return fn.Call(tsr...) +} + +// CallOut calls function of given name, with given set of _input_ +// arguments appropriate for the given function, returning a created +// output tensor, for the common case with just one return value. +// An error is logged if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match, or an error is returned by the function. +func CallOut(name string, tsr ...Tensor) Tensor { + fn, err := FuncByName(name) + if errors.Log(err) != nil { + return nil + } + return errors.Log1(fn.CallOut(tsr...)) +} + +// CallAny calls function of given name, with given set of arguments +// (any, input and output) appropriate for the given function. +// An error is returned if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match. This version of [Call] is for functions that +// have an initial string argument +func CallAny(name string, first any, tsr ...Tensor) error { + fn, err := FuncByName(name) + if err != nil { + return err + } + return fn.CallAny(first, tsr...) +} + +// CallOutMulti calls function of given name, with given set of _input_ +// arguments appropriate for the given function, returning newly created +// output tensors, for the rare case of multiple return values. +// An error is logged if the function name has not been registered +// in the Funcs global function registry, or the argument count +// does not match, or an error is returned by the function. +func CallOutMulti(name string, tsr ...Tensor) []Tensor { + fn, err := FuncByName(name) + if errors.Log(err) != nil { + return nil + } + return errors.Log1(fn.CallOutMulti(tsr...)) +} + // SetCalcFunc sets a function to calculate updated value for given tensor, // storing the function pointer in the Metadata "CalcFunc" key for the tensor. // Can be called by [Calc] function. diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 99d4e5742a..186e8b73f1 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -36,16 +36,18 @@ func TestFuncs(t *testing.T) { oned := NewNumberFromValues(vals...) oneout := oned.Clone() - err = Call("Abs", oned, oneout) + fn, err := FuncByName("Abs") + assert.NoError(t, err) + err = fn.Call(oned, oneout) assert.NoError(t, err) assert.Equal(t, 1.507556722888818, oneout.Float1D(0)) - err = Call("Abs", oned) + err = fn.Call(oned) assert.Error(t, err) - out := CallOut("Abs", oned) - // assert.NoError(t, err) + out, err := fn.CallOut(oned) + assert.NoError(t, err) assert.Equal(t, AsFloat64Scalar(oneout), AsFloat64Scalar(out)) } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 8d27511ae5..1afeb9652f 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -23,7 +23,7 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type MetricFunc func(a, b, out tensor.Tensor) error +type MetricFunc = func(a, b, out tensor.Tensor) error // SumSquaresScaleOut64 computes the sum of squares differences between tensor values, // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. @@ -62,16 +62,16 @@ func SumSquaresOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { return scale64, err } -// SumSquaresFunc computes the sum of squares differences between tensor values, +// SumSquares computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. -func SumSquaresFunc(a, b, out tensor.Tensor) error { +func SumSquares(a, b, out tensor.Tensor) error { _, err := SumSquaresOut64(a, b, out) return err } -// EuclideanFunc computes the Euclidean square root of the sum of squares +// Euclidean computes the Euclidean square root of the sum of squares // differences between tensor values, aka the L2 Norm. -func EuclideanFunc(a, b, out tensor.Tensor) error { +func Euclidean(a, b, out tensor.Tensor) error { scale64, ss64, err := SumSquaresScaleOut64(a, b, out) if err != nil { return err @@ -92,10 +92,10 @@ func EuclideanFunc(a, b, out tensor.Tensor) error { return nil } -// AbsFunc computes the sum of the absolute value of differences between the +// Abs computes the sum of the absolute value of differences between the // tensor values, aka the L1 Norm. // See [MetricFunc] for general information. -func AbsFunc(a, b, out tensor.Tensor) error { +func Abs(a, b, out tensor.Tensor) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -107,10 +107,10 @@ func AbsFunc(a, b, out tensor.Tensor) error { return err } -// HammingFunc computes the sum of 1s for every element that is different, +// Hamming computes the sum of 1s for every element that is different, // i.e., "city block" distance. // See [MetricFunc] for general information. -func HammingFunc(a, b, out tensor.Tensor) error { +func Hamming(a, b, out tensor.Tensor) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -144,10 +144,10 @@ func SumSquaresBinTolScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.T return } -// EuclideanBinTolFunc computes the Euclidean square root of the sum of squares +// EuclideanBinTol computes the Euclidean square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. -func EuclideanBinTolFunc(a, b, out tensor.Tensor) error { +func EuclideanBinTol(a, b, out tensor.Tensor) error { scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) if err != nil { return err @@ -168,9 +168,9 @@ func EuclideanBinTolFunc(a, b, out tensor.Tensor) error { return nil } -// SumSquaresBinTolFunc computes the sum of squares differences between tensor values, +// SumSquaresBinTol computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTolFunc(a, b, out tensor.Tensor) error { +func SumSquaresBinTol(a, b, out tensor.Tensor) error { scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) if err != nil { return err @@ -191,14 +191,14 @@ func SumSquaresBinTolFunc(a, b, out tensor.Tensor) error { return nil } -// CrossEntropyFunc is a standard measure of the difference between two +// CrossEntropy is a standard measure of the difference between two // probabilty distributions, reflecting the additional entropy (uncertainty) associated // with measuring probabilities under distribution b when in fact they come from // distribution a. It is also the entropy of a plus the divergence between a from b, // using Kullback-Leibler (KL) divergence. It is computed as: // a * log(a/b) + (1-a) * log(1-a/1-b). // See [MetricFunc] for general information. -func CrossEntropyFunc(a, b, out tensor.Tensor) error { +func CrossEntropy(a, b, out tensor.Tensor) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -218,9 +218,9 @@ func CrossEntropyFunc(a, b, out tensor.Tensor) error { return err } -// InnerProductFunc computes the sum of the co-products of the two on-NaN tensor values. +// InnerProduct computes the sum of the co-products of the two on-NaN tensor values. // See [MetricFunc] for general information. -func InnerProductFunc(a, b, out tensor.Tensor) error { +func InnerProduct(a, b, out tensor.Tensor) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -232,10 +232,10 @@ func InnerProductFunc(a, b, out tensor.Tensor) error { return err } -// CovarianceFunc computes the co-variance between two vectors, +// Covariance computes the co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. -func CovarianceFunc(a, b, out tensor.Tensor) error { +func Covariance(a, b, out tensor.Tensor) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -304,18 +304,18 @@ func CorrelationOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { return ss64, nil } -// CorrelationFunc computes the correlation between two vectors, +// Correlation computes the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized [CovarianceFunc]). // Equivalent to the [CosineFunc] of mean-normalized vectors. -func CorrelationFunc(a, b, out tensor.Tensor) error { +func Correlation(a, b, out tensor.Tensor) error { _, err := CorrelationOut64(a, b, out) return err } -// InvCorrelationFunc computes 1 minus the correlation between two vectors, +// InvCorrelation computes 1 minus the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). @@ -323,7 +323,7 @@ func CorrelationFunc(a, b, out tensor.Tensor) error { // Equivalent to the [CosineFunc] of mean-normalized vectors. // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCorrelationFunc(a, b, out tensor.Tensor) error { +func InvCorrelation(a, b, out tensor.Tensor) error { cor64, err := CorrelationOut64(a, b, out) if err != nil { return err @@ -364,20 +364,20 @@ func CosineOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { return ss64, nil } -// CosineFunc computes the high-dimensional angle between two vectors, +// Cosine computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc] -func CosineFunc(a, b, out tensor.Tensor) error { +func Cosine(a, b, out tensor.Tensor) error { _, err := CosineOut64(a, b, out) return err } -// InvCosineFunc computes 1 minus the cosine between two vectors, +// InvCosine computes 1 minus the cosine between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCosineFunc(a, b, out tensor.Tensor) error { +func InvCosine(a, b, out tensor.Tensor) error { cos64, err := CosineOut64(a, b, out) if err != nil { return err diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index f5cb89edcd..2789e1a3ce 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -16,9 +16,12 @@ import ( ) func init() { - tensor.AddFunc("metric.Matrix", Matrix, 1, tensor.StringFirstArg) - tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.StringFirstArg) - tensor.AddFunc("metric.CovarianceMatrix", CovarianceMatrix, 1, tensor.StringFirstArg) + tensor.AddFunc("metric.Matrix", Matrix, 1, tensor.AnyFirstArg) + tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.AnyFirstArg) + tensor.AddFunc("metric.CovarianceMatrix", CovarianceMatrix, 1, tensor.AnyFirstArg) + tensor.AddFunc("metric.PCA", PCA, 2) + tensor.AddFunc("metric.SVD", SVD, 2) + tensor.AddFunc("metric.ProjectOnMatrixColumn", ProjectOnMatrixColumn, 1) } // Matrix computes the rows x rows square distance / similarity matrix @@ -27,11 +30,15 @@ func init() { // and within that, 1+dimensional patterns (cells). Use [tensor.NewRowCellsView] // to organize data into the desired split between a 1D outermost Row dimension // and the remaining Cells dimension. -// The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). +// The metric function must have the [MetricFunc] signature. // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied // to the upper triangular region, for maximum efficiency. -func Matrix(funcName string, in, out tensor.Tensor) error { +func Matrix(fun any, in, out tensor.Tensor) error { + mfun, err := AsMetricFunc(fun) + if err != nil { + return err + } rows, cells := in.Shape().RowCellSize() if rows == 0 || cells == 0 { return nil @@ -48,7 +55,7 @@ func Matrix(funcName string, in, out tensor.Tensor) error { c := coords[idx] sa := tensor.Cells1D(tsr[0], c.X) sb := tensor.Cells1D(tsr[0], c.Y) - tensor.Call(funcName, sa, sb, mout) + mfun(sa, sb, mout) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, in, out) for _, c := range coords { // copy to upper @@ -66,10 +73,14 @@ func Matrix(funcName string, in, out tensor.Tensor) error { // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns that the given distance metric // function is applied to, with the results filling in the cells of the output matrix. -// The metric function registered in tensor Funcs can be passed as Metrics.FuncName(). +// The metric function must have the [MetricFunc] signature. // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. -func CrossMatrix(funcName string, a, b, out tensor.Tensor) error { +func CrossMatrix(fun any, a, b, out tensor.Tensor) error { + mfun, err := AsMetricFunc(fun) + if err != nil { + return err + } arows, acells := a.Shape().RowCellSize() if arows == 0 || acells == 0 { return nil @@ -91,7 +102,7 @@ func CrossMatrix(funcName string, a, b, out tensor.Tensor) error { br := idx % brows sa := tensor.Cells1D(tsr[0], ar) sb := tensor.Cells1D(tsr[1], br) - tensor.Call(funcName, sa, sb, mout) + mfun(sa, sb, mout) tsr[2].SetFloat(mout.Float1D(0), ar, br) }, a, b, out) return nil @@ -105,12 +116,16 @@ func CrossMatrix(funcName string, a, b, out tensor.Tensor) error { // value of a given cell covaries across the rows of the tensor with the // value of another cell. // Uses the given metric function, typically [Covariance] or [Correlation], -// which must be registered in tensor Funcs, and can be passed as Metrics.FuncName(). +// The metric function must have the [MetricFunc] signature. // Use Covariance if vars have similar overall scaling, // which is typical in neural network models, and use // Correlation if they are on very different scales, because it effectively rescales). // The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. -func CovarianceMatrix(funcName string, in, out tensor.Tensor) error { +func CovarianceMatrix(fun any, in, out tensor.Tensor) error { + mfun, err := AsMetricFunc(fun) + if err != nil { + return err + } rows, cells := in.Shape().RowCellSize() if rows == 0 || cells == 0 { return nil @@ -138,7 +153,7 @@ func CovarianceMatrix(funcName string, in, out tensor.Tensor) error { bv = tensor.NewSliced(tsr[0], tensor.Slice{}, tensor.Slice{Start: c.Y, Stop: c.Y + 1}) curCoords.Y = c.Y } - tensor.Call(funcName, av, bv, mout) + mfun(av, bv, mout) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, flatvw, out) for _, c := range coords { // copy to upper @@ -243,13 +258,13 @@ func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) error { if err != nil { return err } - stats.SumFunc(mout, msum) + stats.Sum(mout, msum) out.SetFloat1D(msum.Float1D(0), i) } } else { mout := tensor.NewFloat64(1) tmath.Mul(vec, col, mout) - stats.SumFunc(mout, out) + stats.Sum(mout, out) } return nil } diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 1d5a3443ae..c66a81188b 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -25,46 +25,46 @@ func TestFuncs(t *testing.T) { btsr := tensor.NewNumberFromValues(b64...) out := tensor.NewFloat64(1) - EuclideanFunc(atsr, btsr, out) - assert.InDelta(t, results[Euclidean], out.Values[0], tol) + Euclidean(atsr, btsr, out) + assert.InDelta(t, results[MetricEuclidean], out.Values[0], tol) - SumSquaresFunc(atsr, btsr, out) - assert.InDelta(t, results[SumSquares], out.Values[0], tol) + SumSquares(atsr, btsr, out) + assert.InDelta(t, results[MetricSumSquares], out.Values[0], tol) - EuclideanBinTolFunc(atsr, btsr, out) - assert.InDelta(t, results[EuclideanBinTol], out.Values[0], tol) + EuclideanBinTol(atsr, btsr, out) + assert.InDelta(t, results[MetricEuclideanBinTol], out.Values[0], tol) - AbsFunc(atsr, btsr, out) - assert.InDelta(t, results[Abs], out.Values[0], tol) + Abs(atsr, btsr, out) + assert.InDelta(t, results[MetricAbs], out.Values[0], tol) - HammingFunc(atsr, btsr, out) - assert.Equal(t, results[Hamming], out.Values[0]) + Hamming(atsr, btsr, out) + assert.Equal(t, results[MetricHamming], out.Values[0]) - SumSquaresBinTolFunc(atsr, btsr, out) - assert.InDelta(t, results[SumSquaresBinTol], out.Values[0], tol) + SumSquaresBinTol(atsr, btsr, out) + assert.InDelta(t, results[MetricSumSquaresBinTol], out.Values[0], tol) - CovarianceFunc(atsr, btsr, out) - assert.InDelta(t, results[Covariance], out.Values[0], tol) + Covariance(atsr, btsr, out) + assert.InDelta(t, results[MetricCovariance], out.Values[0], tol) - CorrelationFunc(atsr, btsr, out) - assert.InDelta(t, results[Correlation], out.Values[0], tol) + Correlation(atsr, btsr, out) + assert.InDelta(t, results[MetricCorrelation], out.Values[0], tol) - InvCorrelationFunc(atsr, btsr, out) - assert.InDelta(t, results[InvCorrelation], out.Values[0], tol) + InvCorrelation(atsr, btsr, out) + assert.InDelta(t, results[MetricInvCorrelation], out.Values[0], tol) - CrossEntropyFunc(atsr, btsr, out) - assert.InDelta(t, results[CrossEntropy], out.Values[0], tol) + CrossEntropy(atsr, btsr, out) + assert.InDelta(t, results[MetricCrossEntropy], out.Values[0], tol) - InnerProductFunc(atsr, btsr, out) - assert.InDelta(t, results[InnerProduct], out.Values[0], tol) + InnerProduct(atsr, btsr, out) + assert.InDelta(t, results[MetricInnerProduct], out.Values[0], tol) - CosineFunc(atsr, btsr, out) - assert.InDelta(t, results[Cosine], out.Values[0], tol) + Cosine(atsr, btsr, out) + assert.InDelta(t, results[MetricCosine], out.Values[0], tol) - InvCosineFunc(atsr, btsr, out) - assert.InDelta(t, results[InvCosine], out.Values[0], tol) + InvCosine(atsr, btsr, out) + assert.InDelta(t, results[MetricInvCosine], out.Values[0], tol) - for met := Euclidean; met < MetricsN; met++ { + for met := MetricEuclidean; met < MetricsN; met++ { met.Call(atsr, btsr, out) assert.InDelta(t, results[met], out.Values[0], tol) } @@ -90,7 +90,7 @@ func TestMatrix(t *testing.T) { assert.NoError(t, err) in := dt.Column("Input") out := tensor.NewFloat64() - err = Matrix(Euclidean.FuncName(), in, out) + err = Matrix(Euclidean, in, out) assert.NoError(t, err) // fmt.Println(out.Tensor) assert.Equal(t, simres, out.String()) @@ -109,7 +109,7 @@ func TestPCAIris(t *testing.T) { } data := dt.Column("data") covar := tensor.NewFloat64() - err = CovarianceMatrix(Correlation.FuncName(), data, covar) + err = CovarianceMatrix(Correlation, data, covar) assert.NoError(t, err) // fmt.Printf("correl: %s\n", covar.String()) diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index a353bf41d6..a77fa521f1 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -7,60 +7,61 @@ package metric import ( + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) func init() { - tensor.AddFunc(Euclidean.FuncName(), EuclideanFunc, 1) - tensor.AddFunc(SumSquares.FuncName(), SumSquaresFunc, 1) - tensor.AddFunc(Abs.FuncName(), AbsFunc, 1) - tensor.AddFunc(Hamming.FuncName(), HammingFunc, 1) - tensor.AddFunc(EuclideanBinTol.FuncName(), EuclideanBinTolFunc, 1) - tensor.AddFunc(SumSquaresBinTol.FuncName(), SumSquaresBinTolFunc, 1) - tensor.AddFunc(InvCosine.FuncName(), InvCosineFunc, 1) - tensor.AddFunc(InvCorrelation.FuncName(), InvCorrelationFunc, 1) - tensor.AddFunc(InnerProduct.FuncName(), InnerProductFunc, 1) - tensor.AddFunc(CrossEntropy.FuncName(), CrossEntropyFunc, 1) - tensor.AddFunc(Covariance.FuncName(), CovarianceFunc, 1) - tensor.AddFunc(Correlation.FuncName(), CorrelationFunc, 1) - tensor.AddFunc(Cosine.FuncName(), CosineFunc, 1) + tensor.AddFunc(MetricEuclidean.FuncName(), Euclidean, 1) + tensor.AddFunc(MetricSumSquares.FuncName(), SumSquares, 1) + tensor.AddFunc(MetricAbs.FuncName(), Abs, 1) + tensor.AddFunc(MetricHamming.FuncName(), Hamming, 1) + tensor.AddFunc(MetricEuclideanBinTol.FuncName(), EuclideanBinTol, 1) + tensor.AddFunc(MetricSumSquaresBinTol.FuncName(), SumSquaresBinTol, 1) + tensor.AddFunc(MetricInvCosine.FuncName(), InvCosine, 1) + tensor.AddFunc(MetricInvCorrelation.FuncName(), InvCorrelation, 1) + tensor.AddFunc(MetricInnerProduct.FuncName(), InnerProduct, 1) + tensor.AddFunc(MetricCrossEntropy.FuncName(), CrossEntropy, 1) + tensor.AddFunc(MetricCovariance.FuncName(), Covariance, 1) + tensor.AddFunc(MetricCorrelation.FuncName(), Correlation, 1) + tensor.AddFunc(MetricCosine.FuncName(), Cosine, 1) } // Metrics are standard metric functions -type Metrics int32 //enums:enum +type Metrics int32 //enums:enum -trim-prefix Metric const ( // Euclidean is the square root of the sum of squares differences // between tensor values, aka the L2Norm. - Euclidean Metrics = iota + MetricEuclidean Metrics = iota // SumSquares is the sum of squares differences between tensor values. - SumSquares + MetricSumSquares // Abs is the sum of the absolute value of differences // between tensor values, aka the L1Norm. - Abs + MetricAbs // Hamming is the sum of 1s for every element that is different, // i.e., "city block" distance. - Hamming + MetricHamming // EuclideanBinTol is the [Euclidean] square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. - EuclideanBinTol + MetricEuclideanBinTol // SumSquaresBinTol is the [SumSquares] differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. - SumSquaresBinTol + MetricSumSquaresBinTol // InvCosine is 1-[Cosine], which is useful to convert it // to an Increasing metric where more different vectors have larger metric values. - InvCosine + MetricInvCosine // InvCorrelation is 1-[Correlation], which is useful to convert it // to an Increasing metric where more different vectors have larger metric values. - InvCorrelation + MetricInvCorrelation // CrossEntropy is a standard measure of the difference between two // probabilty distributions, reflecting the additional entropy (uncertainty) associated @@ -68,30 +69,30 @@ const ( // distribution a. It is also the entropy of a plus the divergence between a from b, // using Kullback-Leibler (KL) divergence. It is computed as: // a * log(a/b) + (1-a) * log(1-a/1-b). - CrossEntropy + MetricCrossEntropy ///////////////////////////////////////////////////////////////////////// // Everything below here is !Increasing -- larger = closer, not farther // InnerProduct is the sum of the co-products of the tensor values. - InnerProduct + MetricInnerProduct // Covariance is co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. - Covariance + MetricCovariance // Correlation is the standardized [Covariance] in the range (-1..1), // computed as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // Equivalent to the [Cosine] of mean-normalized vectors. - Correlation + MetricCorrelation // Cosine is high-dimensional angle between two vectors, // in range (-1..1) as the normalized [InnerProduct]: // inner product / sqrt(ssA * ssB). See also [Correlation]. - Cosine + MetricCosine ) // FuncName returns the package-qualified function name to use @@ -100,10 +101,16 @@ func (m Metrics) FuncName() string { return "metric." + m.String() } +// Func returns function for given metric. +func (m Metrics) Func() MetricFunc { + fn := errors.Log1(tensor.FuncByName(m.FuncName())) + return fn.Fun.(MetricFunc) +} + // Call calls a standard Metrics enum function on given tensors. // Output results are in the out tensor. -func (m Metrics) Call(a, b, out tensor.Tensor) { - tensor.Call(m.FuncName(), a, b, out) +func (m Metrics) Call(a, b, out tensor.Tensor) error { + return tensor.Call(m.FuncName(), a, b, out) } // CallOut calls a standard Metrics enum function on given tensors, @@ -117,8 +124,18 @@ func (m Metrics) CallOut(a, b tensor.Tensor) tensor.Tensor { // and false if metric values decrease as a function of distance // (e.g., Cosine, Correlation) func (m Metrics) Increasing() bool { - if m >= InnerProduct { + if m >= MetricInnerProduct { return false } return true } + +// AsMetricFunc returns given function as a [MetricFunc] function, +// or an error if it does not fit that signature. +func AsMetricFunc(fun any) (MetricFunc, error) { + mfun, ok := fun.(MetricFunc) + if !ok { + return nil, errors.New("metric.AsMetricFunc: function does not fit the MetricFunc signature") + } + return mfun, nil +} diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 65bcd44058..8f8e156e9b 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -11,21 +11,25 @@ import ( ) func init() { - tensor.AddFunc("ClosestRow", ClosestRow, 1, tensor.StringFirstArg) + tensor.AddFunc("metric.ClosestRow", ClosestRow, 1, tensor.AnyFirstArg) } // ClosestRow returns the closest fit between probe pattern and patterns in // a "vocabulary" tensor with outermost row dimension, using given metric -// function registered in tensor Funcs (e.g., use String() method on Metrics enum). +// function, which must fit the MetricFunc signature. // The metric *must have the Increasing property*, i.e., larger = further. // Output is a 1D tensor with 2 elements: the row index and metric value for that row. // Note: this does _not_ use any existing Indexes for the probe, // but does for the vocab, and the returned index is the logical index // into any existing Indexes. -func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) error { +func ClosestRow(fun any, probe, vocab, out tensor.Tensor) error { if err := tensor.SetShapeSizesMustBeValues(out, 2); err != nil { return err } + mfun, err := AsMetricFunc(fun) + if err != nil { + return err + } rows, _ := vocab.Shape().RowCellSize() mi := -1 mout := tensor.NewFloat64Scalar(0.0) @@ -33,7 +37,7 @@ func ClosestRow(funcName string, probe, vocab, out tensor.Tensor) error { pr1d := tensor.As1D(probe) for ri := range rows { sub := tensor.Cells1D(vocab, ri) - tensor.Call(funcName, pr1d, sub, mout) + mfun(pr1d, sub, mout) d := mout.Float1D(0) if d < mind { mi = ri diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go index c6be23e990..ca9df612d7 100644 --- a/tensor/stats/stats/describe.go +++ b/tensor/stats/stats/describe.go @@ -15,7 +15,7 @@ import ( // DescriptiveStats are the standard descriptive stats used in Describe function. // Cannot apply the final 3 sort-based stats to higher-dimensional data. -var DescriptiveStats = []Stats{Count, Mean, Std, Sem, Min, Max, Q1, Median, Q3} +var DescriptiveStats = []Stats{StatCount, StatMean, StatStd, StatSem, StatMin, StatMax, StatQ1, StatMedian, StatQ3} // Describe adds standard descriptive statistics for given tensor // to the given [datafs] directory, adding a directory for each tensor diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 9b4c8a00fd..c79407bc00 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -20,7 +20,7 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type StatsFunc func(in, out tensor.Tensor) error +type StatsFunc = func(in, out tensor.Tensor) error // CountOut64 computes the count of non-NaN tensor values, // and returns the Float64 output values for subsequent use. @@ -32,9 +32,9 @@ func CountOut64(in, out tensor.Tensor) (tensor.Tensor, error) { }, in, out) } -// CountFunc computes the count of non-NaN tensor values. +// Count computes the count of non-NaN tensor values. // See [StatsFunc] for general information. -func CountFunc(in, out tensor.Tensor) error { +func Count(in, out tensor.Tensor) error { _, err := CountOut64(in, out) return err } @@ -49,17 +49,17 @@ func SumOut64(in, out tensor.Tensor) (tensor.Tensor, error) { }, in, out) } -// SumFunc computes the sum of tensor values. +// Sum computes the sum of tensor values. // See [StatsFunc] for general information. -func SumFunc(in, out tensor.Tensor) error { +func Sum(in, out tensor.Tensor) error { _, err := SumOut64(in, out) return err } -// SumAbsFunc computes the sum of absolute-value-of tensor values. +// SumAbs computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. -func SumAbsFunc(in, out tensor.Tensor) error { +func SumAbs(in, out tensor.Tensor) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) @@ -68,9 +68,9 @@ func SumAbsFunc(in, out tensor.Tensor) error { return err } -// ProdFunc computes the product of tensor values. +// Prod computes the product of tensor values. // See [StatsFunc] for general information. -func ProdFunc(in, out tensor.Tensor) error { +func Prod(in, out tensor.Tensor) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { return agg * val @@ -79,9 +79,9 @@ func ProdFunc(in, out tensor.Tensor) error { return err } -// MinFunc computes the min of tensor values. +// Min computes the min of tensor values. // See [StatsFunc] for general information. -func MinFunc(in, out tensor.Tensor) error { +func Min(in, out tensor.Tensor) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, val) @@ -90,9 +90,9 @@ func MinFunc(in, out tensor.Tensor) error { return err } -// MaxFunc computes the max of tensor values. +// Max computes the max of tensor values. // See [StatsFunc] for general information. -func MaxFunc(in, out tensor.Tensor) error { +func Max(in, out tensor.Tensor) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, val) @@ -101,9 +101,9 @@ func MaxFunc(in, out tensor.Tensor) error { return err } -// MinAbsFunc computes the min of absolute-value-of tensor values. +// MinAbs computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MinAbsFunc(in, out tensor.Tensor) error { +func MinAbs(in, out tensor.Tensor) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, math.Abs(val)) @@ -112,9 +112,9 @@ func MinAbsFunc(in, out tensor.Tensor) error { return err } -// MaxAbsFunc computes the max of absolute-value-of tensor values. +// MaxAbs computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MaxAbsFunc(in, out tensor.Tensor) error { +func MaxAbs(in, out tensor.Tensor) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, math.Abs(val)) @@ -144,9 +144,9 @@ func MeanOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor, err error) return sum64, count64, err } -// MeanFunc computes the mean of tensor values. +// Mean computes the mean of tensor values. // See [StatsFunc] for general information. -func MeanFunc(in, out tensor.Tensor) error { +func Mean(in, out tensor.Tensor) error { _, _, err := MeanOut64(in, out) return err } @@ -187,10 +187,10 @@ func VarOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, err return } -// VarFunc computes the sample variance of tensor values. +// Var computes the sample variance of tensor values. // Squared deviations from mean, divided by n-1. See also [VarPopFunc]. // See [StatsFunc] for general information. -func VarFunc(in, out tensor.Tensor) error { +func Var(in, out tensor.Tensor) error { _, _, _, err := VarOut64(in, out) return err } @@ -211,18 +211,18 @@ func StdOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor, err return } -// StdFunc computes the sample standard deviation of tensor values. +// Std computes the sample standard deviation of tensor values. // Sqrt of variance from [VarFunc]. See also [StdPopFunc]. // See [StatsFunc] for general information. -func StdFunc(in, out tensor.Tensor) error { +func Std(in, out tensor.Tensor) error { _, _, _, err := StdOut64(in, out) return err } -// SemFunc computes the sample standard error of the mean of tensor values. +// Sem computes the sample standard error of the mean of tensor values. // Standard deviation [StdFunc] / sqrt(n). See also [SemPopFunc]. // See [StatsFunc] for general information. -func SemFunc(in, out tensor.Tensor) error { +func Sem(in, out tensor.Tensor) error { var64, _, count64, err := VarOut64(in, out) if err != nil { return err @@ -258,18 +258,18 @@ func VarPopOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, e return } -// VarPopFunc computes the population variance of tensor values. +// VarPop computes the population variance of tensor values. // Squared deviations from mean, divided by n. See also [VarFunc]. // See [StatsFunc] for general information. -func VarPopFunc(in, out tensor.Tensor) error { +func VarPop(in, out tensor.Tensor) error { _, _, _, err := VarPopOut64(in, out) return err } -// StdPopFunc computes the population standard deviation of tensor values. +// StdPop computes the population standard deviation of tensor values. // Sqrt of variance from [VarPopFunc]. See also [StdFunc]. // See [StatsFunc] for general information. -func StdPopFunc(in, out tensor.Tensor) error { +func StdPop(in, out tensor.Tensor) error { var64, _, _, err := VarPopOut64(in, out) if err != nil { return err @@ -281,10 +281,10 @@ func StdPopFunc(in, out tensor.Tensor) error { return nil } -// SemPopFunc computes the population standard error of the mean of tensor values. +// SemPop computes the population standard error of the mean of tensor values. // Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. // See [StatsFunc] for general information. -func SemPopFunc(in, out tensor.Tensor) error { +func SemPop(in, out tensor.Tensor) error { var64, _, count64, err := VarPopOut64(in, out) if err != nil { return err @@ -346,9 +346,9 @@ func SumSqOut64(in, out tensor.Tensor) (tensor.Tensor, error) { return scale64, nil } -// SumSqFunc computes the sum of squares of tensor values, +// SumSq computes the sum of squares of tensor values, // See [StatsFunc] for general information. -func SumSqFunc(in, out tensor.Tensor) error { +func SumSq(in, out tensor.Tensor) error { _, err := SumSqOut64(in, out) return err } @@ -377,10 +377,10 @@ func L2NormOut64(in, out tensor.Tensor) (tensor.Tensor, error) { return scale64, nil } -// L2NormFunc computes the square root of the sum of squares of tensor values, +// L2Norm computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsFunc] for general information. -func L2NormFunc(in, out tensor.Tensor) error { +func L2Norm(in, out tensor.Tensor) error { _, err := L2NormOut64(in, out) return err } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 4e3752470e..1ac70c70ed 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -13,6 +13,13 @@ import ( "cogentcore.org/core/tensor/table" ) +// note: we cannot register these functions because they take vararg tensors!! +// +// func init() { +// tensor.AddFunc("stats.Groups", Groups, 0, tensor.AnyFirstArg) +// tensor.AddFunc("stats.GroupAll", GroupAll, 0, tensor.AnyFirstArg) +// } + // Groups generates indexes for each unique value in each of the given tensors. // One can then use the resulting indexes for the [tensor.Rows] indexes to // perform computations restricted to grouped subsets of data, as in the @@ -119,6 +126,10 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { // todo: GroupCombined +// note: we have to pass stat as a string here because we need the name +// to record the results in the datafs, and we can't get the name directly. +// also we need _2_ anys, and varargs! + // GroupStats computes the given stats function on the unique grouped indexes // produced by the [Groups] function, in the given [datafs] directory, // applied to each of the tensors passed here. diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index 7dcf922e3b..6ec5061b9d 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -33,7 +33,7 @@ func TestGroup(t *testing.T) { assert.Equal(t, []int{0, 1}, tensor.AsIntTensor(ixs[0]).Values) assert.Equal(t, []int{2, 3}, tensor.AsIntTensor(ixs[1]).Values) - err = TableGroupStats(dir, Mean, dt, "Value") + err = TableGroupStats(dir, StatMean, dt, "Value") assert.NoError(t, err) // AggColumn(spl, "Value", stats.Mean) diff --git a/tensor/stats/stats/norm.go b/tensor/stats/stats/norm.go index 3e6c6cd2e1..cd7181f3a0 100644 --- a/tensor/stats/stats/norm.go +++ b/tensor/stats/stats/norm.go @@ -27,12 +27,12 @@ func ZScore(a, out tensor.Tensor) error { // subtracting the Min value and dividing by the Max of the remaining numbers. func UnitNorm(a, out tensor.Tensor) error { mout := tensor.NewFloat64() - err := MinFunc(a, mout) + err := Min(a, mout) if err != nil { return err } tmath.Sub(a, mout, out) - MaxFunc(out, mout) + Max(out, mout) tmath.Div(out, mout, out) return nil } diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index d0e181215b..1978155654 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -11,7 +11,7 @@ import ( "cogentcore.org/core/tensor" ) -// QuantilesFunc returns the given quantile(s) of non-NaN elements in given +// Quantiles returns the given quantile(s) of non-NaN elements in given // 1D tensor. Because sorting uses indexes, this only works for 1D case. // If needed for a sub-space of values, that can be extracted through slicing // and then used. Returns and logs an error if not 1D. @@ -19,7 +19,7 @@ import ( // Uses linear interpolation. // Because this requires a sort, it is more efficient to get as many quantiles // as needed in one pass. -func QuantilesFunc(in, qs, out tensor.Tensor) error { +func Quantiles(in, qs, out tensor.Tensor) error { if in.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D input tensors allowed")) } @@ -60,20 +60,20 @@ func QuantilesFunc(in, qs, out tensor.Tensor) error { return nil } -// MedianFunc computes the median (50% quantile) of tensor values. +// Median computes the median (50% quantile) of tensor values. // See [StatsFunc] for general information. -func MedianFunc(in, out tensor.Tensor) error { - return QuantilesFunc(in, tensor.NewFloat64Scalar(.5), out) +func Median(in, out tensor.Tensor) error { + return Quantiles(in, tensor.NewFloat64Scalar(.5), out) } -// Q1Func computes the first quantile (25%) of tensor values. +// Q1 computes the first quantile (25%) of tensor values. // See [StatsFunc] for general information. -func Q1Func(in, out tensor.Tensor) error { - return QuantilesFunc(in, tensor.NewFloat64Scalar(.25), out) +func Q1(in, out tensor.Tensor) error { + return Quantiles(in, tensor.NewFloat64Scalar(.25), out) } -// Q3Func computes the third quantile (75%) of tensor values. +// Q3 computes the third quantile (75%) of tensor values. // See [StatsFunc] for general information. -func Q3Func(in, out tensor.Tensor) error { - return QuantilesFunc(in, tensor.NewFloat64Scalar(.75), out) +func Q3(in, out tensor.Tensor) error { + return Quantiles(in, tensor.NewFloat64Scalar(.75), out) } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 70684bdec1..1115bfdda5 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -7,102 +7,103 @@ package stats import ( "strings" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) //go:generate core generate func init() { - tensor.AddFunc(Count.FuncName(), CountFunc, 1) - tensor.AddFunc(Sum.FuncName(), SumFunc, 1) - tensor.AddFunc(SumAbs.FuncName(), SumAbsFunc, 1) - tensor.AddFunc(L1Norm.FuncName(), SumAbsFunc, 1) - tensor.AddFunc(Prod.FuncName(), ProdFunc, 1) - tensor.AddFunc(Min.FuncName(), MinFunc, 1) - tensor.AddFunc(Max.FuncName(), MaxFunc, 1) - tensor.AddFunc(MinAbs.FuncName(), MinAbsFunc, 1) - tensor.AddFunc(MaxAbs.FuncName(), MaxAbsFunc, 1) - tensor.AddFunc(Mean.FuncName(), MeanFunc, 1) - tensor.AddFunc(Var.FuncName(), VarFunc, 1) - tensor.AddFunc(Std.FuncName(), StdFunc, 1) - tensor.AddFunc(Sem.FuncName(), SemFunc, 1) - tensor.AddFunc(SumSq.FuncName(), SumSqFunc, 1) - tensor.AddFunc(L2Norm.FuncName(), L2NormFunc, 1) - tensor.AddFunc(VarPop.FuncName(), VarPopFunc, 1) - tensor.AddFunc(StdPop.FuncName(), StdPopFunc, 1) - tensor.AddFunc(SemPop.FuncName(), SemPopFunc, 1) - tensor.AddFunc(Median.FuncName(), MedianFunc, 1) - tensor.AddFunc(Q1.FuncName(), Q1Func, 1) - tensor.AddFunc(Q3.FuncName(), Q3Func, 1) + tensor.AddFunc(StatCount.FuncName(), Count, 1) + tensor.AddFunc(StatSum.FuncName(), Sum, 1) + tensor.AddFunc(StatSumAbs.FuncName(), SumAbs, 1) + tensor.AddFunc(StatL1Norm.FuncName(), SumAbs, 1) + tensor.AddFunc(StatProd.FuncName(), Prod, 1) + tensor.AddFunc(StatMin.FuncName(), Min, 1) + tensor.AddFunc(StatMax.FuncName(), Max, 1) + tensor.AddFunc(StatMinAbs.FuncName(), MinAbs, 1) + tensor.AddFunc(StatMaxAbs.FuncName(), MaxAbs, 1) + tensor.AddFunc(StatMean.FuncName(), Mean, 1) + tensor.AddFunc(StatVar.FuncName(), Var, 1) + tensor.AddFunc(StatStd.FuncName(), Std, 1) + tensor.AddFunc(StatSem.FuncName(), Sem, 1) + tensor.AddFunc(StatSumSq.FuncName(), SumSq, 1) + tensor.AddFunc(StatL2Norm.FuncName(), L2Norm, 1) + tensor.AddFunc(StatVarPop.FuncName(), VarPop, 1) + tensor.AddFunc(StatStdPop.FuncName(), StdPop, 1) + tensor.AddFunc(StatSemPop.FuncName(), SemPop, 1) + tensor.AddFunc(StatMedian.FuncName(), Median, 1) + tensor.AddFunc(StatQ1.FuncName(), Q1, 1) + tensor.AddFunc(StatQ3.FuncName(), Q3, 1) } // Stats is a list of different standard aggregation functions, which can be used // to choose an aggregation function -type Stats int32 //enums:enum +type Stats int32 //enums:enum -trim-prefix Stat const ( // count of number of elements. - Count Stats = iota + StatCount Stats = iota // sum of elements. - Sum + StatSum // sum of absolute-value-of elements (= L1Norm). - SumAbs + StatSumAbs // L1 Norm: sum of absolute values (= SumAbs). - L1Norm + StatL1Norm // product of elements. - Prod + StatProd // minimum value. - Min + StatMin // maximum value. - Max + StatMax // minimum of absolute values. - MinAbs + StatMinAbs // maximum of absolute values. - MaxAbs + StatMaxAbs // mean value = sum / count. - Mean + StatMean // sample variance (squared deviations from mean, divided by n-1). - Var + StatVar // sample standard deviation (sqrt of Var). - Std + StatStd // sample standard error of the mean (Std divided by sqrt(n)). - Sem + StatSem // sum of squared values. - SumSq + StatSumSq // L2 Norm: square-root of sum-of-squares. - L2Norm + StatL2Norm // population variance (squared diffs from mean, divided by n). - VarPop + StatVarPop // population standard deviation (sqrt of VarPop). - StdPop + StatStdPop // population standard error of the mean (StdPop divided by sqrt(n)). - SemPop + StatSemPop // middle value in sorted ordering. - Median + StatMedian // Q1 first quartile = 25%ile value = .25 quantile value. - Q1 + StatQ1 // Q3 third quartile = 75%ile value = .75 quantile value. - Q3 + StatQ3 ) // FuncName returns the package-qualified function name to use @@ -111,13 +112,19 @@ func (s Stats) FuncName() string { return "stats." + s.String() } +// Func returns function for given stat. +func (s Stats) Func() StatsFunc { + fn := errors.Log1(tensor.FuncByName(s.FuncName())) + return fn.Fun.(StatsFunc) +} + // Call calls this statistic function on given tensors. // Output results are in the out tensor. -func (s Stats) Call(in, out tensor.Tensor) { - tensor.Call(s.FuncName(), in, out) +func (s Stats) Call(in, out tensor.Tensor) error { + return tensor.Call(s.FuncName(), in, out) } -// StatOut calls a standard Stats enum function on given tensor, +// CallOut calls a standard Stats enum function on given tensor, // returning output as a newly created tensor. func (s Stats) CallOut(in tensor.Tensor) tensor.Tensor { return tensor.CallOut(s.FuncName(), in) @@ -133,3 +140,13 @@ func StripPackage(name string) string { } return name } + +// AsStatsFunc returns given function as a [StatsFunc] function, +// or an error if it does not fit that signature. +func AsStatsFunc(fun any) (StatsFunc, error) { + sfun, ok := fun.(StatsFunc) + if !ok { + return nil, errors.New("metric.AsStatsFunc: function does not fit the StatsFunc signature") + } + return sfun, nil +} diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index bacbf45be3..4d7039552d 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -21,70 +21,74 @@ func TestFuncs64(t *testing.T) { tol := 1.0e-8 - CountFunc(ix, out) - assert.Equal(t, results[Count], out.Values[0]) + Count(ix, out) + assert.Equal(t, results[StatCount], out.Values[0]) - SumFunc(ix, out) - assert.Equal(t, results[Sum], out.Values[0]) + Sum(ix, out) + assert.Equal(t, results[StatSum], out.Values[0]) - SumAbsFunc(ix, out) - assert.Equal(t, results[SumAbs], out.Values[0]) + SumAbs(ix, out) + assert.Equal(t, results[StatSumAbs], out.Values[0]) - ProdFunc(ix, out) - assert.Equal(t, results[Prod], out.Values[0]) + Prod(ix, out) + assert.Equal(t, results[StatProd], out.Values[0]) - MinFunc(ix, out) - assert.Equal(t, results[Min], out.Values[0]) + Min(ix, out) + assert.Equal(t, results[StatMin], out.Values[0]) - MaxFunc(ix, out) - assert.Equal(t, results[Max], out.Values[0]) + Max(ix, out) + assert.Equal(t, results[StatMax], out.Values[0]) - MinAbsFunc(ix, out) - assert.Equal(t, results[MinAbs], out.Values[0]) + MinAbs(ix, out) + assert.Equal(t, results[StatMinAbs], out.Values[0]) - MaxAbsFunc(ix, out) - assert.Equal(t, results[MaxAbs], out.Values[0]) + MaxAbs(ix, out) + assert.Equal(t, results[StatMaxAbs], out.Values[0]) - MeanFunc(ix, out) - assert.Equal(t, results[Mean], out.Values[0]) + Mean(ix, out) + assert.Equal(t, results[StatMean], out.Values[0]) - VarFunc(ix, out) - assert.InDelta(t, results[Var], out.Values[0], tol) + Var(ix, out) + assert.InDelta(t, results[StatVar], out.Values[0], tol) - StdFunc(ix, out) - assert.InDelta(t, results[Std], out.Values[0], tol) + Std(ix, out) + assert.InDelta(t, results[StatStd], out.Values[0], tol) - SemFunc(ix, out) - assert.InDelta(t, results[Sem], out.Values[0], tol) + Sem(ix, out) + assert.InDelta(t, results[StatSem], out.Values[0], tol) - VarPopFunc(ix, out) - assert.InDelta(t, results[VarPop], out.Values[0], tol) + VarPop(ix, out) + assert.InDelta(t, results[StatVarPop], out.Values[0], tol) - StdPopFunc(ix, out) - assert.InDelta(t, results[StdPop], out.Values[0], tol) + StdPop(ix, out) + assert.InDelta(t, results[StatStdPop], out.Values[0], tol) - SemPopFunc(ix, out) - assert.InDelta(t, results[SemPop], out.Values[0], tol) + SemPop(ix, out) + assert.InDelta(t, results[StatSemPop], out.Values[0], tol) - SumSqFunc(ix, out) - assert.InDelta(t, results[SumSq], out.Values[0], tol) + SumSq(ix, out) + assert.InDelta(t, results[StatSumSq], out.Values[0], tol) - L2NormFunc(ix, out) - assert.InDelta(t, results[L2Norm], out.Values[0], tol) + L2Norm(ix, out) + assert.InDelta(t, results[StatL2Norm], out.Values[0], tol) - MedianFunc(ix, out) - assert.InDelta(t, results[Median], out.Values[0], tol) + Median(ix, out) + assert.InDelta(t, results[StatMedian], out.Values[0], tol) - Q1Func(ix, out) - assert.InDelta(t, results[Q1], out.Values[0], tol) + Q1(ix, out) + assert.InDelta(t, results[StatQ1], out.Values[0], tol) - Q3Func(ix, out) - assert.InDelta(t, results[Q3], out.Values[0], tol) + Q3(ix, out) + assert.InDelta(t, results[StatQ3], out.Values[0], tol) - for stat := Count; stat < StatsN; stat++ { - stat.Call(ix, out) + for stat := StatCount; stat < StatsN; stat++ { + err := stat.Call(ix, out) + assert.NoError(t, err) assert.InDelta(t, results[stat], out.Values[0], tol) } + err := tensor.Call("stats.Mean", ix, out) // ensure plain name is registered. + assert.NoError(t, err) + assert.InDelta(t, results[StatMean], out.Values[0], tol) } func TestFuncsInt(t *testing.T) { @@ -95,58 +99,58 @@ func TestFuncsInt(t *testing.T) { results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} - CountFunc(ix, out) - assert.Equal(t, results[Count], out.Values[0]) + Count(ix, out) + assert.Equal(t, results[StatCount], out.Values[0]) - SumFunc(ix, out) - assert.Equal(t, results[Sum], out.Values[0]) + Sum(ix, out) + assert.Equal(t, results[StatSum], out.Values[0]) - SumAbsFunc(ix, out) - assert.Equal(t, results[SumAbs], out.Values[0]) + SumAbs(ix, out) + assert.Equal(t, results[StatSumAbs], out.Values[0]) - ProdFunc(ix, out) - assert.Equal(t, results[Prod], out.Values[0]) + Prod(ix, out) + assert.Equal(t, results[StatProd], out.Values[0]) - MinFunc(ix, out) - assert.Equal(t, results[Min], out.Values[0]) + Min(ix, out) + assert.Equal(t, results[StatMin], out.Values[0]) - MaxFunc(ix, out) - assert.Equal(t, results[Max], out.Values[0]) + Max(ix, out) + assert.Equal(t, results[StatMax], out.Values[0]) - MinAbsFunc(ix, out) - assert.Equal(t, results[MinAbs], out.Values[0]) + MinAbs(ix, out) + assert.Equal(t, results[StatMinAbs], out.Values[0]) - MaxAbsFunc(ix, out) - assert.Equal(t, results[MaxAbs], out.Values[0]) + MaxAbs(ix, out) + assert.Equal(t, results[StatMaxAbs], out.Values[0]) - MeanFunc(ix, out) - assert.Equal(t, results[Mean], out.Values[0]) + Mean(ix, out) + assert.Equal(t, results[StatMean], out.Values[0]) - VarFunc(ix, out) - assert.Equal(t, results[Var], out.Values[0]) + Var(ix, out) + assert.Equal(t, results[StatVar], out.Values[0]) - StdFunc(ix, out) - assert.Equal(t, results[Std], out.Values[0]) + Std(ix, out) + assert.Equal(t, results[StatStd], out.Values[0]) - SemFunc(ix, out) - assert.Equal(t, results[Sem], out.Values[0]) + Sem(ix, out) + assert.Equal(t, results[StatSem], out.Values[0]) - VarPopFunc(ix, out) - assert.Equal(t, results[VarPop], out.Values[0]) + VarPop(ix, out) + assert.Equal(t, results[StatVarPop], out.Values[0]) - StdPopFunc(ix, out) - assert.Equal(t, results[StdPop], out.Values[0]) + StdPop(ix, out) + assert.Equal(t, results[StatStdPop], out.Values[0]) - SemPopFunc(ix, out) - assert.Equal(t, results[SemPop], out.Values[0]) + SemPop(ix, out) + assert.Equal(t, results[StatSemPop], out.Values[0]) - SumSqFunc(ix, out) - assert.Equal(t, results[SumSq], out.Values[0]) + SumSq(ix, out) + assert.Equal(t, results[StatSumSq], out.Values[0]) - L2NormFunc(ix, out) - assert.Equal(t, results[L2Norm], out.Values[0]) + L2Norm(ix, out) + assert.Equal(t, results[StatL2Norm], out.Values[0]) - for stat := Count; stat <= SemPop; stat++ { + for stat := StatCount; stat <= StatSemPop; stat++ { stat.Call(ix, out) assert.Equal(t, results[stat], out.Values[0]) } @@ -165,16 +169,16 @@ func TestFuncsCell(t *testing.T) { ix := tensor.NewRows(tsr) out := tensor.NewFloat32(20, 10) - CountFunc(ix, out) + Count(ix, out) nsub := out.Len() for i := range nsub { assert.Equal(t, 20.0, out.FloatRowCell(0, i)) } - MeanFunc(ix, out) + Mean(ix, out) for i := range nsub { assert.InDelta(t, vals[i], out.FloatRowCell(0, i), 1.0e-7) // lower tol, using float32 } - VarFunc(ix, out) + Var(ix, out) for i := range nsub { assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) } @@ -194,26 +198,26 @@ func TestNorm(t *testing.T) { assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) UnitNorm(oned, oneout) - MinFunc(oneout, mout) + Min(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - MaxFunc(oneout, mout) + Max(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) minv := tensor.NewFloat64Scalar(0) maxv := tensor.NewFloat64Scalar(1) Clamp(oned, minv, maxv, oneout) - MinFunc(oneout, mout) + Min(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - MaxFunc(oneout, mout) + Max(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) thr := tensor.NewFloat64Scalar(0.5) Binarize(oned, thr, oneout) - MinFunc(oneout, mout) + Min(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - MaxFunc(oneout, mout) + Max(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) } diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index f423198d36..333ac17076 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -68,13 +68,3 @@ func TestMath(t *testing.T) { } } } - -func TestOpsCall(t *testing.T) { - x := tensor.NewIntScalar(1) - y := tensor.NewIntScalar(4) - a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2)) - b := tensor.CallOut("Add", x, y) - c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y), tensor.CallOut("Mul", a, b)) - - assert.Equal(t, 14.0, c.Float1D(0)) -} diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index f858ff23b6..70ebce45de 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -11,6 +11,17 @@ import ( "github.com/stretchr/testify/assert" ) +func TestOpsCall(t *testing.T) { + x := tensor.NewIntScalar(1) + y := tensor.NewIntScalar(4) + + a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2)) + b := tensor.CallOut("Add", x, y) + c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y), tensor.CallOut("Mul", a, b)) + + assert.Equal(t, 14.0, c.Float1D(0)) +} + func TestOps(t *testing.T) { scalar := tensor.NewFloat64Scalar(-5.5) scb := scalar.Clone() diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go index 96b439de4a..4047cf4fe9 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -11,33 +11,36 @@ func init() { Symbols["cogentcore.org/core/tensor/stats/metric/metric"] = map[string]reflect.Value{ // function, constant and variable definitions "Abs": reflect.ValueOf(metric.Abs), - "AbsFunc": reflect.ValueOf(metric.AbsFunc), + "AsMetricFunc": reflect.ValueOf(metric.AsMetricFunc), "ClosestRow": reflect.ValueOf(metric.ClosestRow), "Correlation": reflect.ValueOf(metric.Correlation), - "CorrelationFunc": reflect.ValueOf(metric.CorrelationFunc), "CorrelationOut64": reflect.ValueOf(metric.CorrelationOut64), "Cosine": reflect.ValueOf(metric.Cosine), - "CosineFunc": reflect.ValueOf(metric.CosineFunc), "CosineOut64": reflect.ValueOf(metric.CosineOut64), "Covariance": reflect.ValueOf(metric.Covariance), - "CovarianceFunc": reflect.ValueOf(metric.CovarianceFunc), "CovarianceMatrix": reflect.ValueOf(metric.CovarianceMatrix), "CrossEntropy": reflect.ValueOf(metric.CrossEntropy), - "CrossEntropyFunc": reflect.ValueOf(metric.CrossEntropyFunc), "CrossMatrix": reflect.ValueOf(metric.CrossMatrix), "Euclidean": reflect.ValueOf(metric.Euclidean), "EuclideanBinTol": reflect.ValueOf(metric.EuclideanBinTol), - "EuclideanBinTolFunc": reflect.ValueOf(metric.EuclideanBinTolFunc), - "EuclideanFunc": reflect.ValueOf(metric.EuclideanFunc), "Hamming": reflect.ValueOf(metric.Hamming), - "HammingFunc": reflect.ValueOf(metric.HammingFunc), "InnerProduct": reflect.ValueOf(metric.InnerProduct), - "InnerProductFunc": reflect.ValueOf(metric.InnerProductFunc), "InvCorrelation": reflect.ValueOf(metric.InvCorrelation), - "InvCorrelationFunc": reflect.ValueOf(metric.InvCorrelationFunc), "InvCosine": reflect.ValueOf(metric.InvCosine), - "InvCosineFunc": reflect.ValueOf(metric.InvCosineFunc), "Matrix": reflect.ValueOf(metric.Matrix), + "MetricAbs": reflect.ValueOf(metric.MetricAbs), + "MetricCorrelation": reflect.ValueOf(metric.MetricCorrelation), + "MetricCosine": reflect.ValueOf(metric.MetricCosine), + "MetricCovariance": reflect.ValueOf(metric.MetricCovariance), + "MetricCrossEntropy": reflect.ValueOf(metric.MetricCrossEntropy), + "MetricEuclidean": reflect.ValueOf(metric.MetricEuclidean), + "MetricEuclideanBinTol": reflect.ValueOf(metric.MetricEuclideanBinTol), + "MetricHamming": reflect.ValueOf(metric.MetricHamming), + "MetricInnerProduct": reflect.ValueOf(metric.MetricInnerProduct), + "MetricInvCorrelation": reflect.ValueOf(metric.MetricInvCorrelation), + "MetricInvCosine": reflect.ValueOf(metric.MetricInvCosine), + "MetricSumSquares": reflect.ValueOf(metric.MetricSumSquares), + "MetricSumSquaresBinTol": reflect.ValueOf(metric.MetricSumSquaresBinTol), "MetricsN": reflect.ValueOf(metric.MetricsN), "MetricsValues": reflect.ValueOf(metric.MetricsValues), "NFunc": reflect.ValueOf(metric.NFunc), @@ -46,9 +49,7 @@ func init() { "SVD": reflect.ValueOf(metric.SVD), "SumSquares": reflect.ValueOf(metric.SumSquares), "SumSquaresBinTol": reflect.ValueOf(metric.SumSquaresBinTol), - "SumSquaresBinTolFunc": reflect.ValueOf(metric.SumSquaresBinTolFunc), "SumSquaresBinTolScaleOut64": reflect.ValueOf(metric.SumSquaresBinTolScaleOut64), - "SumSquaresFunc": reflect.ValueOf(metric.SumSquaresFunc), "SumSquaresOut64": reflect.ValueOf(metric.SumSquaresOut64), "SumSquaresScaleOut64": reflect.ValueOf(metric.SumSquaresScaleOut64), "TriangularLIndicies": reflect.ValueOf(metric.TriangularLIndicies), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index 22fec6eb93..2678c4bfae 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -10,10 +10,10 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/stats/stats/stats"] = map[string]reflect.Value{ // function, constant and variable definitions + "AsStatsFunc": reflect.ValueOf(stats.AsStatsFunc), "Binarize": reflect.ValueOf(stats.Binarize), "Clamp": reflect.ValueOf(stats.Clamp), "Count": reflect.ValueOf(stats.Count), - "CountFunc": reflect.ValueOf(stats.CountFunc), "CountOut64": reflect.ValueOf(stats.CountOut64), "Describe": reflect.ValueOf(stats.Describe), "DescribeTable": reflect.ValueOf(stats.DescribeTable), @@ -23,51 +23,54 @@ func init() { "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), "GroupStats": reflect.ValueOf(stats.GroupStats), "Groups": reflect.ValueOf(stats.Groups), - "L1Norm": reflect.ValueOf(stats.L1Norm), "L2Norm": reflect.ValueOf(stats.L2Norm), - "L2NormFunc": reflect.ValueOf(stats.L2NormFunc), "L2NormOut64": reflect.ValueOf(stats.L2NormOut64), "Max": reflect.ValueOf(stats.Max), "MaxAbs": reflect.ValueOf(stats.MaxAbs), - "MaxAbsFunc": reflect.ValueOf(stats.MaxAbsFunc), - "MaxFunc": reflect.ValueOf(stats.MaxFunc), "Mean": reflect.ValueOf(stats.Mean), - "MeanFunc": reflect.ValueOf(stats.MeanFunc), "MeanOut64": reflect.ValueOf(stats.MeanOut64), "Median": reflect.ValueOf(stats.Median), - "MedianFunc": reflect.ValueOf(stats.MedianFunc), "Min": reflect.ValueOf(stats.Min), "MinAbs": reflect.ValueOf(stats.MinAbs), - "MinAbsFunc": reflect.ValueOf(stats.MinAbsFunc), - "MinFunc": reflect.ValueOf(stats.MinFunc), "NFunc": reflect.ValueOf(stats.NFunc), "Prod": reflect.ValueOf(stats.Prod), - "ProdFunc": reflect.ValueOf(stats.ProdFunc), "Q1": reflect.ValueOf(stats.Q1), - "Q1Func": reflect.ValueOf(stats.Q1Func), "Q3": reflect.ValueOf(stats.Q3), - "Q3Func": reflect.ValueOf(stats.Q3Func), - "QuantilesFunc": reflect.ValueOf(stats.QuantilesFunc), + "Quantiles": reflect.ValueOf(stats.Quantiles), "Sem": reflect.ValueOf(stats.Sem), - "SemFunc": reflect.ValueOf(stats.SemFunc), "SemPop": reflect.ValueOf(stats.SemPop), - "SemPopFunc": reflect.ValueOf(stats.SemPopFunc), + "StatCount": reflect.ValueOf(stats.StatCount), + "StatL1Norm": reflect.ValueOf(stats.StatL1Norm), + "StatL2Norm": reflect.ValueOf(stats.StatL2Norm), + "StatMax": reflect.ValueOf(stats.StatMax), + "StatMaxAbs": reflect.ValueOf(stats.StatMaxAbs), + "StatMean": reflect.ValueOf(stats.StatMean), + "StatMedian": reflect.ValueOf(stats.StatMedian), + "StatMin": reflect.ValueOf(stats.StatMin), + "StatMinAbs": reflect.ValueOf(stats.StatMinAbs), + "StatProd": reflect.ValueOf(stats.StatProd), + "StatQ1": reflect.ValueOf(stats.StatQ1), + "StatQ3": reflect.ValueOf(stats.StatQ3), + "StatSem": reflect.ValueOf(stats.StatSem), + "StatSemPop": reflect.ValueOf(stats.StatSemPop), + "StatStd": reflect.ValueOf(stats.StatStd), + "StatStdPop": reflect.ValueOf(stats.StatStdPop), + "StatSum": reflect.ValueOf(stats.StatSum), + "StatSumAbs": reflect.ValueOf(stats.StatSumAbs), + "StatSumSq": reflect.ValueOf(stats.StatSumSq), + "StatVar": reflect.ValueOf(stats.StatVar), + "StatVarPop": reflect.ValueOf(stats.StatVarPop), "StatsN": reflect.ValueOf(stats.StatsN), "StatsValues": reflect.ValueOf(stats.StatsValues), "Std": reflect.ValueOf(stats.Std), - "StdFunc": reflect.ValueOf(stats.StdFunc), "StdOut64": reflect.ValueOf(stats.StdOut64), "StdPop": reflect.ValueOf(stats.StdPop), - "StdPopFunc": reflect.ValueOf(stats.StdPopFunc), "StripPackage": reflect.ValueOf(stats.StripPackage), "Sum": reflect.ValueOf(stats.Sum), "SumAbs": reflect.ValueOf(stats.SumAbs), - "SumAbsFunc": reflect.ValueOf(stats.SumAbsFunc), - "SumFunc": reflect.ValueOf(stats.SumFunc), "SumOut64": reflect.ValueOf(stats.SumOut64), "SumSq": reflect.ValueOf(stats.SumSq), "SumSqDevOut64": reflect.ValueOf(stats.SumSqDevOut64), - "SumSqFunc": reflect.ValueOf(stats.SumSqFunc), "SumSqOut64": reflect.ValueOf(stats.SumSqOut64), "SumSqScaleOut64": reflect.ValueOf(stats.SumSqScaleOut64), "TableGroupDescribe": reflect.ValueOf(stats.TableGroupDescribe), @@ -75,10 +78,8 @@ func init() { "TableGroups": reflect.ValueOf(stats.TableGroups), "UnitNorm": reflect.ValueOf(stats.UnitNorm), "Var": reflect.ValueOf(stats.Var), - "VarFunc": reflect.ValueOf(stats.VarFunc), "VarOut64": reflect.ValueOf(stats.VarOut64), "VarPop": reflect.ValueOf(stats.VarPop), - "VarPopFunc": reflect.ValueOf(stats.VarPopFunc), "VarPopOut64": reflect.ValueOf(stats.VarPopOut64), "Vec2inFunc": reflect.ValueOf(stats.Vec2inFunc), "Vec2outFunc": reflect.ValueOf(stats.Vec2outFunc), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 5049614e28..6bf47d9349 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -16,6 +16,7 @@ func init() { "AddFunc": reflect.ValueOf(tensor.AddFunc), "AddShapes": reflect.ValueOf(tensor.AddShapes), "AlignShapes": reflect.ValueOf(tensor.AlignShapes), + "AnyFirstArg": reflect.ValueOf(tensor.AnyFirstArg), "As1D": reflect.ValueOf(tensor.As1D), "AsFloat32Tensor": reflect.ValueOf(tensor.AsFloat32Tensor), "AsFloat64Scalar": reflect.ValueOf(tensor.AsFloat64Scalar), @@ -37,9 +38,9 @@ func init() { "BoolToInt": reflect.ValueOf(tensor.BoolToInt), "Calc": reflect.ValueOf(tensor.Calc), "Call": reflect.ValueOf(tensor.Call), + "CallAny": reflect.ValueOf(tensor.CallAny), "CallOut": reflect.ValueOf(tensor.CallOut), "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), - "CallString": reflect.ValueOf(tensor.CallString), "Cells1D": reflect.ValueOf(tensor.Cells1D), "CellsSize": reflect.ValueOf(tensor.CellsSize), "Clone": reflect.ValueOf(tensor.Clone), @@ -55,11 +56,13 @@ func init() { "Exclude": reflect.ValueOf(tensor.Exclude), "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "FuncByName": reflect.ValueOf(tensor.FuncByName), "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), "Include": reflect.ValueOf(tensor.Include), "IntToBool": reflect.ValueOf(tensor.IntToBool), "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), + "MustBeSameShape": reflect.ValueOf(tensor.MustBeSameShape), "MustBeValues": reflect.ValueOf(tensor.MustBeValues), "NFirstLen": reflect.ValueOf(tensor.NFirstLen), "NFirstRows": reflect.ValueOf(tensor.NFirstRows), @@ -120,7 +123,6 @@ func init() { "Space": reflect.ValueOf(tensor.Space), "Sprint": reflect.ValueOf(tensor.Sprint), "Stable": reflect.ValueOf(tensor.Stable), - "StringFirstArg": reflect.ValueOf(tensor.StringFirstArg), "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), "Tab": reflect.ValueOf(tensor.Tab), "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), From a9fb9164a30014f032cf1be10f44fb742df5b295 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 15:37:52 -0700 Subject: [PATCH 110/311] add FilterOptions struct instead of named bool constants, rename Stable -> StableSort constant; --- tensor/rows.go | 58 ++++++++++--------- tensor/stats/stats/group.go | 2 +- tensor/table/indexes.go | 11 ++-- tensor/typegen.go | 12 ++-- .../symbols/cogentcore_org-core-tensor.go | 41 ++++++------- 5 files changed, 62 insertions(+), 62 deletions(-) diff --git a/tensor/rows.go b/tensor/rows.go index c07dcdbf4c..835ff5384b 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -217,11 +217,11 @@ const ( // Descending specifies a descending sort direction for tensor Sort routines Descending = false - // Stable specifies using stable, original order-preserving sort, which is slower. - Stable = true + // StableSort specifies using stable, original order-preserving sort, which is slower. + StableSort = true - // Unstable specifies using faster but unstable sorting. - Unstable = false + // Unstable specifies using faster but unstable sorting. + UnstableSort = false ) // SortFunc sorts the row-wise indexes using given compare function. @@ -246,6 +246,8 @@ func (rw *Rows) SortIndexes() { sort.Ints(rw.Indexes) } +// CompareAscending is a sort compare function that reverses direction +// based on the ascending bool. func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { if ascending { return cmp.Compare(a, b) @@ -313,44 +315,44 @@ func (rw *Rows) Filter(filterer func(tsr Values, row int) bool) { } } -// Named arg values for FilterString -const ( - // Include means include matches - Include = false - // Exclude means exclude matches - Exclude = true - // Contains means the string only needs to contain the target string (see Equals) - Contains = true - // Equals means the string must equal the target string (see Contains) - Equals = false - // IgnoreCase means that differences in case are ignored in comparing strings - IgnoreCase = true - // UseCase means that case matters when comparing strings - UseCase = false -) +// FilterOptions are options to a Filter function +// determining how the string filter value is used for matching. +type FilterOptions struct { //types:add + + // Exclude means to exclude matches, + // with the default (false) being to include + Exclude bool + + // Contains means the string only needs to contain the target string, + // with the default (false) requiring a complete match to entire string. + Contains bool + + // IgnoreCase means that differences in case are ignored in comparing strings, + // with the default (false) using case. + IgnoreCase bool +} // FilterString filters the indexes using string values compared to given -// string. Includes rows with matching values unless exclude is set. -// If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use the named const args [Include], [Exclude], [Contains], [Equals], -// [IgnoreCase], [UseCase] for greater clarity. +// string. Includes rows with matching values unless the Exclude option is set. +// If Contains option is set, it only checks if row contains string; +// if IgnoreCase, ignores case, otherwise filtering is case sensitive. // Uses first cell of higher dimensional data. -func (rw *Rows) FilterString(str string, exclude, contains, ignoreCase bool) { //types:add +func (rw *Rows) FilterString(str string, opts FilterOptions) { //types:add lowstr := strings.ToLower(str) rw.Filter(func(tsr Values, row int) bool { val := tsr.StringRowCell(row, 0) has := false switch { - case contains && ignoreCase: + case opts.Contains && opts.IgnoreCase: has = strings.Contains(strings.ToLower(val), lowstr) - case contains: + case opts.Contains: has = strings.Contains(val, str) - case ignoreCase: + case opts.IgnoreCase: has = strings.EqualFold(val, str) default: has = (val == str) } - if exclude { + if opts.Exclude { return !has } return has diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 1ac70c70ed..94dfb3746b 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -98,7 +98,7 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) error { func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) error { dv := table.NewView(dt) // important for consistency across columns, to do full outer product sort first. - dv.SortColumns(tensor.Ascending, tensor.Stable, columns...) + dv.SortColumns(tensor.Ascending, tensor.StableSort, columns...) return Groups(dir, dv.ColumnList(columns...)...) } diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index 3fce2e8fff..de2021a8aa 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -196,19 +196,18 @@ func (dt *Table) Filter(filterer func(dt *Table, row int) bool) { } // FilterString filters the indexes using string values in column compared to given -// string. Includes rows with matching values unless exclude is set. -// If contains, only checks if row contains string; if ignoreCase, ignores case. -// Use the named const args [tensor.Include], [tensor.Exclude], [tensor.Contains], -// [tensor.Equals], [tensor.IgnoreCase], [tensor.UseCase] for greater clarity. +// string. Includes rows with matching values unless the Exclude option is set. +// If Contains option is set, it only checks if row contains string; +// if IgnoreCase, ignores case, otherwise filtering is case sensitive. // Uses first cell from higher dimensions. // Returns error if column name not found. -func (dt *Table) FilterString(columnName string, str string, exclude, contains, ignoreCase bool) error { //types:add +func (dt *Table) FilterString(columnName string, str string, opts tensor.FilterOptions) error { //types:add dt.IndexesNeeded() cl, err := dt.ColumnTry(columnName) if err != nil { return err } - cl.FilterString(str, exclude, contains, ignoreCase) + cl.FilterString(str, opts) dt.IndexesFromTensor(cl) return nil } diff --git a/tensor/typegen.go b/tensor/typegen.go index 35574646c5..87bb8ef925 100644 --- a/tensor/typegen.go +++ b/tensor/typegen.go @@ -6,10 +6,14 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed is a wrapper around another [Tensor] that provides a\nindexed view onto the Tensor provided by an [Int] tensor with\nindex coordinates into the source tensor. The innermost dimension\nsize of the indexes is equal to the number of dimensions in\nthe source tensor, and the remaining outer dimensions provide the\nshape for the [Indexed] tensor view.\nTo produce a new concrete [Values] that has raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Indexed.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Indexed", IDName: "indexed", Doc: "Indexed provides an arbitrarily indexed view onto another \"source\" [Tensor]\nwith each index value providing a full n-dimensional index into the source.\nThe shape of this view is determined by the shape of the [Indexed.Indexes]\ntensor up to the final innermost dimension, which holds the index values.\nThus the innermost dimension size of the indexes is equal to the number\nof dimensions in the source tensor. Given the essential role of the\nindexes in this view, it is not usable without the indexes.\nThis view is not memory-contiguous and does not support the [RowMajor]\ninterface or efficient access to inner-dimensional subspaces.\nTo produce a new concrete [Values] that has raw data actually\norganized according to the indexed order (i.e., the copy function\nof numpy), call [Indexed.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor source that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes is the list of indexes into the source tensor,\nwith the innermost dimension providing the index values\n(size = number of dimensions in the source tensor), and\nthe remaining outer dimensions determine the shape\nof this [Indexed] tensor view."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Masked", IDName: "masked", Doc: "Masked is a wrapper around another [Tensor] that provides a\nbit-masked view onto the Tensor defined by a [Bool] [Values]\ntensor with a matching shape. If the bool mask has a 'false'\nthen the corresponding value cannot be set and Float access returns\nNaN indicating missing data.\nTo produce a new [Values] tensor with only the 'true' cases,\n(i.e., the copy function of numpy), call [Masked.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are a masked view onto."}, {Name: "Mask", Doc: "Bool tensor with same shape as source tensor, providing mask."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Masked", IDName: "masked", Doc: "Masked is a filtering wrapper around another \"source\" [Tensor],\nthat provides a bit-masked view onto the Tensor defined by a [Bool] [Values]\ntensor with a matching shape. If the bool mask has a 'false'\nthen the corresponding value cannot be Set, and Float access returns\nNaN indicating missing data (other type access returns the zero value).\nA new Masked view defaults to a full transparent view of the source tensor.\nTo produce a new [Values] tensor with only the 'true' cases,\n(i.e., the copy function of numpy), call [Masked.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor source that we are a masked view onto."}, {Name: "Mask", Doc: "Bool tensor with same shape as source tensor, providing mask."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Rows", IDName: "rows", Doc: "Rows is a row-indexed wrapper around a [Values] [Tensor] that provides a\nspecific view onto the Tensor defined by the set of [Rows.Indexes],\nwhich apply to the outermost row dimension (with default row-major indexing).\nSorting and filtering a tensor only requires updating the indexes while\nleaving the underlying Tensor alone.\nUse [AsValues] to obtain a concrete [Values] representation with the current row\nsorting. Use the [Set]FloatRow[Cell] methods wherever possible,\nfor the most efficient and natural indirection through the indexes.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing", Doc: "ExcludeMissing deletes indexes where the values are missing, as indicated by NaN.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values compared to given\nstring. Includes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse the named const args [Include], [Exclude], [Contains], [Equals],\n[IgnoreCase], [UseCase] for greater clarity.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"str", "exclude", "contains", "ignoreCase"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nUse the Index() method for nil-aware logic."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Reshaped", IDName: "reshaped", Doc: "Reshaped is a reshaping wrapper around another \"source\" [Tensor],\nthat provides a length-preserving reshaped view onto the source Tensor.\nReshaping by adding new size=1 dimensions (via [NewAxis] value) is\noften important for properly aligning two tensors in a computationally\ncompatible manner; see the [AlignShapes] function.\n[Reshaped.AsValues] on this view returns a new [Values] with the view\nshape, calling [Clone] on the source tensor to get the values.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor source that we are a masked view onto."}, {Name: "Reshape", Doc: "Reshape is the effective shape we use for access.\nThis must have the same Len() as the source Tensor."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Sliced", IDName: "sliced", Doc: "Sliced is a fully indexed wrapper around another [Tensor] that provides a\nre-sliced view onto the Tensor defined by the set of [Sliced.Indexes],\nfor each dimension (must have at least 1 per dimension).\nThus, every dimension can be transformed in arbitrary ways relative\nto the original tensor. There is some additional cost for every\naccess operation associated with the additional indexed indirection.\nSee also [Rows] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case.\nTo produce a new concrete [Values] that has raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Sliced.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes automatically provides a full, sequential view of that\ndimension."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Rows", IDName: "rows", Doc: "Rows is a row-indexed wrapper view around a [Values] [Tensor] that allows\narbitrary row-wise ordering and filtering according to the [Rows.Indexes].\nSorting and filtering a tensor along this outermost row dimension only\nrequires updating the indexes while leaving the underlying Tensor alone.\nUnlike the more general [Sliced] view, Rows maintains memory contiguity\nfor the inner dimensions (\"cells\") within each row, and supports the [RowMajor]\ninterface, with the [Set]FloatRow[Cell] methods providing efficient access.\nUse [Rows.AsValues] to obtain a concrete [Values] representation with the\ncurrent row sorting.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "ExcludeMissing", Doc: "ExcludeMissing deletes indexes where the values are missing, as indicated by NaN.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values compared to given\nstring. Includes rows with matching values unless the Exclude option is set.\nIf Contains option is set, it only checks if row contains string;\nif IgnoreCase, ignores case, otherwise filtering is case sensitive.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"str", "opts"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Tensor, and to the indexes in this view", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor source that we are an indexed view onto.\nNote that this must be a concrete [Values] tensor, to enable efficient\n[RowMajor] access and subspace functions."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nUse the [Rows.RowIndex] method for nil-aware logic."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.FilterOptions", IDName: "filter-options", Doc: "FilterOptions are options to a Filter function\ndetermining how the string filter value is used for matching.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Exclude", Doc: "Exclude means to exclude matches,\nwith the default (false) being to include"}, {Name: "Contains", Doc: "Contains means the string only needs to contain the target string,\nwith the default (false) requiring a complete match to entire string."}, {Name: "IgnoreCase", Doc: "IgnoreCase means that differences in case are ignored in comparing strings,\nwith the default (false) using case."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor.Sliced", IDName: "sliced", Doc: "Sliced provides a re-sliced view onto another \"source\" [Tensor],\ndefined by a set of [Sliced.Indexes] for each dimension (must have\nat least 1 index per dimension to avoid a null view).\nThus, each dimension can be transformed in arbitrary ways relative\nto the original tensor (filtered subsets, reversals, sorting, etc).\nThis view is not memory-contiguous and does not support the [RowMajor]\ninterface or efficient access to inner-dimensional subspaces.\nA new Sliced view defaults to a full transparent view of the source tensor.\nThere is additional cost for every access operation associated with the\nindexed indirection, and access is always via the full n-dimensional indexes.\nSee also [Rows] for a version that only indexes the outermost row dimension,\nwhich is much more efficient for this common use-case, and does support [RowMajor].\nTo produce a new concrete [Values] that has raw data actually organized according\nto the indexed order (i.e., the copy function of numpy), call [Sliced.AsValues].", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets all Indexes to nil, resulting in full sequential access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "Tensor", Doc: "Tensor source that we are an indexed view onto."}, {Name: "Indexes", Doc: "Indexes are the indexes for each dimension, with dimensions as the outer\nslice (enforced to be the same length as the NumDims of the source Tensor),\nand a list of dimension index values (within range of DimSize(d)).\nA nil list of indexes for a dimension automatically provides a full,\nsequential view of that dimension."}}}) diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 6bf47d9349..8e50a03e6a 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -46,20 +46,15 @@ func init() { "Clone": reflect.ValueOf(tensor.Clone), "ColumnMajorStrides": reflect.ValueOf(tensor.ColumnMajorStrides), "Comma": reflect.ValueOf(tensor.Comma), - "Contains": reflect.ValueOf(tensor.Contains), "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), "DelimsN": reflect.ValueOf(tensor.DelimsN), "DelimsValues": reflect.ValueOf(tensor.DelimsValues), "Descending": reflect.ValueOf(tensor.Descending), "Detect": reflect.ValueOf(tensor.Detect), - "Equals": reflect.ValueOf(tensor.Equals), - "Exclude": reflect.ValueOf(tensor.Exclude), "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), "Float64ToString": reflect.ValueOf(tensor.Float64ToString), "FuncByName": reflect.ValueOf(tensor.FuncByName), "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), - "IgnoreCase": reflect.ValueOf(tensor.IgnoreCase), - "Include": reflect.ValueOf(tensor.Include), "IntToBool": reflect.ValueOf(tensor.IntToBool), "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), "MustBeSameShape": reflect.ValueOf(tensor.MustBeSameShape), @@ -122,12 +117,11 @@ func init() { "ShapeNames": reflect.ValueOf(tensor.ShapeNames), "Space": reflect.ValueOf(tensor.Space), "Sprint": reflect.ValueOf(tensor.Sprint), - "Stable": reflect.ValueOf(tensor.Stable), + "StableSort": reflect.ValueOf(tensor.StableSort), "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), "Tab": reflect.ValueOf(tensor.Tab), "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), - "Unstable": reflect.ValueOf(tensor.Unstable), - "UseCase": reflect.ValueOf(tensor.UseCase), + "UnstableSort": reflect.ValueOf(tensor.UnstableSort), "Vectorize": reflect.ValueOf(tensor.Vectorize), "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), @@ -135,21 +129,22 @@ func init() { "WriteCSV": reflect.ValueOf(tensor.WriteCSV), // type definitions - "Bool": reflect.ValueOf((*tensor.Bool)(nil)), - "Delims": reflect.ValueOf((*tensor.Delims)(nil)), - "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), - "Func": reflect.ValueOf((*tensor.Func)(nil)), - "Indexed": reflect.ValueOf((*tensor.Indexed)(nil)), - "Masked": reflect.ValueOf((*tensor.Masked)(nil)), - "Reshaped": reflect.ValueOf((*tensor.Reshaped)(nil)), - "RowMajor": reflect.ValueOf((*tensor.RowMajor)(nil)), - "Rows": reflect.ValueOf((*tensor.Rows)(nil)), - "Shape": reflect.ValueOf((*tensor.Shape)(nil)), - "Slice": reflect.ValueOf((*tensor.Slice)(nil)), - "Sliced": reflect.ValueOf((*tensor.Sliced)(nil)), - "String": reflect.ValueOf((*tensor.String)(nil)), - "Tensor": reflect.ValueOf((*tensor.Tensor)(nil)), - "Values": reflect.ValueOf((*tensor.Values)(nil)), + "Bool": reflect.ValueOf((*tensor.Bool)(nil)), + "Delims": reflect.ValueOf((*tensor.Delims)(nil)), + "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), + "FilterOptions": reflect.ValueOf((*tensor.FilterOptions)(nil)), + "Func": reflect.ValueOf((*tensor.Func)(nil)), + "Indexed": reflect.ValueOf((*tensor.Indexed)(nil)), + "Masked": reflect.ValueOf((*tensor.Masked)(nil)), + "Reshaped": reflect.ValueOf((*tensor.Reshaped)(nil)), + "RowMajor": reflect.ValueOf((*tensor.RowMajor)(nil)), + "Rows": reflect.ValueOf((*tensor.Rows)(nil)), + "Shape": reflect.ValueOf((*tensor.Shape)(nil)), + "Slice": reflect.ValueOf((*tensor.Slice)(nil)), + "Sliced": reflect.ValueOf((*tensor.Sliced)(nil)), + "String": reflect.ValueOf((*tensor.String)(nil)), + "Tensor": reflect.ValueOf((*tensor.Tensor)(nil)), + "Values": reflect.ValueOf((*tensor.Values)(nil)), // interface wrapper definitions "_RowMajor": reflect.ValueOf((*_cogentcore_org_core_tensor_RowMajor)(nil)), From aa76c952f2d196838c025c4c991d2063c824064c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 15:49:36 -0700 Subject: [PATCH 111/311] fixes from filtering etc --- tensor/cmd/tablecat/tablecat.go | 2 +- tensor/examples/datafs-sim/sim.go | 6 +++--- tensor/stats/cluster/clust_test.go | 2 +- tensor/stats/cluster/cluster.go | 2 +- tensor/table/table_test.go | 2 +- tensor/tensor_test.go | 4 ++-- yaegicore/symbols/cogentcore_org-core-base-num.go | 11 +++++++++++ 7 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 yaegicore/symbols/cogentcore_org-core-base-num.go diff --git a/tensor/cmd/tablecat/tablecat.go b/tensor/cmd/tablecat/tablecat.go index 710eef58ba..ceae97c74b 100644 --- a/tensor/cmd/tablecat/tablecat.go +++ b/tensor/cmd/tablecat/tablecat.go @@ -159,7 +159,7 @@ func AvgByColumn(files []string, column string) { } cols = append(cols, dt.Columns.Keys[ci]) } - stats.TableGroupStats(dir, stats.Mean, dt, cols...) + stats.TableGroupStats(dir, stats.StatMean, dt, cols...) std := dir.Item("Stats") avgdt := std.GetDirTable(nil) // todo: has stat name slash tensor.SetPrecision(avgdt.Meta, LogPrec) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 1ef817f11e..c926d16644 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -55,7 +55,7 @@ func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { logd, _ := dir.Mkdir("Log") trial := ss.ConfigTrialLog(logd) - ss.ConfigAggLog(logd, "Epoch", trial, stats.Mean, stats.Sem, stats.Min) + ss.ConfigAggLog(logd, "Epoch", trial, stats.StatMean, stats.StatSem, stats.StatMin) return logd } @@ -157,8 +157,8 @@ func (ss *Sim) Run() { dir, _ := ss.Logs.Mkdir("Stats") stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") sts := []string{"SSE", "AvgSSE", "TrlErr"} - stats.TableGroupStats(dir, stats.Mean, alldt, sts...) - stats.TableGroupStats(dir, stats.Sem, alldt, sts...) + stats.TableGroupStats(dir, stats.StatMean, alldt, sts...) + stats.TableGroupStats(dir, stats.StatSem, alldt, sts...) } diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index fb58983a7d..47222c170b 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -35,7 +35,7 @@ func TestClust(t *testing.T) { } in := dt.Column("Input") out := tensor.NewFloat64() - metric.Matrix(metric.Euclidean.FuncName(), in, out) + metric.Matrix(metric.Euclidean, in, out) cl := Cluster(Avg.String(), out, dt.Column("Name")) diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index c61797f429..274ca79190 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -114,7 +114,7 @@ func InitAllLeaves(ntot int) *Node { func Glom(root *Node, funcName string, dmat tensor.Tensor) *Node { ntot := dmat.DimSize(0) // number of leaves mout := tensor.NewFloat64Scalar(0) - stats.MaxFunc(tensor.As1D(dmat), mout) + stats.Max(tensor.As1D(dmat), mout) maxd := mout.Float1D(0) // indexes in each group aidx := make([]int, ntot) diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index 70ab02825f..a666921bff 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -83,7 +83,7 @@ func TestAppendRowsEtc(t *testing.T) { assert.Equal(t, []int{2, 5, 8, 11, 1, 4, 7, 10, 0, 3, 6, 9}, dt.Indexes) dt.Sequential() - dt.FilterString("Int", "1", tensor.Include, tensor.Contains, tensor.IgnoreCase) + dt.FilterString("Int", "1", tensor.FilterOptions{Contains: true, IgnoreCase: true}) assert.Equal(t, []int{1, 4, 7, 10}, dt.Indexes) dt.Sequential() diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index ab02c6c6bb..3331c53991 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -272,11 +272,11 @@ func TestSortFilter(t *testing.T) { assert.Equal(t, []int{4, 3, 2, 1, 0}, tsr.Indexes) tsr.Sequential() - tsr.FilterString("1", Include, Equals, UseCase) + tsr.FilterString("1", FilterOptions{}) assert.Equal(t, []int{1}, tsr.Indexes) tsr.Sequential() - tsr.FilterString("1", Exclude, Equals, UseCase) + tsr.FilterString("1", FilterOptions{Exclude: true}) assert.Equal(t, []int{0, 2, 3, 4}, tsr.Indexes) } diff --git a/yaegicore/symbols/cogentcore_org-core-base-num.go b/yaegicore/symbols/cogentcore_org-core-base-num.go new file mode 100644 index 0000000000..765ec2c6c0 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-base-num.go @@ -0,0 +1,11 @@ +// Code generated by 'yaegi extract cogentcore.org/core/base/num'. DO NOT EDIT. + +package symbols + +import ( + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/base/num/num"] = map[string]reflect.Value{} +} From c68e5f0517c0eb95b4c5b986c87446c377194e46 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 17:28:29 -0700 Subject: [PATCH 112/311] correctly parsing a basic literal tensor indexing expression --- goal/transpile/math.go | 22 +++++++++++++++++++++- goal/transpile/parser.go | 12 ++++++++++++ goal/transpile/transpile_test.go | 15 ++++++++------- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 362c61172c..cc979e5b02 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -24,7 +24,7 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { // fmt.Println(str) mp := mathParse{toks: toks, code: code} - mods := AllErrors // | Trace + mods := AllErrors | Trace if fullLine { stmts, err := ParseLine(str, mods) @@ -175,6 +175,9 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.TypeAssertExpr: + case *ast.IndexExpr: + mp.indexExpr(x) + case *ast.IndexListExpr: if x.X == nil { // array literal mp.arrayLiteral(x) @@ -311,6 +314,23 @@ func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { // fmt.Println("slice expr", se) } +func (mp *mathParse) indexExpr(il *ast.IndexExpr) { + fmt.Println("index expr", il) + + if iil, ok := il.Index.(*ast.IndexListExpr); ok { + // todo: need to analyze and see what kind of expr it is + mp.expr(il.X) + mp.out.Add(token.PERIOD) + // note: we do not know what type to use, so use float + mp.out.Add(token.IDENT, "Float") + mp.addToken(token.LPAREN) // replaces [ + mp.goLiteral = true + mp.exprList(iil.Indices) + mp.goLiteral = true + mp.addToken(token.RPAREN) // replaces ] + } +} + func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { kind := inferKindExprList(il.Indices) if kind == token.ILLEGAL { diff --git a/goal/transpile/parser.go b/goal/transpile/parser.go index 7320ffc128..086e5d87b4 100644 --- a/goal/transpile/parser.go +++ b/goal/transpile/parser.go @@ -1622,6 +1622,16 @@ func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { Rbrack: rbrack, } } + + ix := p.parseArrayList(lbrack) + return &ast.IndexExpr{ + X: x, + Lbrack: lbrack, + Index: ix, + Rbrack: ix.Rbrack, + } + + /* math todo: p.exprLev++ const N = 3 // change the 3 to 2 to disable 3-index slices @@ -1685,6 +1695,8 @@ func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { // instance expression return nil // typeparams.PackIndexExpr(x, lbrack, args, rbrack) + */ + } func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 839d13ff19..06ada59b4a 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -200,13 +200,14 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + // {"# x := 1", `x := tensor.NewIntScalar(1)`}, + // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a[1,2]", `a.Float(1, 2)`}, } st := NewState() From c50733cb0ae2a1875d571e1e17f201f078fd44d6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 23 Sep 2024 23:32:48 -0700 Subject: [PATCH 113/311] tensor sliced processes int, slice, elipses --- goal/README.md | 48 ++++++++++++++++++-------------- goal/transpile/math.go | 1 + goal/transpile/transpile_test.go | 1 + tensor/sliced.go | 35 ++++++++++++++++++----- tensor/slices.go | 6 ++++ tensor/slices_test.go | 47 +++++++++++++++++++++++++++++++ tensor/stats/metric/matrix.go | 6 ++-- 7 files changed, 113 insertions(+), 31 deletions(-) diff --git a/goal/README.md b/goal/README.md index 285030c0db..db52c639af 100644 --- a/goal/README.md +++ b/goal/README.md @@ -1,16 +1,16 @@ # Goal: Go augmented language -_Goal_ is an augmented version of the _Go_ language, which combines the best parts of _Go_, `bash`, and Python, to provide and integrated shell and numerical expression processing experience, which can be combined with the [yaegi](https://github.com/traefik/yaegi) interpreter to provide an interactive "REPL" (read, evaluate, print loop). +Goal is an augmented version of the Go language, which combines the best parts of Go, `bash`, and Python, to provide and integrated shell and numerical expression processing experience, which can be combined with the [yaegi](https://github.com/traefik/yaegi) interpreter to provide an interactive "REPL" (read, evaluate, print loop). -_Goal_ transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: +Goal transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: * Shell scripting, where you want to be able to directly call other executable programs with arguments, without having to navigate all the complexity of the standard [os.exec](https://pkg.go.dev/os/exec) package. * Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and need extended indexing and slicing expressions. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. -The main goal of _Goal_ is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where the Go language has been a barrier. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, _Goal_ retains in its _Go_ foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. +The main goal of Goal is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where the Go language has been a barrier. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, Goal retains in its Go foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. -For the shell scripting aspect of _Goal_, the simple idea is that each line of code is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below). If a line starts off with something like `ls -la...` then it is clear that it is not valid Go code, and it is therefore processed as a shell command. +For the shell scripting aspect of Goal, the simple idea is that each line of code is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below). If a line starts off with something like `ls -la...` then it is clear that it is not valid Go code, and it is therefore processed as a shell command. You can intermix Go within a shell line by wrapping an expression with `{ }` braces, and a Go expression can contain shell code by using `$`. Here's an example: ```go @@ -37,7 +37,7 @@ for _, x := range #[1,2,3]# { } ``` -In general, the math mode syntax in _Goal_ is designed to be as compatible with Python NumPy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a _Goal_ math expression are [tensors](../tensor), which can represent everything from a scalar to an n-dimenstional tensor. These are called an "ndarray" in NumPy terms. +In general, the math mode syntax in Goal is designed to be as compatible with Python NumPy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a Goal math expression are [tensors](../tensor), which can represent everything from a scalar to an n-dimenstional tensor. These are called an "ndarray" in NumPy terms. The rationale and mnemonics for using `$` and `#` are as follows: @@ -49,7 +49,7 @@ The rationale and mnemonics for using `$` and `#` are as follows: # Examples -Here are a few useful examples of _Goal_ code: +Here are a few useful examples of Goal code: You can easily perform handy duration and data size formatting: @@ -221,20 +221,22 @@ TODO: update aboven # Math mode -In general, _Goal_ is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. +In general, Goal is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. -All elements of a _Goal_ math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that _Goal_ does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately. +All elements of a Goal math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that Goal does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately. -The _view_ versions of `Tensor` include `tensor.Sliced`, `tensor.Rows`, `tensor.Masked`, and `tensor.Indexed`, each of which wraps around another "source" `Tensor`, and provides its own way of accessing the underlying data: +The _view_ versions of `Tensor` include `Sliced`, `Reshaped`, `Masked`, `Indexed`, and `Rows`, each of which wraps around another "source" `Tensor`, and provides its own way of accessing the underlying data: * `Sliced` has an arbitrary set of indexes for each dimension, so access to values along that dimension go through the indexes. Thus, you could reverse the order of the columns (dimension 1), or only operate on a subset of them. -* `Rows` is an optimized version of `Sliced` with indexes only for the first, outermost, _row_ dimension, which - * `Masked` has a `tensor.Bool` tensor that filters access to the underlying source tensor through a mask: anywhere the bool value is `false`, the corresponding source value is not settable, and returns `NaN` (missing value) when accessed. * `Indexed` uses a tensor of indexes where the final, innermost dimension is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. + The current NumPy version of indexed is rather complex and difficult for many people to understand, as articulated in this [NEP 21 proposal](https://numpy.org/neps/nep-0021-advanced-indexing.html). We probably want to fix. + +* `Rows` is an optimized version of `Sliced` with indexes only for the first, outermost, _row_ dimension. + Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) | Goal | Python | MATLAB | Notes | @@ -243,8 +245,8 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | `len(a)` or `a.len` or `size(a)` or `a.size` | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | | `shape(a)` or `a.shape` | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | | `a.shape[n-1]` | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of tensor `a` | -| `a.reshape([10, 2])` or `a.reshape(10, 2)` or `reshape(a, [10, 2])` | `a.reshape(10, 2)` or `np.reshape(a, 10, 2` | `reshape(a,10,2)` | set the shape of `a`, preserving existing values (and adding zeros if larger, in _Goal_) | -| | `y = x.flatten()` | `y=x(:)` | turn tensor into vector (forces a copy in NumPy). | +| `tensor.NewReshaped(a, 10, 2)` or `a.reshape([10, 2])` or `reshape(a, [10, 2])` or: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major. | +| `tensor.Clone(tensor.As1D(a))` or: | `y = x.flatten()` | `y=x(:)` | turn tensor into a 1D vector (forces a copy) |. | | | | | **Construction** | | | | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | @@ -269,22 +271,22 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | | | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | | | | | -| **Access and Slicing** | | | -| (returns a _view_, changes modify original) | (returns a _reference_, changes modify original) | (returns a _copy_) | | -| `y = x.copy()` or `y = x.Clone()` | `y = x.copy()` | `y=x` | `y=x` just assigns `y` to point to `x`, so that changes to `y` also change `x`; need to make a `copy` to get distinct values | -| `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | -| | `a[-1]` | `a(end)` | access last element | +| **Basic Indexing** | | | | | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | +| | `a[-1]` | `a(end)` | access last element | | | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | | | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | | | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | | | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | -| | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | | | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | | | `a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | | | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | | | | | -| **Boolean Tensors** | | | +| **Advanced Indexing** | | | +| if indexes are themselves an array | then advanced indexing takes place | | indexes are parallel lists of dimension coordinates; not the clearest | +| | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | +| | | | +| **Boolean Tensors and Indexing** | | | | (bool tensor of same shape can filter access to other tensor) | | | | `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | @@ -305,6 +307,10 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the tensor `a` as tensor `b` with rows sorted by the first column | | | `np.unique(a)` | `unique(a)` | a vector of unique values in tensor `a` | | | | | +| **Copy vs. View** | | | +| `y = x.copy()` or `y = x.Clone()` | `y = x.copy()` | `y=x` | `y=x` just assigns `y` to point to `x`, so that changes to `y` also change `x`; need to make a `copy` to get distinct values | +| `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | +| | | | | **Math** | | | | | `a * b` | `a .* b` | element-wise multiply | | | `a/b` | `a./b` | element-wise divide | @@ -332,7 +338,7 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `P,L,U = linalg.lu(a)` where `a == P@L@U` | `[L,U,P]=lu(a)` where `a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | | | | | | **Statistics** | | | -| `a.max()` or `max(a)` or `stats.Max(a)` | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of `a`, _Goal_ always ignores `NaN` as missing data | +| `a.max()` or `max(a)` or `stats.Max(a)` | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of `a`, Goal always ignores `NaN` as missing data | | | `a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | | | `a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | | | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index cc979e5b02..217ddd54e9 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -319,6 +319,7 @@ func (mp *mathParse) indexExpr(il *ast.IndexExpr) { if iil, ok := il.Index.(*ast.IndexListExpr); ok { // todo: need to analyze and see what kind of expr it is + mp.out.Add(token.IDENT, "token.NewSliced") mp.expr(il.X) mp.out.Add(token.PERIOD) // note: we do not know what type to use, so use float diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 06ada59b4a..45e8f170a7 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -207,6 +207,7 @@ func TestMath(t *testing.T) { // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a[1,2]", `tensor.NewSliced(a, 1, 2)`}, {"# a[1,2]", `a.Float(1, 2)`}, } diff --git a/tensor/sliced.go b/tensor/sliced.go index 3b5a9d9773..94cec44b5f 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -52,18 +52,39 @@ func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { } // NewSliced returns a new [Sliced] view of given tensor, -// with given slices for each dimension (none / nil = sequential). -// Any dimensions without indexes default to nil = full sequential view. -func NewSliced(tsr Tensor, sls ...Slice) *Sliced { +// with given slice expressions for each dimension, which can be: +// - an integer, indicating a specific index value along that dimension. +// - a [Slice] object expressing a range of indexes. +// - [Elipses] which acts as a kind of "spacer" to include all dimensions +// up to the next expression. +// - any remaining dimensions without indexes default to nil = full sequential view. +func NewSliced(tsr Tensor, sls ...any) *Sliced { ns := len(sls) if ns == 0 { return NewSlicedIndexes(tsr) } - ns = min(ns, tsr.NumDims()) - ixs := make([][]int, ns) + nd := tsr.NumDims() + ed := nd - ns // extra dimensions + ixs := make([][]int, nd) + ci := 0 for d := range ns { - sl := sls[d] - ixs[d] = sl.IntSlice(tsr.DimSize(d)) + s := sls[d] + switch x := s.(type) { + case int: + if x < 0 { + ixs[ci] = []int{tsr.DimSize(ci) + x} + } else { + ixs[ci] = []int{x} + } + case Slice: + ixs[ci] = x.IntSlice(tsr.DimSize(ci)) + case ElipsesType: + for range ed { + ixs[ci] = Slice{}.IntSlice(tsr.DimSize(ci)) + ci++ + } + } + ci++ } return NewSlicedIndexes(tsr, ixs...) } diff --git a/tensor/slices.go b/tensor/slices.go index 6a1b96b84d..e386fb76d3 100644 --- a/tensor/slices.go +++ b/tensor/slices.go @@ -4,6 +4,12 @@ package tensor +// ElipsesType is a special type for marking the Elipses in [Sliced] expressions. +type ElipsesType int + +// Elipses is used in [Sliced] expressions to stretch between elements. +const Elipses ElipsesType = 0 + // Slice represents a slice of index values, for extracting slices of data, // along a dimension of a given size, which is provided separately as an argument. // Uses standard 'for' loop logic with a Start and _exclusive_ Stop value, diff --git a/tensor/slices_test.go b/tensor/slices_test.go index bbfa14b47a..04ac27a411 100644 --- a/tensor/slices_test.go +++ b/tensor/slices_test.go @@ -61,3 +61,50 @@ func TestSlice(t *testing.T) { assert.Equal(t, []int{5, 2}, Slice{0, 0, -3}.IntSlice(6)) assert.Equal(t, []int{6, 3, 0}, Slice{0, 0, -3}.IntSlice(7)) } + +func TestSlicedExpr(t *testing.T) { + ft := NewFloat64(3, 4) + for y := range 3 { + for x := range 4 { + v := y*10 + x + ft.SetFloat(float64(v), y, x) + } + } + + res := `[3, 4] +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 +` + assert.Equal(t, res, ft.String()) + // fmt.Println(ft) + + res = `[1, 1] +[0]: 12 +` + sl := NewSliced(ft, 1, 2) + // fmt.Println(sl) + assert.Equal(t, res, sl.String()) + + res = `[1, 4] +[0]: 10 11 12 13 +` + sl = NewSliced(ft, 1) + assert.Equal(t, res, sl.String()) + + res = `[3, 1] +[0]: 2 +[1]: 12 +[2]: 22 +` + sl = NewSliced(ft, Elipses, 2) + assert.Equal(t, res, sl.String()) + + res = `[3, 4] +[0]: 3 2 1 0 +[1]: 13 12 11 10 +[2]: 23 22 21 20 +` + sl = NewSliced(ft, Elipses, Slice{Step: -1}) + assert.Equal(t, res, sl.String()) +} diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 2789e1a3ce..db20bacd4d 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -146,11 +146,11 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { func(idx int, tsr ...tensor.Tensor) { c := coords[idx] if c.X != curCoords.X { - av = tensor.NewSliced(tsr[0], tensor.Slice{}, tensor.Slice{Start: c.X, Stop: c.X + 1}) + av = tensor.NewSliced(tsr[0], tensor.Elipses, c.X) curCoords.X = c.X } if c.Y != curCoords.Y { - bv = tensor.NewSliced(tsr[0], tensor.Slice{}, tensor.Slice{Start: c.Y, Stop: c.Y + 1}) + bv = tensor.NewSliced(tsr[0], tensor.Elipses, c.Y) curCoords.Y = c.Y } mfun(av, bv, mout) @@ -244,7 +244,7 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) error { // This is typically done with results from SVD or PCA. func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) error { ci := int(colindex.Float1D(0)) - col := tensor.As1D(tensor.NewSliced(mtx, tensor.Slice{}, tensor.Slice{Start: ci, Stop: ci + 1})) + col := tensor.As1D(tensor.NewSliced(mtx, tensor.Slice{}, ci)) // fmt.Println(mtx.String(), col.String()) rows, cells := vec.Shape().RowCellSize() if rows > 0 && cells > 0 { From 3fb463e8b2d7899e13c436592a6af1e7187cda18 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 24 Sep 2024 03:36:04 -0700 Subject: [PATCH 114/311] full slice parsing working --- goal/README.md | 6 +- goal/transpile/math.go | 58 +++++++-- goal/transpile/parser.go | 122 +++++++++++------- goal/transpile/transpile_test.go | 26 ++-- tensor/slices.go | 17 ++- .../symbols/cogentcore_org-core-tensor.go | 2 + 6 files changed, 159 insertions(+), 72 deletions(-) diff --git a/goal/README.md b/goal/README.md index db52c639af..ce39543aee 100644 --- a/goal/README.md +++ b/goal/README.md @@ -262,11 +262,11 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | | | | | | **Ranges and Grids** [numpy](https://numpy.org/doc/stable/user/how-to-partition.html) | | | -| | `np.arange(1., 11.) or np.r_[1.:11.] or np.r_[1:10:10j]` | `1:10` | create an increasing vector | -| | `np.arange(10.) or np.r_[:10.] or np.r_[:9:10j]` | `0:9` | create an increasing vector | +| | `np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector | +| | `np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector | | | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | | | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | -| | `np.mgrid[0:9.,0:6.] or np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | +| | `np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | | | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | | | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | | | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 217ddd54e9..7803f3f98b 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -24,7 +24,7 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { // fmt.Println(str) mp := mathParse{toks: toks, code: code} - mods := AllErrors | Trace + mods := AllErrors // | Trace if fullLine { stmts, err := ParseLine(str, mods) @@ -160,6 +160,9 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.Ident: mp.ident(x) + case *ast.UnaryExpr: + mp.unaryExpr(x) + case *ast.BinaryExpr: mp.binaryExpr(x) @@ -186,8 +189,7 @@ func (mp *mathParse) expr(ex ast.Expr) { } case *ast.SliceExpr: - // mp.sliceExpr(x) - // todo: probably this is subsumed in indexListExpr + mp.sliceExpr(x) case *ast.CallExpr: mp.callExpr(x) @@ -241,6 +243,11 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { mp.out.Add(token.RPAREN) } +func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { + mp.addToken(ex.Op) + mp.expr(ex.X) +} + func (mp *mathParse) basicLit(lit *ast.BasicLit) { if mp.goLiteral { mp.out.Add(lit.Kind, lit.Value) @@ -315,16 +322,12 @@ func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { } func (mp *mathParse) indexExpr(il *ast.IndexExpr) { - fmt.Println("index expr", il) - if iil, ok := il.Index.(*ast.IndexListExpr); ok { // todo: need to analyze and see what kind of expr it is - mp.out.Add(token.IDENT, "token.NewSliced") + mp.out.Add(token.IDENT, "tensor.NewSliced") + mp.out.Add(token.LPAREN) mp.expr(il.X) - mp.out.Add(token.PERIOD) - // note: we do not know what type to use, so use float - mp.out.Add(token.IDENT, "Float") - mp.addToken(token.LPAREN) // replaces [ + mp.addToken(token.COMMA) // use the [ mp.goLiteral = true mp.exprList(iil.Indices) mp.goLiteral = true @@ -332,6 +335,33 @@ func (mp *mathParse) indexExpr(il *ast.IndexExpr) { } } +func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { + mp.out.Add(token.IDENT, "tensor.Slice") + mp.addToken(token.LBRACE) + prev := false + if se.Low != nil { + mp.out.Add(token.IDENT, "Start:") + mp.expr(se.Low) + prev = true + } + if se.High != nil { + if prev { + mp.out.Add(token.COMMA) + } + mp.out.Add(token.IDENT, "Stop:") + mp.expr(se.High) + prev = true + } + if se.Max != nil { + if prev { + mp.out.Add(token.COMMA) + } + mp.out.Add(token.IDENT, "Step:") + mp.expr(se.Max) + } + mp.addToken(token.RBRACE) +} + func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { kind := inferKindExprList(il.Indices) if kind == token.ILLEGAL { @@ -362,7 +392,9 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { } var numpyFuncs = map[string]funWrap{ - "zeros": {"tensor.NewFloat64", ""}, + "zeros": {"tensor.NewFloat64", ""}, + "arange": {"tensor.NewSliceInts", ""}, + "reshape": {"tensor.NewReshaped", ""}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { @@ -370,14 +402,16 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { if fw, ok := numpyFuncs[fnm.Name]; ok { // todo: wrap mp.out.Add(token.IDENT, fw.fun) + mp.idx++ } else { mp.out.Add(token.IDENT, fnm.Name) + mp.idx++ } } else { mp.expr(ex.Fun) } mp.addToken(token.LPAREN) - mp.goLiteral = true + mp.goLiteral = true // todo: need a stack for this. mp.exprList(ex.Args) mp.goLiteral = false // todo: ellipsis diff --git a/goal/transpile/parser.go b/goal/transpile/parser.go index 086e5d87b4..2de3f924ed 100644 --- a/goal/transpile/parser.go +++ b/goal/transpile/parser.go @@ -1545,8 +1545,13 @@ func (p *parser) parseOperand() ast.Expr { case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: x := &ast.BasicLit{ValuePos: p.pos, Kind: p.tok, Value: p.lit} + // fmt.Println("operand lit:", p.lit) p.next() - return x + if p.tok == token.COLON { + return p.parseSliceExpr(x) + } else { + return x + } case token.LPAREN: lparen := p.pos @@ -1559,6 +1564,10 @@ func (p *parser) parseOperand() ast.Expr { case token.FUNC: return p.parseFuncTypeOrLit() + + case token.COLON: + p.expect(token.COLON) + return p.parseSliceExpr(nil) } if typ := p.tryIdentOrType(); typ != nil { // do not consume trailing type parameters @@ -1630,20 +1639,25 @@ func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { Index: ix, Rbrack: ix.Rbrack, } +} - /* math todo: +func (p *parser) parseSliceExpr(ex ast.Expr) *ast.SliceExpr { + if p.trace { + defer un(trace(p, "parseSliceExpr")) + } + lbrack := p.pos p.exprLev++ const N = 3 // change the 3 to 2 to disable 3-index slices - var args []ast.Expr var index [N]ast.Expr + index[0] = ex var colons [N - 1]token.Pos - if p.tok != token.COLON { - // We can't know if we have an index expression or a type instantiation; - // so even if we see a (named) type we are not going to be in type context. - index[0] = p.parseRhs() - } ncolons := 0 + if ex == nil { + ncolons++ + } + var rpos token.Pos + // fmt.Println(ncolons, p.tok) switch p.tok { case token.COLON: // slice expression @@ -1651,52 +1665,72 @@ func (p *parser) parseIndexOrSliceOrInstance(x ast.Expr) ast.Expr { colons[ncolons] = p.pos ncolons++ p.next() - if p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF { - index[ncolons] = p.parseRhs() + if p.tok != token.COMMA && p.tok != token.COLON && p.tok != token.RBRACK && p.tok != token.EOF { + ix := p.parseRhs() + if se, ok := ix.(*ast.SliceExpr); ok { + index[ncolons] = se.Low + if ncolons == 1 && se.High != nil { + ncolons++ + index[ncolons] = se.High + } + // fmt.Printf("nc: %d low: %#v hi: %#v max: %#v\n", ncolons, se.Low, se.High, se.Max) + } else { + // fmt.Printf("nc: %d low: %#v\n", ncolons, ix) + if _, ok := ix.(*ast.BadExpr); !ok { + index[ncolons] = ix + } + } + // } else { + // fmt.Println(ncolons, "else") } } case token.COMMA: + rpos = p.pos // expect(token.COMMA) + case token.RBRACK: + rpos = p.pos // expect(token.RBRACK) // instance expression - args = append(args, index[0]) - for p.tok == token.COMMA { - p.next() - if p.tok != token.RBRACK && p.tok != token.EOF { - args = append(args, p.parseType()) - } - } + // args = append(args, index[0]) + // for p.tok == token.COMMA { + // p.next() + // if p.tok != token.RBRACK && p.tok != token.EOF { + // args = append(args, p.parseType()) + // } + // } + default: + ix := p.parseRhs() + // fmt.Printf("nc: %d ix: %#v\n", ncolons, ix) + index[ncolons] = ix } p.exprLev-- - rbrack := p.expect(token.RBRACK) - - if ncolons > 0 { - // slice expression - slice3 := false - if ncolons == 2 { - slice3 = true - // Check presence of middle and final index here rather than during type-checking - // to prevent erroneous programs from passing through gofmt (was go.dev/issue/7305). - if index[1] == nil { - p.error(colons[0], "middle index required in 3-index slice") - index[1] = &ast.BadExpr{From: colons[0] + 1, To: colons[1]} - } - if index[2] == nil { - p.error(colons[1], "final index required in 3-index slice") - index[2] = &ast.BadExpr{From: colons[1] + 1, To: rbrack} - } - } - return &ast.SliceExpr{X: x, Lbrack: lbrack, Low: index[0], High: index[1], Max: index[2], Slice3: slice3, Rbrack: rbrack} - } - - if len(args) == 0 { - // index expression - return &ast.IndexExpr{X: x, Lbrack: lbrack, Index: index[0], Rbrack: rbrack} - } + // rbrack := p.expect(token.RBRACK) + + // slice expression + slice3 := false + if ncolons == 2 { + slice3 = true + // Check presence of middle and final index here rather than during type-checking + // to prevent erroneous programs from passing through gofmt (was go.dev/issue/7305). + // if index[1] == nil { + // p.error(colons[0], "middle index required in 3-index slice") + // index[1] = &ast.BadExpr{From: colons[0] + 1, To: colons[1]} + // } + // if index[2] == nil { + // p.error(colons[1], "final index required in 3-index slice") + // index[2] = &ast.BadExpr{From: colons[1] + 1} // , To: rbrack + // } + } + se := &ast.SliceExpr{Lbrack: lbrack, Low: index[0], High: index[1], Max: index[2], Slice3: slice3, Rbrack: rpos} + // fmt.Printf("final: %#v\n", se) + return se + // + // if len(args) == 0 { + // // index expression + // return &ast.IndexExpr{X: x, Lbrack: lbrack, Index: index[0], Rbrack: rbrack} + // } // instance expression return nil // typeparams.PackIndexExpr(x, lbrack, args, rbrack) - */ - } func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 45e8f170a7..71eb9214e3 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -200,15 +200,23 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - // {"# x := 1", `x := tensor.NewIntScalar(1)`}, - // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - {"# a[1,2]", `tensor.NewSliced(a, 1, 2)`}, - {"# a[1,2]", `a.Float(1, 2)`}, + {"# x := 1", `x := tensor.NewIntScalar(1)`}, + {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + {"# a := reshape(x, 6, 6)", `a := tensor.NewReshaped(x, 6, 6)`}, + {"# a := reshape(arange(36), 6, 6)", `a := tensor.NewReshaped(tensor.NewSliceInts(36), 6, 6)`}, + {"# a[1, 2]", `tensor.NewSliced(a, 1, 2)`}, + {"# a[:, 2]", `tensor.NewSliced(a, tensor.Slice { } , 2)`}, + {"# a[1:3:1, 2]", `tensor.NewSliced(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + {"# a[::-1, 2]", `tensor.NewSliced(a, tensor.Slice { Step: - 1 } , 2)`}, + {"# a[:3, 2]", `tensor.NewSliced(a, tensor.Slice { Stop:3 } , 2)`}, + {"# a[2:, 2]", `tensor.NewSliced(a, tensor.Slice { Start:2 } , 2)`}, } st := NewState() diff --git a/tensor/slices.go b/tensor/slices.go index e386fb76d3..3e437e7ea8 100644 --- a/tensor/slices.go +++ b/tensor/slices.go @@ -145,9 +145,18 @@ func (sl Slice) IntTensor(size int) *Int { return tsr } -// NewSliceInts returns a new [Int] [Tensor] with given [Slice] start, stop, step values, -// for given dimension size. -func NewSliceInts(size, start, stop, step int) *Int { - sl := NewSlice(start, stop, step) +// NewSliceInts returns a new [Int] [Tensor] with given [Slice] +// start, stop, step values (optional), for given dimension size. +func NewSliceInts(size int, svals ...int) *Int { + sl := Slice{} + if len(svals) > 0 { + sl.Start = svals[0] + } + if len(svals) > 1 { + sl.Stop = svals[1] + } + if len(svals) > 2 { + sl.Step = svals[2] + } return sl.IntTensor(size) } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 8e50a03e6a..8766751f04 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -51,6 +51,7 @@ func init() { "DelimsValues": reflect.ValueOf(tensor.DelimsValues), "Descending": reflect.ValueOf(tensor.Descending), "Detect": reflect.ValueOf(tensor.Detect), + "Elipses": reflect.ValueOf(tensor.Elipses), "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), "Float64ToString": reflect.ValueOf(tensor.Float64ToString), "FuncByName": reflect.ValueOf(tensor.FuncByName), @@ -131,6 +132,7 @@ func init() { // type definitions "Bool": reflect.ValueOf((*tensor.Bool)(nil)), "Delims": reflect.ValueOf((*tensor.Delims)(nil)), + "ElipsesType": reflect.ValueOf((*tensor.ElipsesType)(nil)), "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), "FilterOptions": reflect.ValueOf((*tensor.FilterOptions)(nil)), "Func": reflect.ValueOf((*tensor.Func)(nil)), From 56aa0cbad8cf95ce572ec14e96391fc836cfe844 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 24 Sep 2024 11:59:42 -0700 Subject: [PATCH 115/311] added NewAxis and FullAxis in NewSliced, which now auto-handles the reshaped -- full basic NumPy basic slicing spec handled. --- tensor/enumgen.go | 43 ++++++++++++++++++++++++++ tensor/reshaped.go | 4 --- tensor/sliced.go | 57 +++++++++++++++++++++++++++++------ tensor/slices.go | 22 +++++++++++--- tensor/slices_test.go | 32 ++++++++++++++------ tensor/stats/metric/matrix.go | 4 +-- tensor/tensor_test.go | 8 ++--- 7 files changed, 137 insertions(+), 33 deletions(-) diff --git a/tensor/enumgen.go b/tensor/enumgen.go index eaeb7bfb3b..86353996ec 100644 --- a/tensor/enumgen.go +++ b/tensor/enumgen.go @@ -44,3 +44,46 @@ func (i Delims) MarshalText() ([]byte, error) { return []byte(i.String()), nil } // UnmarshalText implements the [encoding.TextUnmarshaler] interface. func (i *Delims) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Delims") } + +var _SlicesMagicValues = []SlicesMagic{0, 1, 2} + +// SlicesMagicN is the highest valid value for type SlicesMagic, plus one. +const SlicesMagicN SlicesMagic = 3 + +var _SlicesMagicValueMap = map[string]SlicesMagic{`FullAxis`: 0, `NewAxis`: 1, `Ellipsis`: 2} + +var _SlicesMagicDescMap = map[SlicesMagic]string{0: `FullAxis indicates that the full existing axis length should be used. This is equivalent to Slice{}, but is more semantic. In NumPy it is equivalent to a single : colon.`, 1: `NewAxis creates a new singleton (length=1) axis, used to to reshape without changing the size. Can also be used in [Reshaped].`, 2: `Ellipsis (...) is used in [NewSliced] expressions to produce a flexibly-sized stretch of FullAxis dimensions, which automatically aligns the remaining slice elements based on the source dimensionality.`} + +var _SlicesMagicMap = map[SlicesMagic]string{0: `FullAxis`, 1: `NewAxis`, 2: `Ellipsis`} + +// String returns the string representation of this SlicesMagic value. +func (i SlicesMagic) String() string { return enums.String(i, _SlicesMagicMap) } + +// SetString sets the SlicesMagic value from its string representation, +// and returns an error if the string is invalid. +func (i *SlicesMagic) SetString(s string) error { + return enums.SetString(i, s, _SlicesMagicValueMap, "SlicesMagic") +} + +// Int64 returns the SlicesMagic value as an int64. +func (i SlicesMagic) Int64() int64 { return int64(i) } + +// SetInt64 sets the SlicesMagic value from an int64. +func (i *SlicesMagic) SetInt64(in int64) { *i = SlicesMagic(in) } + +// Desc returns the description of the SlicesMagic value. +func (i SlicesMagic) Desc() string { return enums.Desc(i, _SlicesMagicDescMap) } + +// SlicesMagicValues returns all possible values for the type SlicesMagic. +func SlicesMagicValues() []SlicesMagic { return _SlicesMagicValues } + +// Values returns all possible values for the type SlicesMagic. +func (i SlicesMagic) Values() []enums.Enum { return enums.Values(_SlicesMagicValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i SlicesMagic) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *SlicesMagic) UnmarshalText(text []byte) error { + return enums.UnmarshalText(i, text, "SlicesMagic") +} diff --git a/tensor/reshaped.go b/tensor/reshaped.go index 38e40d69b6..0c12b6eec1 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -73,10 +73,6 @@ func AsReshaped(tsr Tensor) *Reshaped { return NewReshaped(tsr) } -// NewAxis can be used in [Reshaped.SetShapeSizes] to indicate where a -// new dimension (axis) is being added relative to the source shape. -const NewAxis = 1 - // SetShapeSizes sets our shape sizes to the given values, which must result in // the same length as the source tensor. An error is returned if not. // If a different subset of content is desired, use another view such as [Sliced]. diff --git a/tensor/sliced.go b/tensor/sliced.go index 94cec44b5f..a7bdcab4fa 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -51,14 +51,20 @@ func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { return sl } -// NewSliced returns a new [Sliced] view of given tensor, +// NewSliced returns a new [Sliced] (and potentially [Reshaped]) view of given tensor, // with given slice expressions for each dimension, which can be: // - an integer, indicating a specific index value along that dimension. +// Can use negative numbers to index from the end. +// This axis will also be removed using a [Reshaped]. // - a [Slice] object expressing a range of indexes. -// - [Elipses] which acts as a kind of "spacer" to include all dimensions -// up to the next expression. +// - [FullAxis] includes the full original axis (equivalent to `Slice{}`). +// - [Ellipsis] creates a flexibly-sized stretch of FullAxis dimensions, +// which automatically aligns the remaining slice elements based on the source +// dimensionality. +// - [NewAxis] creates a new singleton (length=1) axis, used to to reshape +// without changing the size. This triggers a [Reshaped]. // - any remaining dimensions without indexes default to nil = full sequential view. -func NewSliced(tsr Tensor, sls ...any) *Sliced { +func NewSliced(tsr Tensor, sls ...any) Tensor { ns := len(sls) if ns == 0 { return NewSlicedIndexes(tsr) @@ -66,11 +72,14 @@ func NewSliced(tsr Tensor, sls ...any) *Sliced { nd := tsr.NumDims() ed := nd - ns // extra dimensions ixs := make([][]int, nd) + doReshape := false // indicates if we need a Reshaped + reshape := make([]int, 0, nd+2) // if we need one, this is the target shape ci := 0 for d := range ns { s := sls[d] switch x := s.(type) { case int: + doReshape = true // doesn't add to new shape. if x < 0 { ixs[ci] = []int{tsr.DimSize(ci) + x} } else { @@ -78,15 +87,45 @@ func NewSliced(tsr Tensor, sls ...any) *Sliced { } case Slice: ixs[ci] = x.IntSlice(tsr.DimSize(ci)) - case ElipsesType: - for range ed { + reshape = append(reshape, len(ixs[ci])) + case SlicesMagic: + switch x { + case FullAxis: ixs[ci] = Slice{}.IntSlice(tsr.DimSize(ci)) - ci++ + reshape = append(reshape, len(ixs[ci])) + case NewAxis: + ed++ // we are not real + doReshape = true + reshape = append(reshape, 1) + continue // skip the increment in ci + case Ellipsis: + ed++ // extra for us + for range ed { + ixs[ci] = Slice{}.IntSlice(tsr.DimSize(ci)) + reshape = append(reshape, len(ixs[ci])) + ci++ + } + if ed > 0 { + ci-- + } + ed = 0 // ate them up } } ci++ } - return NewSlicedIndexes(tsr, ixs...) + for range ed { // fill any extra dimensions + ixs[ci] = Slice{}.IntSlice(tsr.DimSize(ci)) + reshape = append(reshape, len(ixs[ci])) + ci++ + } + sl := NewSlicedIndexes(tsr, ixs...) + if doReshape { + if len(reshape) == 0 { // all indexes + reshape = []int{1} + } + return NewReshaped(sl, reshape...) + } + return sl } // AsSliced returns the tensor as a [Sliced] view. @@ -96,7 +135,7 @@ func AsSliced(tsr Tensor) *Sliced { if sl, ok := tsr.(*Sliced); ok { return sl } - return NewSliced(tsr) + return NewSlicedIndexes(tsr) } // SetTensor sets tensor as source for this view, and initializes a full diff --git a/tensor/slices.go b/tensor/slices.go index 3e437e7ea8..f396f9041d 100644 --- a/tensor/slices.go +++ b/tensor/slices.go @@ -4,11 +4,25 @@ package tensor -// ElipsesType is a special type for marking the Elipses in [Sliced] expressions. -type ElipsesType int +// SlicesMagic are special elements in slice expressions, including +// NewAxis, FullAxis, and Ellipsis in [NewSliced] expressions. +type SlicesMagic int //enums:enum -// Elipses is used in [Sliced] expressions to stretch between elements. -const Elipses ElipsesType = 0 +const ( + // FullAxis indicates that the full existing axis length should be used. + // This is equivalent to Slice{}, but is more semantic. In NumPy it is + // equivalent to a single : colon. + FullAxis SlicesMagic = iota + + // NewAxis creates a new singleton (length=1) axis, used to to reshape + // without changing the size. Can also be used in [Reshaped]. + NewAxis + + // Ellipsis (...) is used in [NewSliced] expressions to produce + // a flexibly-sized stretch of FullAxis dimensions, which automatically + // aligns the remaining slice elements based on the source dimensionality. + Ellipsis +) // Slice represents a slice of index values, for extracting slices of data, // along a dimension of a given size, which is provided separately as an argument. diff --git a/tensor/slices_test.go b/tensor/slices_test.go index 04ac27a411..8f75280684 100644 --- a/tensor/slices_test.go +++ b/tensor/slices_test.go @@ -79,25 +79,20 @@ func TestSlicedExpr(t *testing.T) { assert.Equal(t, res, ft.String()) // fmt.Println(ft) - res = `[1, 1] -[0]: 12 + res = `[1] 12 ` sl := NewSliced(ft, 1, 2) // fmt.Println(sl) assert.Equal(t, res, sl.String()) - res = `[1, 4] -[0]: 10 11 12 13 + res = `[4] 10 11 12 13 ` sl = NewSliced(ft, 1) assert.Equal(t, res, sl.String()) - res = `[3, 1] -[0]: 2 -[1]: 12 -[2]: 22 + res = `[3] 2 12 22 ` - sl = NewSliced(ft, Elipses, 2) + sl = NewSliced(ft, Ellipsis, 2) assert.Equal(t, res, sl.String()) res = `[3, 4] @@ -105,6 +100,23 @@ func TestSlicedExpr(t *testing.T) { [1]: 13 12 11 10 [2]: 23 22 21 20 ` - sl = NewSliced(ft, Elipses, Slice{Step: -1}) + sl = NewSliced(ft, Ellipsis, Slice{Step: -1}) + assert.Equal(t, res, sl.String()) + + res = `[1, 4] +[0]: 10 11 12 13 +` + sl = NewSliced(ft, NewAxis, 1) + assert.Equal(t, res, sl.String()) + + res = `[1, 3] +[0]: 1 11 21 +` + sl = NewSliced(ft, NewAxis, FullAxis, 1) // keeps result as a column vector + assert.Equal(t, res, sl.String()) + + res = `[3] 1 11 21 +` + sl = NewSliced(ft, FullAxis, 1) assert.Equal(t, res, sl.String()) } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index db20bacd4d..f700466a09 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -146,11 +146,11 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { func(idx int, tsr ...tensor.Tensor) { c := coords[idx] if c.X != curCoords.X { - av = tensor.NewSliced(tsr[0], tensor.Elipses, c.X) + av = tensor.NewSliced(tsr[0], tensor.FullAxis, c.X) curCoords.X = c.X } if c.Y != curCoords.Y { - bv = tensor.NewSliced(tsr[0], tensor.Elipses, c.Y) + bv = tensor.NewSliced(tsr[0], tensor.FullAxis, c.Y) curCoords.Y = c.Y } mfun(av, bv, mout) diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 3331c53991..7e8d74b687 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -155,11 +155,11 @@ func TestSliced(t *testing.T) { [1]: 12 [2]: 22 ` - sl = NewSliced(ft, Slice{}, Slice{2, 3, 0}) + sl2 := NewSliced(ft, FullAxis, Slice{2, 3, 0}) // fmt.Println(sl) - assert.Equal(t, res, sl.String()) + assert.Equal(t, res, sl2.String()) - vl = sl.AsValues() + vl = sl2.AsValues() assert.Equal(t, res, vl.String()) } @@ -256,7 +256,7 @@ func TestReshaped(t *testing.T) { [0]: 10 11 12 13 [0]: 20 21 22 23 ` - rs = NewReshaped(ft, NewAxis, 3, 4) + rs = NewReshaped(ft, int(NewAxis), 3, 4) // fmt.Println(rs) assert.Equal(t, res, rs.String()) } From 46a35871b901dbf852743bc2b1cdd0b02a337d8e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 24 Sep 2024 12:23:09 -0700 Subject: [PATCH 116/311] yaegicore update --- yaegicore/symbols/cogentcore_org-core-tensor.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 8766751f04..83a792ba39 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -5,8 +5,6 @@ package symbols import ( "cogentcore.org/core/base/metadata" "cogentcore.org/core/tensor" - "go/constant" - "go/token" "reflect" ) @@ -51,9 +49,10 @@ func init() { "DelimsValues": reflect.ValueOf(tensor.DelimsValues), "Descending": reflect.ValueOf(tensor.Descending), "Detect": reflect.ValueOf(tensor.Detect), - "Elipses": reflect.ValueOf(tensor.Elipses), + "Ellipsis": reflect.ValueOf(tensor.Ellipsis), "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "FullAxis": reflect.ValueOf(tensor.FullAxis), "FuncByName": reflect.ValueOf(tensor.FuncByName), "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), "IntToBool": reflect.ValueOf(tensor.IntToBool), @@ -63,7 +62,7 @@ func init() { "NFirstLen": reflect.ValueOf(tensor.NFirstLen), "NFirstRows": reflect.ValueOf(tensor.NFirstRows), "NMinLen": reflect.ValueOf(tensor.NMinLen), - "NewAxis": reflect.ValueOf(constant.MakeFromLiteral("1", token.INT, 0)), + "NewAxis": reflect.ValueOf(tensor.NewAxis), "NewBool": reflect.ValueOf(tensor.NewBool), "NewBoolShape": reflect.ValueOf(tensor.NewBoolShape), "NewByte": reflect.ValueOf(tensor.NewByte), @@ -116,6 +115,8 @@ func init() { "SetShapeSizesFromTensor": reflect.ValueOf(tensor.SetShapeSizesFromTensor), "SetShapeSizesMustBeValues": reflect.ValueOf(tensor.SetShapeSizesMustBeValues), "ShapeNames": reflect.ValueOf(tensor.ShapeNames), + "SlicesMagicN": reflect.ValueOf(tensor.SlicesMagicN), + "SlicesMagicValues": reflect.ValueOf(tensor.SlicesMagicValues), "Space": reflect.ValueOf(tensor.Space), "Sprint": reflect.ValueOf(tensor.Sprint), "StableSort": reflect.ValueOf(tensor.StableSort), @@ -132,7 +133,6 @@ func init() { // type definitions "Bool": reflect.ValueOf((*tensor.Bool)(nil)), "Delims": reflect.ValueOf((*tensor.Delims)(nil)), - "ElipsesType": reflect.ValueOf((*tensor.ElipsesType)(nil)), "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), "FilterOptions": reflect.ValueOf((*tensor.FilterOptions)(nil)), "Func": reflect.ValueOf((*tensor.Func)(nil)), @@ -144,6 +144,7 @@ func init() { "Shape": reflect.ValueOf((*tensor.Shape)(nil)), "Slice": reflect.ValueOf((*tensor.Slice)(nil)), "Sliced": reflect.ValueOf((*tensor.Sliced)(nil)), + "SlicesMagic": reflect.ValueOf((*tensor.SlicesMagic)(nil)), "String": reflect.ValueOf((*tensor.String)(nil)), "Tensor": reflect.ValueOf((*tensor.Tensor)(nil)), "Values": reflect.ValueOf((*tensor.Values)(nil)), From 72e7b8bfd367151150a40e8c249e9fbd886aaf03 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 24 Sep 2024 13:27:10 -0700 Subject: [PATCH 117/311] goal: new function stack and appropriate contextualization of tensor args -- tests now passing --- goal/transpile/math.go | 114 ++++++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 30 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 7803f3f98b..1a18107e1d 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -8,6 +8,9 @@ import ( "fmt" "go/ast" "go/token" + + "cogentcore.org/core/base/stack" + "cogentcore.org/core/tensor" ) func MathParse(toks Tokens, code string, fullLine bool) Tokens { @@ -45,17 +48,39 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { return mp.out } +// funcInfo is info about the function being processed +type funcInfo struct { + tensor.Func + + // true if this function takes tensor args + tensorArgs bool +} + // mathParse has the parsing state type mathParse struct { code string // code string toks Tokens // source tokens we are parsing - idx int // current index in source tokens + idx int // current index in source tokens -- critical to sync as we "use" source out Tokens // output tokens we generate - // goLiteral means generate basic literals as standard go literals instead of - // wrapping them in tensor constructors. for inner expressions contstructing go - // objects etc. - goLiteral bool + // stack of function info -- top of stack reflects the current function + funcs stack.Stack[*funcInfo] +} + +// startFunc is called when starting a new function -- sets context +func (mp *mathParse) startFunc(name string, tensorArgs bool) *funcInfo { + fn := &funcInfo{} + fn.Name = name + fn.tensorArgs = tensorArgs + mp.funcs.Push(fn) + if name != "" { + mp.out.Add(token.IDENT, name) + } + return fn +} + +func (mp *mathParse) endFunc() { + mp.funcs.Pop() } // addToken adds output token and increments idx @@ -102,9 +127,12 @@ func (mp *mathParse) stmt(st ast.Stmt) { mp.addToken(x.Tok) case *ast.AssignStmt: - mp.exprList(x.Lhs) - mp.addToken(x.Tok) - mp.exprList(x.Rhs) + switch x.Tok { + case token.DEFINE: + mp.defineStmt(x) + case token.ASSIGN: + mp.assignStmt(x) + } case *ast.GoStmt: mp.addToken(token.GO) @@ -232,7 +260,7 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { case token.QUO: fn = "Div" } - mp.out.Add(token.IDENT, "tensor.CallOut") + mp.startFunc("tensor.CallOut", true) // yes tensor args mp.out.Add(token.LPAREN) mp.out.Add(token.STRING, `"`+fn+`"`) mp.out.Add(token.COMMA) @@ -241,6 +269,7 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { mp.idx++ mp.expr(ex.Y) mp.out.Add(token.RPAREN) + mp.endFunc() } func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { @@ -248,11 +277,34 @@ func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { mp.expr(ex.X) } +func (mp *mathParse) defineStmt(as *ast.AssignStmt) { + mp.exprList(as.Lhs) + mp.addToken(as.Tok) + mp.startFunc("", true) // just to trigger tensor args + mp.exprList(as.Rhs) + mp.endFunc() +} + +func (mp *mathParse) assignStmt(as *ast.AssignStmt) { + // todo: use assign op if lhs is not ident + mp.exprList(as.Lhs) + mp.addToken(as.Tok) + mp.startFunc("", true) // just to trigger tensor args + mp.exprList(as.Rhs) + mp.endFunc() +} + func (mp *mathParse) basicLit(lit *ast.BasicLit) { - if mp.goLiteral { - mp.out.Add(lit.Kind, lit.Value) + cfun := mp.funcs.Peek() + if cfun != nil && cfun.tensorArgs { + mp.tensorLit(lit) return } + mp.out.Add(lit.Kind, lit.Value) + return +} + +func (mp *mathParse) tensorLit(lit *ast.BasicLit) { switch lit.Kind { case token.INT: mp.out.Add(token.IDENT, "tensor.NewIntScalar("+lit.Value+")") @@ -322,19 +374,22 @@ func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { } func (mp *mathParse) indexExpr(il *ast.IndexExpr) { - if iil, ok := il.Index.(*ast.IndexListExpr); ok { - // todo: need to analyze and see what kind of expr it is - mp.out.Add(token.IDENT, "tensor.NewSliced") - mp.out.Add(token.LPAREN) - mp.expr(il.X) - mp.addToken(token.COMMA) // use the [ - mp.goLiteral = true - mp.exprList(iil.Indices) - mp.goLiteral = true - mp.addToken(token.RPAREN) // replaces ] + if _, ok := il.Index.(*ast.IndexListExpr); ok { + mp.basicSlicingExpr(il) } } +func (mp *mathParse) basicSlicingExpr(il *ast.IndexExpr) { + iil := il.Index.(*ast.IndexListExpr) + mp.startFunc("tensor.NewSliced", false) + mp.out.Add(token.LPAREN) + mp.expr(il.X) + mp.addToken(token.COMMA) // use the [ + mp.exprList(iil.Indices) + mp.addToken(token.RPAREN) // replaces ] + mp.endFunc() +} + func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { mp.out.Add(token.IDENT, "tensor.Slice") mp.addToken(token.LBRACE) @@ -379,16 +434,15 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { typ = "string" fun = "String" } - mp.out.Add(token.IDENT, "tensor.New"+fun+"FromValues") + mp.startFunc("tensor.New"+fun+"FromValues", false) mp.out.Add(token.LPAREN) mp.out.Add(token.IDENT, "[]"+typ) mp.addToken(token.LBRACE) - mp.goLiteral = true mp.exprList(il.Indices) - mp.goLiteral = false mp.addToken(token.RBRACE) mp.addToken(token.ELLIPSIS) mp.out.Add(token.RPAREN) + mp.endFunc() } var numpyFuncs = map[string]funWrap{ @@ -401,21 +455,21 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { if fnm, ok := ex.Fun.(*ast.Ident); ok { if fw, ok := numpyFuncs[fnm.Name]; ok { // todo: wrap - mp.out.Add(token.IDENT, fw.fun) - mp.idx++ + mp.startFunc(fw.fun, false) + mp.addToken(token.LPAREN) // use the ( + mp.idx++ // paren too } else { - mp.out.Add(token.IDENT, fnm.Name) + mp.startFunc(fnm.Name, false) + mp.addToken(token.LPAREN) // use the ( mp.idx++ } } else { mp.expr(ex.Fun) } - mp.addToken(token.LPAREN) - mp.goLiteral = true // todo: need a stack for this. mp.exprList(ex.Args) - mp.goLiteral = false // todo: ellipsis mp.addToken(token.RPAREN) + mp.endFunc() } func (mp *mathParse) ident(id *ast.Ident) { From 976a67844d119215676542d75ea18b220306da74 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 24 Sep 2024 17:19:39 -0700 Subject: [PATCH 118/311] tensor function calling working -- allows lowercase on non-package-qualified names, e.g., cos, sin etc. --- goal/transpile/math.go | 51 +++++++++++++++++++++++++------- goal/transpile/transpile_test.go | 7 +++++ tensor/funcs.go | 2 +- 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 1a18107e1d..ba295901e7 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -8,6 +8,7 @@ import ( "fmt" "go/ast" "go/token" + "strings" "cogentcore.org/core/base/stack" "cogentcore.org/core/tensor" @@ -200,6 +201,9 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.FuncLit: case *ast.ParenExpr: + mp.addToken(token.LPAREN) + mp.expr(x.X) + mp.addToken(token.RPAREN) case *ast.SelectorExpr: mp.selectorExpr(x) @@ -452,18 +456,16 @@ var numpyFuncs = map[string]funWrap{ } func (mp *mathParse) callExpr(ex *ast.CallExpr) { - if fnm, ok := ex.Fun.(*ast.Ident); ok { - if fw, ok := numpyFuncs[fnm.Name]; ok { - // todo: wrap - mp.startFunc(fw.fun, false) - mp.addToken(token.LPAREN) // use the ( - mp.idx++ // paren too + switch x := ex.Fun.(type) { + case *ast.Ident: + mp.callName(ex, x.Name) + case *ast.SelectorExpr: + if pkg, ok := x.X.(*ast.Ident); ok { + mp.callName(ex, pkg.Name+"."+x.Sel.Name) } else { - mp.startFunc(fnm.Name, false) - mp.addToken(token.LPAREN) // use the ( - mp.idx++ + fmt.Printf("call, weird sel: %#v\n", x.X) } - } else { + default: mp.expr(ex.Fun) } mp.exprList(ex.Args) @@ -472,6 +474,35 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { mp.endFunc() } +func (mp *mathParse) callName(ex *ast.CallExpr, funName string) { + if fw, ok := numpyFuncs[funName]; ok { + // todo: wrap + mp.startFunc(fw.fun, false) + mp.addToken(token.LPAREN) // use the ( + mp.idx++ // paren too + return + } + _, err := tensor.FuncByName(funName) + if err != nil { + funName = strings.ToUpper(funName[:1]) + funName[1:] + _, err = tensor.FuncByName(funName) + } + if err != nil { + fmt.Println("name not found:", funName) + mp.startFunc(funName, false) + mp.addToken(token.LPAREN) // use the ( + mp.idx++ + return + } + mp.startFunc("tensor.CallOut", true) // tensors + mp.addToken(token.LPAREN) + if strings.Contains(funName, ".") { + mp.idx += 2 // . and selector + } + mp.out.Add(token.IDENT, `"`+funName+`"`) + mp.addToken(token.COMMA) // use the name -- need more +} + func (mp *mathParse) ident(id *ast.Ident) { if id == nil { return diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 71eb9214e3..09cebbc41b 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -7,6 +7,9 @@ package transpile import ( "testing" + _ "cogentcore.org/core/tensor/stats/metric" + _ "cogentcore.org/core/tensor/stats/stats" + _ "cogentcore.org/core/tensor/tmath" "github.com/stretchr/testify/assert" ) @@ -217,6 +220,10 @@ func TestMath(t *testing.T) { {"# a[::-1, 2]", `tensor.NewSliced(a, tensor.Slice { Step: - 1 } , 2)`}, {"# a[:3, 2]", `tensor.NewSliced(a, tensor.Slice { Stop:3 } , 2)`}, {"# a[2:, 2]", `tensor.NewSliced(a, tensor.Slice { Start:2 } , 2)`}, + {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, + {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, + {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, + {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.NewReshaped(a, 36))`}, } st := NewState() diff --git a/tensor/funcs.go b/tensor/funcs.go index b029197b70..10a63eb954 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -100,7 +100,7 @@ func AddFunc(name string, fun any, out int, anyFirst ...bool) error { func FuncByName(name string) (*Func, error) { fn, ok := Funcs[name] if !ok { - return nil, errors.Log(fmt.Errorf("tensor.FuncByName: function of name %q not registered", name)) + return nil, fmt.Errorf("tensor.FuncByName: function of name %q not registered", name) } return fn, nil } From b6acc9e5e6ab373315a24a3757a25415c719563c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 00:07:30 -0700 Subject: [PATCH 119/311] add Reshape (which now supports -1 args) and Reslice tensor functions; existing New* functions are specific limited-functionality cases that just make a new instance of correponding types. --- goal/transpile/math.go | 4 +- goal/transpile/transpile_test.go | 19 +++++----- tensor/README.md | 10 +++-- tensor/convert.go | 2 +- tensor/reshaped.go | 65 +++++++++++++++++++++++++++++--- tensor/sliced.go | 14 +++---- tensor/slices_test.go | 14 +++---- tensor/stats/metric/matrix.go | 6 +-- tensor/tensor_test.go | 22 ++++++++++- 9 files changed, 116 insertions(+), 40 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index ba295901e7..bfa3429f40 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -385,7 +385,7 @@ func (mp *mathParse) indexExpr(il *ast.IndexExpr) { func (mp *mathParse) basicSlicingExpr(il *ast.IndexExpr) { iil := il.Index.(*ast.IndexListExpr) - mp.startFunc("tensor.NewSliced", false) + mp.startFunc("tensor.Reslice", false) mp.out.Add(token.LPAREN) mp.expr(il.X) mp.addToken(token.COMMA) // use the [ @@ -452,7 +452,7 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { var numpyFuncs = map[string]funWrap{ "zeros": {"tensor.NewFloat64", ""}, "arange": {"tensor.NewSliceInts", ""}, - "reshape": {"tensor.NewReshaped", ""}, + "reshape": {"tensor.Reshape", ""}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 09cebbc41b..20424b84fa 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -212,18 +212,19 @@ func TestMath(t *testing.T) { {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - {"# a := reshape(x, 6, 6)", `a := tensor.NewReshaped(x, 6, 6)`}, - {"# a := reshape(arange(36), 6, 6)", `a := tensor.NewReshaped(tensor.NewSliceInts(36), 6, 6)`}, - {"# a[1, 2]", `tensor.NewSliced(a, 1, 2)`}, - {"# a[:, 2]", `tensor.NewSliced(a, tensor.Slice { } , 2)`}, - {"# a[1:3:1, 2]", `tensor.NewSliced(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - {"# a[::-1, 2]", `tensor.NewSliced(a, tensor.Slice { Step: - 1 } , 2)`}, - {"# a[:3, 2]", `tensor.NewSliced(a, tensor.Slice { Stop:3 } , 2)`}, - {"# a[2:, 2]", `tensor.NewSliced(a, tensor.Slice { Start:2 } , 2)`}, + {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + {"# a[:, 2]", `tensor.Reslice(a, tensor.Slice { } , 2)`}, + {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, - {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.NewReshaped(a, 36))`}, + {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.Reshape(a, 36))`}, + {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tensor.CallOut("Sub", tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), tensor.CallOut("stats.Mean", ra))`}, } st := NewState() diff --git a/tensor/README.md b/tensor/README.md index 2e412e8d7e..0fdb28e410 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -54,9 +54,13 @@ All tensor package functions are registered using a global name-to-function map # Standard shapes -In general, **1D** refers to a flat, 1-dimensional list. There are various standard shapes of tensor data that different functions expect: +There are various standard shapes of tensor data that different functions expect, listed below. The two most general-purpose functions for shaping and slicing any tensor to get it into the right shape for a given computation are: -* **Flat, 1D**: this is the simplest data shape, and any tensor can be turned into a flat 1D list using the `As1D` function, which either returns the tensor itself it is already 1D, or a `Reshaped` 1D view. The [stats](stats) functions for example report summary statistics across the outermost row dimension, so converting data to this 1D view gives stats across all the data. +* `Reshape` returns a `Reshaped` view with the same total length as the source tensor, functioning like the NumPy `reshape` function. + +* `Reslice` returns a re-sliced view of a tensor, extracting or rearranging dimenstions. It supports the full NumPy [basic indexing](https://numpy.org/doc/stable/user/basics.indexing.html#basic-indexing) syntax. It also does reshaping as needed, including processing the `NewAxis` option. + +* **Flat, 1D**: this is the simplest data shape, and any tensor can be turned into a flat 1D list using `NewReshaped(-1)` or the `As1D` function, which either returns the tensor itself it is already 1D, or a `Reshaped` 1D view. The [stats](stats) functions for example report summary statistics across the outermost row dimension, so converting data to this 1D view gives stats across all the data. * **Row, Cell 2D**: This is the natural shape for tabular data, and the `RowMajor` type and `Rows` view provide methods for efficiently accessing data in this way. In addition, the [stats](stats) and [metric](metric) packages automatically compute statistics across the outermost row dimension, aggregating results across rows for each cell. Thus, you end up with the "average cell-wise pattern" when you do `stats.Mean` for example. The `NewRowCellsView` function returns a `Reshaped` view of any tensor organized into this 2D shape, with the row vs. cell split specified at any point in the list of dimensions, which can be useful in obtaining the desired results. @@ -140,7 +144,7 @@ a[a > 0.5] = 1 # boolean advanced indexing Although powerful, the semantics of all of this is a bit confusing. In the `tensor` package, we provide what are hopefully more clear and concrete _view_ types that have well-defined semantics, and cover the relevant functionality, while perhaps being a bit easier to reason with. These were described at the start of this README. The correspondence to NumPy indexing is as follows: -* Basic indexing by individual integer index coordinate values is supported by the `Number`, `String`, `Bool` `Values` Tensors. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) and `Reshaped` views then complete the basic indexing with arbitrary reordering and filtering along entire dimension values, and reshaping dimensions. +* Basic indexing by individual integer index coordinate values is supported by the `Number`, `String`, `Bool` `Values` Tensors. For example, `Float(3,1,2)` returns the value at the given coordinates. The `Sliced` (and `Rows`) and `Reshaped` views then complete the basic indexing with arbitrary reordering and filtering along entire dimension values, and reshaping dimensions. As noted above, `Reslice` supports the full NumPy basic indexing syntax, and `Reshape` implements the NumPy `reshape` function. * The `Masked` view corresponds to the NumPy _advanced_ indexing using a same-shape boolean mask, although in the NumPy case it makes a copy (although practically it is widely used for direct assignment as shown above.) Critically, you can always extract just the `true` values from a Masked view by using the `AsValues` method on the view, which returns a 1D tensor of those values, similar to what the boolean advanced indexing produces in NumPy. In addition, the `SourceIndexes` method returns a 1D list of indexes of the `true` (or `false`) values, which can be used for the `Indexed` view. diff --git a/tensor/convert.go b/tensor/convert.go index 368b0a7b1b..d8b32bfcc6 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -95,7 +95,7 @@ func Cells1D(tsr Tensor, row int) Tensor { if rm, ok := tsr.(RowMajor); ok { return As1D(rm.RowTensor(row)) } - return As1D(NewSlicedIndexes(tsr, []int{row})) + return As1D(NewSliced(tsr, []int{row})) } // NewFloat64Scalar is a convenience method for a Tensor diff --git a/tensor/reshaped.go b/tensor/reshaped.go index 0c12b6eec1..5b86ca6a6c 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -6,6 +6,7 @@ package tensor import ( "reflect" + "slices" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" @@ -28,9 +29,11 @@ type Reshaped struct { //types:add Reshape Shape } -// NewReshaped returns a new [Reshaped] view of given tensor, -// with given shape sizes. If no such sizes are provided, -// the source shape is used. +// NewReshaped returns a new [Reshaped] view of given tensor, with given shape +// sizes. If no such sizes are provided, the source shape is used. +// A single -1 value can be used to automatically specify the remaining tensor +// length, as long as the other sizes are an even multiple of the total length. +// A single -1 returns a 1D view of the entire tensor. func NewReshaped(tsr Tensor, sizes ...int) *Reshaped { rs := &Reshaped{Tensor: tsr} if len(sizes) == 0 { @@ -41,6 +44,27 @@ func NewReshaped(tsr Tensor, sizes ...int) *Reshaped { return rs } +// Reshape returns a view of the given tensor with given shape sizes. +// A single -1 value can be used to automatically specify the remaining tensor +// length, as long as the other sizes are an even multiple of the total length. +// A single -1 returns a 1D view of the entire tensor. +func Reshape(tsr Tensor, sizes ...int) Tensor { + if len(sizes) == 0 { + err := errors.New("tensor.Reshape: must pass shape sizes") + errors.Log(err) + return tsr + } + if len(sizes) == 1 { + sz := sizes[0] + if sz == -1 { + return As1D(tsr) + } + } + rs := &Reshaped{Tensor: tsr} + errors.Log(rs.SetShapeSizes(sizes...)) + return rs +} + // NewRowCellsView returns a 2D [Reshaped] view onto the given tensor, // with a single outer "row" dimension and a single inner "cells" dimension, // with the given 'split' dimension specifying where the cells start. @@ -80,10 +104,39 @@ func AsReshaped(tsr Tensor) *Reshaped { // the length, and the [NewAxis] value can be used to semantically // indicate when such a new dimension is being inserted. This is often useful // for aligning two tensors to achieve a desired computation; see [AlignShapes] -// function. +// function. A single -1 can be used to specify a dimension size that takes the +// remaining length, as long as the other sizes are an even multiple of the length. +// A single -1 indicates to use the full length. func (rs *Reshaped) SetShapeSizes(sizes ...int) error { - rs.Reshape.SetShapeSizes(sizes...) - if rs.Reshape.Len() != rs.Tensor.Len() { + sln := rs.Tensor.Len() + if sln == 0 { + return nil + } + if sln == 1 { + sz := sizes[0] + if sz < 0 { + rs.Reshape.SetShapeSizes(sln) + return nil + } + } + sz := slices.Clone(sizes) + ln := 1 + negIdx := -1 + for i, s := range sz { + if s < 0 { + negIdx = i + } else { + ln *= s + } + } + if negIdx >= 0 { + if sln%ln != 0 { + return errors.New("tensor.Reshaped SetShapeSizes: -1 cannot be used because the remaining dimensions are not an even multiple of the source tensor length") + } + sz[negIdx] = sln / ln + } + rs.Reshape.SetShapeSizes(sz...) + if rs.Reshape.Len() != sln { return errors.New("tensor.Reshaped SetShapeSizes: new length is different from source tensor; use Sliced or other views to change view content") } return nil diff --git a/tensor/sliced.go b/tensor/sliced.go index a7bdcab4fa..c02df03a59 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -42,16 +42,16 @@ type Sliced struct { //types:add Indexes [][]int } -// NewSlicedIndexes returns a new [Sliced] view of given tensor, +// NewSliced returns a new [Sliced] view of given tensor, // with optional list of indexes for each dimension (none / nil = sequential). // Any dimensions without indexes default to nil = full sequential view. -func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { +func NewSliced(tsr Tensor, idxs ...[]int) *Sliced { sl := &Sliced{Tensor: tsr, Indexes: idxs} sl.ValidIndexes() return sl } -// NewSliced returns a new [Sliced] (and potentially [Reshaped]) view of given tensor, +// Reslice returns a new [Sliced] (and potentially [Reshaped]) view of given tensor, // with given slice expressions for each dimension, which can be: // - an integer, indicating a specific index value along that dimension. // Can use negative numbers to index from the end. @@ -64,10 +64,10 @@ func NewSlicedIndexes(tsr Tensor, idxs ...[]int) *Sliced { // - [NewAxis] creates a new singleton (length=1) axis, used to to reshape // without changing the size. This triggers a [Reshaped]. // - any remaining dimensions without indexes default to nil = full sequential view. -func NewSliced(tsr Tensor, sls ...any) Tensor { +func Reslice(tsr Tensor, sls ...any) Tensor { ns := len(sls) if ns == 0 { - return NewSlicedIndexes(tsr) + return NewSliced(tsr) } nd := tsr.NumDims() ed := nd - ns // extra dimensions @@ -118,7 +118,7 @@ func NewSliced(tsr Tensor, sls ...any) Tensor { reshape = append(reshape, len(ixs[ci])) ci++ } - sl := NewSlicedIndexes(tsr, ixs...) + sl := NewSliced(tsr, ixs...) if doReshape { if len(reshape) == 0 { // all indexes reshape = []int{1} @@ -135,7 +135,7 @@ func AsSliced(tsr Tensor) *Sliced { if sl, ok := tsr.(*Sliced); ok { return sl } - return NewSlicedIndexes(tsr) + return NewSliced(tsr) } // SetTensor sets tensor as source for this view, and initializes a full diff --git a/tensor/slices_test.go b/tensor/slices_test.go index 8f75280684..432aea67ea 100644 --- a/tensor/slices_test.go +++ b/tensor/slices_test.go @@ -81,18 +81,18 @@ func TestSlicedExpr(t *testing.T) { res = `[1] 12 ` - sl := NewSliced(ft, 1, 2) + sl := Reslice(ft, 1, 2) // fmt.Println(sl) assert.Equal(t, res, sl.String()) res = `[4] 10 11 12 13 ` - sl = NewSliced(ft, 1) + sl = Reslice(ft, 1) assert.Equal(t, res, sl.String()) res = `[3] 2 12 22 ` - sl = NewSliced(ft, Ellipsis, 2) + sl = Reslice(ft, Ellipsis, 2) assert.Equal(t, res, sl.String()) res = `[3, 4] @@ -100,23 +100,23 @@ func TestSlicedExpr(t *testing.T) { [1]: 13 12 11 10 [2]: 23 22 21 20 ` - sl = NewSliced(ft, Ellipsis, Slice{Step: -1}) + sl = Reslice(ft, Ellipsis, Slice{Step: -1}) assert.Equal(t, res, sl.String()) res = `[1, 4] [0]: 10 11 12 13 ` - sl = NewSliced(ft, NewAxis, 1) + sl = Reslice(ft, NewAxis, 1) assert.Equal(t, res, sl.String()) res = `[1, 3] [0]: 1 11 21 ` - sl = NewSliced(ft, NewAxis, FullAxis, 1) // keeps result as a column vector + sl = Reslice(ft, NewAxis, FullAxis, 1) // keeps result as a column vector assert.Equal(t, res, sl.String()) res = `[3] 1 11 21 ` - sl = NewSliced(ft, FullAxis, 1) + sl = Reslice(ft, FullAxis, 1) assert.Equal(t, res, sl.String()) } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index f700466a09..f1de5e8fba 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -146,11 +146,11 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { func(idx int, tsr ...tensor.Tensor) { c := coords[idx] if c.X != curCoords.X { - av = tensor.NewSliced(tsr[0], tensor.FullAxis, c.X) + av = tensor.Reslice(tsr[0], tensor.FullAxis, c.X) curCoords.X = c.X } if c.Y != curCoords.Y { - bv = tensor.NewSliced(tsr[0], tensor.FullAxis, c.Y) + bv = tensor.Reslice(tsr[0], tensor.FullAxis, c.Y) curCoords.Y = c.Y } mfun(av, bv, mout) @@ -244,7 +244,7 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) error { // This is typically done with results from SVD or PCA. func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) error { ci := int(colindex.Float1D(0)) - col := tensor.As1D(tensor.NewSliced(mtx, tensor.Slice{}, ci)) + col := tensor.As1D(tensor.Reslice(mtx, tensor.Slice{}, ci)) // fmt.Println(mtx.String(), col.String()) rows, cells := vec.Shape().RowCellSize() if rows > 0 && cells > 0 { diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 7e8d74b687..009fd775f7 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -144,7 +144,7 @@ func TestSliced(t *testing.T) { [0]: 23 22 [1]: 13 12 ` - sl := NewSlicedIndexes(ft, []int{2, 1}, []int{3, 2}) + sl := NewSliced(ft, []int{2, 1}, []int{3, 2}) // fmt.Println(sl) assert.Equal(t, res, sl.String()) @@ -155,7 +155,7 @@ func TestSliced(t *testing.T) { [1]: 12 [2]: 22 ` - sl2 := NewSliced(ft, FullAxis, Slice{2, 3, 0}) + sl2 := Reslice(ft, FullAxis, Slice{2, 3, 0}) // fmt.Println(sl) assert.Equal(t, res, sl2.String()) @@ -257,8 +257,26 @@ func TestReshaped(t *testing.T) { [0]: 20 21 22 23 ` rs = NewReshaped(ft, int(NewAxis), 3, 4) + assert.Equal(t, res, rs.String()) + + res = `[12] +[0]: 0 1 2 3 10 11 12 13 20 21 22 23 +` + rs = NewReshaped(ft, -1) + assert.Equal(t, res, rs.String()) + + res = `[4, 3] +[0]: 0 1 2 +[1]: 3 10 11 +[2]: 12 13 20 +[3]: 21 22 23 +` + rs = NewReshaped(ft, 4, -1) // fmt.Println(rs) assert.Equal(t, res, rs.String()) + + err := rs.SetShapeSizes(5, -1) + assert.Error(t, err) } func TestSortFilter(t *testing.T) { From 09fb87074c6168699f172118144ffa3b3fbfad00 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 01:37:29 -0700 Subject: [PATCH 120/311] parsing alignment fixed -- full expressions work --- goal/transpile/math.go | 58 ++++++++++++++----- goal/transpile/transpile_test.go | 2 +- .../symbols/cogentcore_org-core-tensor.go | 3 +- 3 files changed, 48 insertions(+), 15 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index bfa3429f40..7a0f48ebdb 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -27,6 +27,7 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { } // fmt.Println(str) mp := mathParse{toks: toks, code: code} + // mp.trace = true mods := AllErrors // | Trace @@ -46,6 +47,12 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { mp.expr(ex) } + if mp.idx != len(toks) { + fmt.Println(code) + fmt.Println(mp.out.Code()) + fmt.Printf("parsing error: index: %d != len(toks): %d\n", mp.idx, len(toks)) + } + return mp.out } @@ -59,10 +66,11 @@ type funcInfo struct { // mathParse has the parsing state type mathParse struct { - code string // code string - toks Tokens // source tokens we are parsing - idx int // current index in source tokens -- critical to sync as we "use" source - out Tokens // output tokens we generate + code string // code string + toks Tokens // source tokens we are parsing + idx int // current index in source tokens -- critical to sync as we "use" source + out Tokens // output tokens we generate + trace bool // trace of parsing -- turn on to see alignment // stack of function info -- top of stack reflects the current function funcs stack.Stack[*funcInfo] @@ -87,6 +95,13 @@ func (mp *mathParse) endFunc() { // addToken adds output token and increments idx func (mp *mathParse) addToken(tok token.Token) { mp.out.Add(tok) + if mp.trace { + ctok := &Token{} + if mp.idx < len(mp.toks) { + ctok = mp.toks[mp.idx] + } + fmt.Printf("%d\ttok: %s \t replaces: %s\n", mp.idx, tok, ctok) + } mp.idx++ } @@ -305,6 +320,10 @@ func (mp *mathParse) basicLit(lit *ast.BasicLit) { return } mp.out.Add(lit.Kind, lit.Value) + if mp.trace { + fmt.Printf("%d\ttok: %s literal\n", mp.idx, lit.Value) + } + mp.idx++ return } @@ -344,7 +363,7 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { mp.idx++ return } - elip := false + ellip := false switch fw.wrap { case "nis": mp.out.Add(token.IDENT, "tensor.NewIntScalar") @@ -354,20 +373,20 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { mp.out.Add(token.IDENT, "tensor.NewStringScalar") case "nifs": mp.out.Add(token.IDENT, "tensor.NewIntFromValues") - elip = true + ellip = true case "nffs": mp.out.Add(token.IDENT, "tensor.NewFloat64FromValues") - elip = true + ellip = true case "nsfs": mp.out.Add(token.IDENT, "tensor.NewStringFromValues") - elip = true + ellip = true } mp.out.Add(token.LPAREN) mp.expr(ex.X) mp.addToken(token.PERIOD) mp.out.Add(token.IDENT, fw.fun) mp.idx++ - if elip { + if ellip { mp.out.Add(token.ELLIPSIS) } mp.out.Add(token.RPAREN) @@ -388,26 +407,35 @@ func (mp *mathParse) basicSlicingExpr(il *ast.IndexExpr) { mp.startFunc("tensor.Reslice", false) mp.out.Add(token.LPAREN) mp.expr(il.X) - mp.addToken(token.COMMA) // use the [ + mp.addToken(token.COMMA) // use the [ -- can't use ( to preserve X mp.exprList(iil.Indices) mp.addToken(token.RPAREN) // replaces ] mp.endFunc() } func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { + if se.Low == nil && se.High == nil && se.Max == nil { + mp.out.Add(token.IDENT, "tensor.FullAxis") + mp.idx++ + return + } mp.out.Add(token.IDENT, "tensor.Slice") - mp.addToken(token.LBRACE) + mp.out.Add(token.LBRACE) prev := false if se.Low != nil { mp.out.Add(token.IDENT, "Start:") mp.expr(se.Low) prev = true + if se.High == nil && se.Max == nil { + mp.idx++ + } } if se.High != nil { if prev { mp.out.Add(token.COMMA) } mp.out.Add(token.IDENT, "Stop:") + mp.idx++ mp.expr(se.High) prev = true } @@ -415,10 +443,14 @@ func (mp *mathParse) sliceExpr(se *ast.SliceExpr) { if prev { mp.out.Add(token.COMMA) } + mp.idx++ + if se.Low == nil && se.High == nil { + mp.idx++ + } mp.out.Add(token.IDENT, "Step:") mp.expr(se.Max) } - mp.addToken(token.RBRACE) + mp.out.Add(token.RBRACE) } func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { @@ -444,7 +476,7 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { mp.addToken(token.LBRACE) mp.exprList(il.Indices) mp.addToken(token.RBRACE) - mp.addToken(token.ELLIPSIS) + mp.out.Add(token.ELLIPSIS) mp.out.Add(token.RPAREN) mp.endFunc() } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 20424b84fa..047a589c84 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -215,7 +215,7 @@ func TestMath(t *testing.T) { {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - {"# a[:, 2]", `tensor.Reslice(a, tensor.Slice { } , 2)`}, + {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 83a792ba39..d8c4c104f6 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -85,7 +85,6 @@ func init() { "NewSlice": reflect.ValueOf(tensor.NewSlice), "NewSliceInts": reflect.ValueOf(tensor.NewSliceInts), "NewSliced": reflect.ValueOf(tensor.NewSliced), - "NewSlicedIndexes": reflect.ValueOf(tensor.NewSlicedIndexes), "NewString": reflect.ValueOf(tensor.NewString), "NewStringFromValues": reflect.ValueOf(tensor.NewStringFromValues), "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), @@ -104,6 +103,8 @@ func init() { "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), "Range": reflect.ValueOf(tensor.Range), "ReadCSV": reflect.ValueOf(tensor.ReadCSV), + "Reshape": reflect.ValueOf(tensor.Reshape), + "Reslice": reflect.ValueOf(tensor.Reslice), "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), "SaveCSV": reflect.ValueOf(tensor.SaveCSV), "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), From 604eb7ce0eb4f114accee70c69b6c82bc22979cb Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 02:46:14 -0700 Subject: [PATCH 121/311] assignment through slicing is working --- goal/transpile/math.go | 35 +++++++-- goal/transpile/transpile_test.go | 46 ++++++------ tensor/align.go | 39 ++++++++++ tensor/tmath/ops.go | 123 +++++++++++++++++++++++++++++++ 4 files changed, 216 insertions(+), 27 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 7a0f48ebdb..d13b67ac22 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -146,7 +146,7 @@ func (mp *mathParse) stmt(st ast.Stmt) { switch x.Tok { case token.DEFINE: mp.defineStmt(x) - case token.ASSIGN: + default: mp.assignStmt(x) } @@ -305,11 +305,36 @@ func (mp *mathParse) defineStmt(as *ast.AssignStmt) { } func (mp *mathParse) assignStmt(as *ast.AssignStmt) { - // todo: use assign op if lhs is not ident + if _, ok := as.Lhs[0].(*ast.Ident); ok { + mp.exprList(as.Lhs) + mp.addToken(as.Tok) + mp.startFunc("", true) // just to trigger tensor args + mp.exprList(as.Rhs) + mp.endFunc() + return + } + fn := "" + switch as.Tok { + case token.ASSIGN: + fn = "Assign" + case token.ADD_ASSIGN: + fn = "AddAssign" + case token.SUB_ASSIGN: + fn = "SubAssign" + case token.MUL_ASSIGN: + fn = "MulAssign" + case token.QUO_ASSIGN: + fn = "DivAssign" + } + mp.startFunc("tensor.Call", true) // yes tensor args + mp.out.Add(token.LPAREN) + mp.out.Add(token.STRING, `"`+fn+`"`) + mp.out.Add(token.COMMA) mp.exprList(as.Lhs) - mp.addToken(as.Tok) - mp.startFunc("", true) // just to trigger tensor args + mp.out.Add(token.COMMA) + mp.idx++ mp.exprList(as.Rhs) + mp.out.Add(token.RPAREN) mp.endFunc() } @@ -333,7 +358,7 @@ func (mp *mathParse) tensorLit(lit *ast.BasicLit) { mp.out.Add(token.IDENT, "tensor.NewIntScalar("+lit.Value+")") mp.idx++ case token.FLOAT: - mp.out.Add(token.IDENT, "tensor.NewFloatScalar("+lit.Value+")") + mp.out.Add(token.IDENT, "tensor.NewFloat64Scalar("+lit.Value+")") mp.idx++ case token.STRING: mp.out.Add(token.IDENT, "tensor.NewStringScalar("+lit.Value+")") diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 047a589c84..f8667b79de 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -203,28 +203,30 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, - {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, - {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, - {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, - {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, - {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, - {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, - {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.Reshape(a, 36))`}, - {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tensor.CallOut("Sub", tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), tensor.CallOut("stats.Mean", ra))`}, + // {"# x := 1", `x := tensor.NewIntScalar(1)`}, + // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + // {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + // {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + // {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + // {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + // {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + // {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, + // {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + // {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + // {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + // {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + // {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, + // {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, + // {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, + // {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.Reshape(a, 36))`}, + // {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tensor.CallOut("Sub", tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), tensor.CallOut("stats.Mean", ra))`}, + // {"# a[:, 2] = b", `tensor.Call("Assign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# a[:, 2] += b", `tensor.Call("AddAssign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, } st := NewState() diff --git a/tensor/align.go b/tensor/align.go index 70c7f0396f..12bf4c1e72 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -69,3 +69,42 @@ func WrapIndex1D(sh *Shape, i ...int) int { } return sh.IndexTo1D(ai...) } + +// AlignForAssign ensures that the shapes of two tensors, a and b +// have the proper alignment for assigning b into a. +// Alignment proceeds from the innermost dimension out, with 1s provided +// beyond the number of dimensions for a or b. +// An error is returned if the rules of alignment are violated: +// each dimension size must be either the same, or b is equal to 1. +// This corresponds to the "broadcasting" logic of NumPy. +func AlignForAssign(a, b Tensor) (as, bs *Shape, err error) { + asz := a.ShapeSizes() + bsz := b.ShapeSizes() + an := len(asz) + bn := len(bsz) + n := max(an, bn) + asizes := make([]int, n) + bsizes := make([]int, n) + for d := range n { + ai := an - 1 - d + bi := bn - 1 - d + oi := n - 1 - d + ad := 1 + bd := 1 + if ai >= 0 { + ad = asz[ai] + } + if bi >= 0 { + bd = bsz[bi] + } + if ad != bd && bd != 1 { + err = fmt.Errorf("tensor.AlignShapes: dimension %d does not align for a=%d b=%d: must be either the same or b is a 1", oi, ad, bd) + return + } + asizes[oi] = ad + bsizes[oi] = bd + } + as = NewShape(asizes...) + bs = NewShape(bsizes...) + return +} diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 0ea56df491..9fa7b73ba8 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -9,12 +9,135 @@ import ( ) func init() { + tensor.AddFunc("Assign", Assign, 0) + tensor.AddFunc("AddAssign", AddAssign, 0) + tensor.AddFunc("SubAssign", SubAssign, 0) + tensor.AddFunc("MulAssign", MulAssign, 0) + tensor.AddFunc("DivAssign", DivAssign, 0) + + tensor.AddFunc("Inc", Inc, 0) + tensor.AddFunc("Dec", Dec, 0) + tensor.AddFunc("Add", Add, 1) tensor.AddFunc("Sub", Sub, 1) tensor.AddFunc("Mul", Mul, 1) tensor.AddFunc("Div", Div, 1) } +// Assign assigns values from b into a. +func Assign(a, b tensor.Tensor) error { + as, bs, err := tensor.AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + ai := as.IndexFrom1D(idx) + bi := tensor.WrapIndex1D(bs, ai...) + tsr[0].SetFloat1D(tsr[1].Float1D(bi), idx) + }, a, b) + return nil +} + +// AddAssign does += add assign values from b into a. +func AddAssign(a, b tensor.Tensor) error { + as, bs, err := tensor.AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + ai := as.IndexFrom1D(idx) + bi := tensor.WrapIndex1D(bs, ai...) + tsr[0].SetFloat1D(tsr[0].Float1D(idx)+tsr[1].Float1D(bi), idx) + }, a, b) + return nil +} + +// SubAssign does -= sub assign values from b into a. +func SubAssign(a, b tensor.Tensor) error { + as, bs, err := tensor.AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + ai := as.IndexFrom1D(idx) + bi := tensor.WrapIndex1D(bs, ai...) + tsr[0].SetFloat1D(tsr[0].Float1D(idx)-tsr[1].Float1D(bi), idx) + }, a, b) + return nil +} + +// MulAssign does *= mul assign values from b into a. +func MulAssign(a, b tensor.Tensor) error { + as, bs, err := tensor.AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + ai := as.IndexFrom1D(idx) + bi := tensor.WrapIndex1D(bs, ai...) + tsr[0].SetFloat1D(tsr[0].Float1D(idx)*tsr[1].Float1D(bi), idx) + }, a, b) + return nil +} + +// DivAssign does /= divide assign values from b into a. +func DivAssign(a, b tensor.Tensor) error { + as, bs, err := tensor.AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + ai := as.IndexFrom1D(idx) + bi := tensor.WrapIndex1D(bs, ai...) + tsr[0].SetFloat1D(tsr[0].Float1D(idx)/tsr[1].Float1D(bi), idx) + }, a, b) + return nil +} + +// Inc increments values in given tensor by 1. +func Inc(a tensor.Tensor) error { + alen := a.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + tsr[0].SetFloat1D(tsr[0].Float1D(idx)+1.0, idx) + }, a) + return nil +} + +// Dec decrements values in given tensor by 1. +func Dec(a tensor.Tensor) error { + alen := a.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return alen + }, + func(idx int, tsr ...tensor.Tensor) { + tsr[0].SetFloat1D(tsr[0].Float1D(idx)-1.0, idx) + }, a) + return nil +} + // Add adds two tensors into output. func Add(a, b, out tensor.Tensor) error { as, bs, os, err := tensor.AlignShapes(a, b) From 48563a69b2bd9ef3819f8265663546ef77fc91dc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 02:49:48 -0700 Subject: [PATCH 122/311] yaegi update --- goal/transpile/transpile_test.go | 46 +++++++++---------- .../cogentcore_org-core-tensor-tmath.go | 7 +++ .../symbols/cogentcore_org-core-tensor.go | 1 + 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index f8667b79de..d03e9ec8c5 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -203,29 +203,29 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - // {"# x := 1", `x := tensor.NewIntScalar(1)`}, - // {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - // {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - // {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, - // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - // {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - // {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - // {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - // {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, - // {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - // {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, - // {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - // {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, - // {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, - // {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - // {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, - // {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, - // {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, - // {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.Reshape(a, 36))`}, - // {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tensor.CallOut("Sub", tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), tensor.CallOut("stats.Mean", ra))`}, - // {"# a[:, 2] = b", `tensor.Call("Assign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# x := 1", `x := tensor.NewIntScalar(1)`}, + {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, + {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, + {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, + {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, + {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.Reshape(a, 36))`}, + {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tensor.CallOut("Sub", tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), tensor.CallOut("stats.Mean", ra))`}, + {"# a[:, 2] = b", `tensor.Call("Assign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, {"# a[:, 2] += b", `tensor.Call("AddAssign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go index 3f442d0e68..cd596d5bbb 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -14,15 +14,19 @@ func init() { "Acos": reflect.ValueOf(tmath.Acos), "Acosh": reflect.ValueOf(tmath.Acosh), "Add": reflect.ValueOf(tmath.Add), + "AddAssign": reflect.ValueOf(tmath.AddAssign), "Asin": reflect.ValueOf(tmath.Asin), "Asinh": reflect.ValueOf(tmath.Asinh), + "Assign": reflect.ValueOf(tmath.Assign), "Atan": reflect.ValueOf(tmath.Atan), "Atanh": reflect.ValueOf(tmath.Atanh), "Cbrt": reflect.ValueOf(tmath.Cbrt), "Ceil": reflect.ValueOf(tmath.Ceil), "Cos": reflect.ValueOf(tmath.Cos), "Cosh": reflect.ValueOf(tmath.Cosh), + "Dec": reflect.ValueOf(tmath.Dec), "Div": reflect.ValueOf(tmath.Div), + "DivAssign": reflect.ValueOf(tmath.DivAssign), "Erf": reflect.ValueOf(tmath.Erf), "Erfc": reflect.ValueOf(tmath.Erfc), "Erfcinv": reflect.ValueOf(tmath.Erfcinv), @@ -32,6 +36,7 @@ func init() { "Expm1": reflect.ValueOf(tmath.Expm1), "Floor": reflect.ValueOf(tmath.Floor), "Gamma": reflect.ValueOf(tmath.Gamma), + "Inc": reflect.ValueOf(tmath.Inc), "J0": reflect.ValueOf(tmath.J0), "J1": reflect.ValueOf(tmath.J1), "Log": reflect.ValueOf(tmath.Log), @@ -40,12 +45,14 @@ func init() { "Log2": reflect.ValueOf(tmath.Log2), "Logb": reflect.ValueOf(tmath.Logb), "Mul": reflect.ValueOf(tmath.Mul), + "MulAssign": reflect.ValueOf(tmath.MulAssign), "Round": reflect.ValueOf(tmath.Round), "RoundToEven": reflect.ValueOf(tmath.RoundToEven), "Sin": reflect.ValueOf(tmath.Sin), "Sinh": reflect.ValueOf(tmath.Sinh), "Sqrt": reflect.ValueOf(tmath.Sqrt), "Sub": reflect.ValueOf(tmath.Sub), + "SubAssign": reflect.ValueOf(tmath.SubAssign), "Tan": reflect.ValueOf(tmath.Tan), "Tanh": reflect.ValueOf(tmath.Tanh), "Trunc": reflect.ValueOf(tmath.Trunc), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index d8c4c104f6..2ec835c2a4 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -13,6 +13,7 @@ func init() { // function, constant and variable definitions "AddFunc": reflect.ValueOf(tensor.AddFunc), "AddShapes": reflect.ValueOf(tensor.AddShapes), + "AlignForAssign": reflect.ValueOf(tensor.AlignForAssign), "AlignShapes": reflect.ValueOf(tensor.AlignShapes), "AnyFirstArg": reflect.ValueOf(tensor.AnyFirstArg), "As1D": reflect.ValueOf(tensor.As1D), From a7d95b938071fa866bf571f7e38639b6bf90c5c8 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 03:18:16 -0700 Subject: [PATCH 123/311] ## math mode toggling --- goal/interpreter/interpreter.go | 8 ++++++-- goal/transpile/state.go | 7 +++++-- goal/transpile/transpile.go | 13 +++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index 191087644e..09efaab334 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -76,10 +76,14 @@ func NewInterpreter(options interp.Options) *Interpreter { // Prompt returns the appropriate REPL prompt to show the user. func (in *Interpreter) Prompt() string { dp := in.Goal.TrState.TotalDepth() + pc := ">" + if in.Goal.TrState.MathMode { + pc = "#" + } if dp == 0 { - return in.Goal.HostAndDir() + " > " + return in.Goal.HostAndDir() + " " + pc + " " } - res := "> " + res := pc + " " for range dp { res += " " // note: /t confuses readline } diff --git a/goal/transpile/state.go b/goal/transpile/state.go index 48682a105e..ad687ab3ad 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -27,6 +27,9 @@ type State struct { // this should be turned off. FuncToVar bool + // MathMode is on when math mode is turned on. + MathMode bool + // depth of delim at the end of the current line. if 0, was complete. ParenDepth, BraceDepth, BrackDepth, TypeDepth, DeclDepth int @@ -36,10 +39,10 @@ type State struct { Chunks []string // current stack of transpiled lines, that are accumulated into - // code Chunks + // code Chunks. Lines []string - // stack of runtime errors + // stack of runtime errors. Errors []error // if this is non-empty, it is the name of the last command defined. diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 246c70267b..79a58d9869 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -47,6 +47,15 @@ func (st *State) TranspileLineTokens(code string) Tokens { if n == 0 { return toks } + if st.MathMode { + if len(toks) >= 2 { + if toks[0].Tok == token.ILLEGAL && toks[0].Str == "#" && toks[1].Tok == token.ILLEGAL && toks[1].Str == "#" { + st.MathMode = false + return nil + } + } + return st.TranspileMath(toks, code, true) + } ewords, err := ExecWords(code) if err != nil { st.AddError(err) @@ -76,6 +85,10 @@ func (st *State) TranspileLineTokens(code string) Tokens { case t0.Tok == token.ILLEGAL: if t0.Str == "#" { logx.PrintlnDebug("math #") + if toks[1].Tok == token.ILLEGAL && toks[1].Str == "#" { + st.MathMode = true + return nil + } return st.TranspileMath(toks[1:], code, true) } return st.TranspileExec(ewords, false) From 2d80a8316d0609c34b0274af3bd5a8023e2c4ef4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 14:22:28 -0700 Subject: [PATCH 124/311] reshape, ndim, copy etc working as methods and functions --- goal/README.md | 191 +++++++++--------- goal/transpile/math.go | 116 ++++++++--- goal/transpile/transpile_test.go | 2 + tensor/convert.go | 10 +- .../symbols/cogentcore_org-core-tensor.go | 1 + 5 files changed, 191 insertions(+), 129 deletions(-) diff --git a/goal/README.md b/goal/README.md index ce39543aee..02cb8b136e 100644 --- a/goal/README.md +++ b/goal/README.md @@ -221,9 +221,9 @@ TODO: update aboven # Math mode -In general, Goal is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. +In general, Goal is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. -All elements of a Goal math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an "array" in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that Goal does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately. +All elements of a Goal math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an `ndarray` in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that Goal does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately via the [matrix](../tensor/matrix) package. The _view_ versions of `Tensor` include `Sliced`, `Reshaped`, `Masked`, `Indexed`, and `Rows`, each of which wraps around another "source" `Tensor`, and provides its own way of accessing the underlying data: @@ -239,115 +239,114 @@ The _view_ versions of `Tensor` include `Sliced`, `Reshaped`, `Masked`, `Indexe Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) -| Goal | Python | MATLAB | Notes | -| -------------------------- | ------ | ------ | ------ | -| `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of tensor `a` | -| `len(a)` or `a.len` or `size(a)` or `a.size` | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | -| `shape(a)` or `a.shape` | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | -| `a.shape[n-1]` | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of tensor `a` | -| `tensor.NewReshaped(a, 10, 2)` or `a.reshape([10, 2])` or `reshape(a, [10, 2])` or: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major. | -| `tensor.Clone(tensor.As1D(a))` or: | `y = x.flatten()` | `y=x(:)` | turn tensor into a 1D vector (forces a copy) |. -| | | | -| **Construction** | | | +| tensor | Goal | Python | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | +| `a.NumDim()` | `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of tensor `a` | +| `a.Len()` | `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | +| `a.Shape().Sizes` | same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | +| `a.Shape().Sizes[n-1]` | same: | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of tensor `a` | +| `tensor.Reshape(a, 10, 2)` | `a.reshape([10, 2])` or `reshape(a, [10, 2])` or: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major. | +| `tensor.Reshape(a, -1)` or `tensor.As1D(a)` | same: | `a.reshape(-1)` or `np.reshape(a, -1)` | `reshape(a,-1)` | a 1D vector view of `a`; Goal does not support `ravel`, which is nearly identical. | +| `tensor.Flatten(a)` | same: | `b = a.flatten()` | `b=a(:)` | turn tensor into a 1D vector, and force a copy | +| `b := tensor.Clone(a)` | `b := copy(a)` or: | `b = a.copy()` | `b=a` | direct assignment `b = a` in Goal or NumPy just makes variable b point to tensor a; `copy` is needed to generate new underlying values (MATLAB always makes a copy) | +| | | | | +| **Construction** | | | | | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | | `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | | `zeros([3,4]` or `zeros(3, 4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional tensor of float64 zeros | | `zeros([3, 4, 5])` or `zeros(3, 4, 5)` | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional tensor of float64 zeros | | `ones([3, 4])` or `ones(3, 4)` | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional tensor of 64-bit floating point ones | | `rand([3, 4])` or `slrand([3,4], c, fi)` | `rng.random(3, 4)` | `rand(3,4)` | 3x4 2D float64 tensor with uniform random 0..1 elements; `rand` uses current Go `rand` source, while `slrand` uses [gosl](../gpu/gosl/slrand) GPU-safe call with counter `c` and function index `fi` and key = index of element | -| | `np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | -| | `np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | -| | `np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | -| | `a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | -| | `a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | -| | | | -| **Ranges and Grids** [numpy](https://numpy.org/doc/stable/user/how-to-partition.html) | | | -| | `np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector | -| | `np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector | -| | `np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | -| | `np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | -| | `np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | -| | `ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | -| | `np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | -| | `np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | -| | | | -| **Basic Indexing** | | | -| | `a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | -| | `a[-1]` | `a(end)` | access last element | -| | `a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | -| | `a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | -| | `a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | -| | `a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | -| | `a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | -| | `a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | -| | `a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | -| | | | -| **Advanced Indexing** | | | -| if indexes are themselves an array | then advanced indexing takes place | | indexes are parallel lists of dimension coordinates; not the clearest | -| | `a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | -| | | | -| **Boolean Tensors and Indexing** | | | +| | |`np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | +| | |`np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | +| | |`np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | +| | |`a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | +| | |`a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | +| | | | | +| **Ranges and Grids** [numpy](https://numpy.org/doc/stable/user/how-to-partition.html) | | | | +| | |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector | +| | |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector | +| | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | +| | |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | +| | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | +| | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | +| | |`np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | +| | |`np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | +| | | | | +| **Basic Indexing** | | | | +| | |`a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | +| | |`a[-1]` | `a(end)` | access last element | +| | |`a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | +| | |`a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | +| | |`a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | +| | |`a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | +| | |`a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | +| | |`a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | +| | |`a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | +| `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | +| | | | | +| **Boolean Tensors and Indexing** | | | | | (bool tensor of same shape can filter access to other tensor) | | | | `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | | `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | | `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | | `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | -| | `a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | -| | `a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | -| | | | -| **Indexed Filtering and Sorting** | | | +| | |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | +| | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | +| | | | | +| **Advanced Indexing** | | | | +| if indexes are themselves an array | then advanced indexing takes place | | indexes are parallel lists of dimension coordinates; not the clearest | +| | |`a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | +| | | | | +| **Indexed Filtering and Sorting** | | | | | (indexes only on outer row dim) | index on all elements | | | -| | `np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | -| | `a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of `a` where column vector `v` > 0.5 | -| | `a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of `a` where vector `v` > 0.5 | -| | `a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | -| | `np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D tensor, `a` | -| | `np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, `a` | -| | `I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the tensor `a` as tensor `b` with rows sorted by the first column | -| | `np.unique(a)` | `unique(a)` | a vector of unique values in tensor `a` | -| | | | -| **Copy vs. View** | | | -| `y = x.copy()` or `y = x.Clone()` | `y = x.copy()` | `y=x` | `y=x` just assigns `y` to point to `x`, so that changes to `y` also change `x`; need to make a `copy` to get distinct values | -| `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | -| | | | -| **Math** | | | -| | `a * b` | `a .* b` | element-wise multiply | -| | `a/b` | `a./b` | element-wise divide | +| | |`np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | +| | |`a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of `a` where column vector `v` > 0.5 | +| | |`a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of `a` where vector `v` > 0.5 | +| | |`a[:] = 3` | `a(:) = 3` | set all values to the same scalar value | +| | |`np.sort(a)` or `a.sort(axis=0)` | `sort(a)` | sort each column of a 2D tensor, `a` | +| | |`np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, `a` | +| | |`I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the tensor `a` as tensor `b` with rows sorted by the first column | +| | |`np.unique(a)` | `unique(a)` | a vector of unique values in tensor `a` | +| | | | | +| **Math** | | | | +| | |`a * b` | `a .* b` | element-wise multiply | +| | |`a/b` | `a./b` | element-wise divide | | `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | | `cos(a)` | `cos(a)` | `cos(a)` | element-wise function application | -| | | | -| **2D Matrix Linear Algebra** | | | -| (n-dimensional tensors resliced as lists of 2D matricies) | | | -| | `a @ b` | `a * b` | matrix multiply | -| | `a.transpose() or a.T` | `a.'` | transpose of a | -| | `a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | -| | `np.eye(3)` | `eye(3)` | 3x3 identity matrix | -| | `np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D tensor, `a` | -| | `np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | -| | `linalg.inv(a)` | `inv(a)` | inverse of square 2D tensor a | -| | `linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D tensor a | -| | `np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D tensor a | -| | `linalg.solve(a, b)` if `a` is square; `linalg.lstsq(a, b)` otherwise | `a\b` | solution of `a x = b` for x | -| | Solve `a.T x.T = b.T` instead | `b/a` | solution of x a = b for x | -| | `U, S, Vh = linalg.svd(a); V = Vh.T` | `[U,S,V]=svd(a)` | singular value decomposition of a | -| | `linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D tensor | -| | `D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | -| | `D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D tensor, a | -| | `Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition -| | `P,L,U = linalg.lu(a)` where `a == P@L@U` | `[L,U,P]=lu(a)` where `a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | -| | | | -| **Statistics** | | | +| | | | | +| **2D Matrix Linear Algebra** | | | | +| (n-dimensional tensors resliced as lists of 2D matricies) | | | | +| | |`a @ b` | `a * b` | matrix multiply | +| | |`a.transpose() or a.T` | `a.'` | transpose of a | +| | |`a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | +| | |`np.eye(3)` | `eye(3)` | 3x3 identity matrix | +| | |`np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D tensor, `a` | +| | |`np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | +| | |`linalg.inv(a)` | `inv(a)` | inverse of square 2D tensor a | +| | |`linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D tensor a | +| | |`np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D tensor a | +| | |`linalg.solve(a, b)` if `a` is square; `linalg.lstsq(a, b)` otherwise | `a\b` | solution of `a x = b` for x | +| | |Solve `a.T x.T = b.T` instead | `b/a` | solution of x a = b for x | +| | |`U, S, Vh = linalg.svd(a); V = Vh.T` | `[U,S,V]=svd(a)` | singular value decomposition of a | +| | |`linalg.cholesky(a)` | `chol(a)` | Cholesky factorization of a 2D tensor | +| | |`D,V = linalg.eig(a)` | `[V,D]=eig(a)` | eigenvalues and eigenvectors of `a`, where `[V,D]=eig(a,b)` eigenvalues and eigenvectors of `a, b` where | +| | |`D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D tensor, a | +| | |`Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition +| | |`P,L,U = linalg.lu(a)` where `a == P@L@U` | `[L,U,P]=lu(a)` where `a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | +| | | | | +| **Statistics** | | | | | `a.max()` or `max(a)` or `stats.Max(a)` | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of `a`, Goal always ignores `NaN` as missing data | -| | `a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | -| | `a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | -| | `np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | +| | |`a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | +| | |`a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | +| | |`np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | | `stats.L2Norm(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | -| | `cg` | `conjgrad` | conjugate gradients solver | -| | | | -| **Misc Functions** | | | -| | `np.fft.fft(a)` | `fft(a)` | Fourier transform of `a` | -| | `np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of `a` | -| | `x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | -| | `signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | +| | |`cg` | `conjgrad` | conjugate gradients solver | +| | | | | +| **Misc Functions** | | | | +| | |`np.fft.fft(a)` | `fft(a)` | Fourier transform of `a` | +| | |`np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of `a` | +| | |`x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | +| | |`signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index d13b67ac22..9b1053c6fa 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -366,47 +366,56 @@ func (mp *mathParse) tensorLit(lit *ast.BasicLit) { } } +// funWrap is a function wrapper for simple numpy property / functions type funWrap struct { - fun string - wrap string + fun string // function to call on tensor + wrap string // code for wrapping function for results of call } -// nis: NewIntScalar, nifs: NewIntFromValues +// nis: NewIntScalar, niv: NewIntFromValues, etc var numpyProps = map[string]funWrap{ "ndim": {"NumDims()", "nis"}, "len": {"Len()", "nis"}, "size": {"Len()", "nis"}, - "shape": {"Shape().Sizes", "nifs"}, + "shape": {"Shape().Sizes", "niv"}, } -func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { - fw, ok := numpyProps[ex.Sel.Name] - if !ok { - mp.expr(ex.X) - mp.addToken(token.PERIOD) - mp.out.Add(token.IDENT, ex.Sel.Name) - mp.idx++ - return - } +// tensorFunc outputs the wrapping function and whether it needs ellipsis +func (fw *funWrap) wrapFunc(mp *mathParse) bool { ellip := false + wrapFun := fw.wrap switch fw.wrap { case "nis": - mp.out.Add(token.IDENT, "tensor.NewIntScalar") + wrapFun = "tensor.NewIntScalar" case "nfs": - mp.out.Add(token.IDENT, "tensor.NewFloat64Scalar") + wrapFun = "tensor.NewFloat64Scalar" case "nss": - mp.out.Add(token.IDENT, "tensor.NewStringScalar") - case "nifs": - mp.out.Add(token.IDENT, "tensor.NewIntFromValues") + wrapFun = "tensor.NewStringScalar" + case "niv": + wrapFun = "tensor.NewIntFromValues" ellip = true - case "nffs": - mp.out.Add(token.IDENT, "tensor.NewFloat64FromValues") + case "nfv": + wrapFun = "tensor.NewFloat64FromValues" ellip = true - case "nsfs": - mp.out.Add(token.IDENT, "tensor.NewStringFromValues") + case "nsv": + wrapFun = "tensor.NewStringFromValues" ellip = true } + mp.startFunc(wrapFun, false) mp.out.Add(token.LPAREN) + return ellip +} + +func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { + fw, ok := numpyProps[ex.Sel.Name] + if !ok { + mp.expr(ex.X) + mp.addToken(token.PERIOD) + mp.out.Add(token.IDENT, ex.Sel.Name) + mp.idx++ + return + } + ellip := fw.wrapFunc(mp) mp.expr(ex.X) mp.addToken(token.PERIOD) mp.out.Add(token.IDENT, fw.fun) @@ -415,6 +424,7 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { mp.out.Add(token.ELLIPSIS) } mp.out.Add(token.RPAREN) + mp.endFunc() } func (mp *mathParse) indexListExpr(il *ast.IndexListExpr) { @@ -510,15 +520,27 @@ var numpyFuncs = map[string]funWrap{ "zeros": {"tensor.NewFloat64", ""}, "arange": {"tensor.NewSliceInts", ""}, "reshape": {"tensor.Reshape", ""}, + "copy": {"tensor.Clone", ""}, + "flatten": {"tensor.Flatten", ""}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { switch x := ex.Fun.(type) { case *ast.Ident: - mp.callName(ex, x.Name) + if fw, ok := numpyProps[x.Name]; ok { + mp.callPropFun(ex, fw) + return + } + mp.callName(ex, x.Name, "") case *ast.SelectorExpr: if pkg, ok := x.X.(*ast.Ident); ok { - mp.callName(ex, pkg.Name+"."+x.Sel.Name) + fun := x.Sel.Name + if fw, ok := numpyFuncs[fun]; ok { + mp.callPropSelFun(ex, pkg.Name, fw) + return + } else { + mp.callName(ex, fun, pkg.Name) + } } else { fmt.Printf("call, weird sel: %#v\n", x.X) } @@ -531,21 +553,51 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { mp.endFunc() } -func (mp *mathParse) callName(ex *ast.CallExpr, funName string) { +// this calls a "prop" function like ndim(a) on the object. +func (mp *mathParse) callPropFun(ex *ast.CallExpr, fw funWrap) { + ellip := fw.wrapFunc(mp) + mp.idx += 2 + mp.exprList(ex.Args) // this is the tensor + mp.addToken(token.PERIOD) + mp.out.Add(token.IDENT, fw.fun) + if ellip { + mp.out.Add(token.ELLIPSIS) + } + mp.out.Add(token.RPAREN) + mp.endFunc() +} + +// this calls global function through selector like: a.reshape() +func (mp *mathParse) callPropSelFun(ex *ast.CallExpr, obj string, fw funWrap) { + mp.startFunc(fw.fun, false) + mp.addToken(token.LPAREN) // use the ( + mp.out.Add(token.IDENT, obj) + mp.idx += 2 + mp.addToken(token.COMMA) + mp.exprList(ex.Args) + mp.addToken(token.RPAREN) + mp.endFunc() +} + +func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { if fw, ok := numpyFuncs[funName]; ok { - // todo: wrap mp.startFunc(fw.fun, false) mp.addToken(token.LPAREN) // use the ( mp.idx++ // paren too return } - _, err := tensor.FuncByName(funName) - if err != nil { - funName = strings.ToUpper(funName[:1]) + funName[1:] + var err error // validate name + if pkgName != "" { + funName = pkgName + "." + funName + _, err = tensor.FuncByName(funName) + } else { _, err = tensor.FuncByName(funName) + if err != nil { + funName = strings.ToUpper(funName[:1]) + funName[1:] // first letter uppercased + _, err = tensor.FuncByName(funName) + } } - if err != nil { - fmt.Println("name not found:", funName) + if err != nil { // not a registered tensor function mp.startFunc(funName, false) mp.addToken(token.LPAREN) // use the ( mp.idx++ @@ -553,7 +605,7 @@ func (mp *mathParse) callName(ex *ast.CallExpr, funName string) { } mp.startFunc("tensor.CallOut", true) // tensors mp.addToken(token.LPAREN) - if strings.Contains(funName, ".") { + if pkgName != "" { mp.idx += 2 // . and selector } mp.out.Add(token.IDENT, `"`+funName+`"`) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index d03e9ec8c5..d54c8bffdb 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -209,11 +209,13 @@ func TestMath(t *testing.T) { {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, diff --git a/tensor/convert.go b/tensor/convert.go index d8b32bfcc6..8bf8eb8bd0 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -13,6 +13,7 @@ import ( // Clone returns a copy of the given tensor. // If it is raw [Values] then a [Values.Clone] is returned. // Otherwise if it is a view, then [Tensor.AsValues] is returned. +// This is equivalent to the NumPy copy function. func Clone(tsr Tensor) Values { if vl, ok := tsr.(Values); ok { return vl.Clone() @@ -20,6 +21,13 @@ func Clone(tsr Tensor) Values { return tsr.AsValues() } +// Flatten returns a copy of the given tensor as a 1D flat list +// of values, by calling Clone(As1D(tsr)). +// It is equivalent to the NumPy flatten function. +func Flatten(tsr Tensor) Values { + return Clone(As1D(tsr)) +} + // MustBeValues returns the given tensor as a [Values] subtype, or nil and // an error if it is not one. Typically outputs of compute operations must // be values, and are reshaped to hold the results as needed. @@ -79,7 +87,7 @@ func SetShapeSizesMustBeValues(tsr Tensor, sizes ...int) error { // As1D returns a 1D tensor, which is either the input tensor if it is // already 1D, or a new [Reshaped] 1D view of it. // This can be useful e.g., for stats and metric functions that operate -// on a 1D list of values. +// on a 1D list of values. See also [Flatten]. func As1D(tsr Tensor) Tensor { if tsr.NumDims() == 1 { return tsr diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 2ec835c2a4..94362845ee 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -51,6 +51,7 @@ func init() { "Descending": reflect.ValueOf(tensor.Descending), "Detect": reflect.ValueOf(tensor.Detect), "Ellipsis": reflect.ValueOf(tensor.Ellipsis), + "Flatten": reflect.ValueOf(tensor.Flatten), "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), "Float64ToString": reflect.ValueOf(tensor.Float64ToString), "FullAxis": reflect.ValueOf(tensor.FullAxis), From 547517cd585dd4987cb36b9851b6b1a8144d13d5 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 14:40:16 -0700 Subject: [PATCH 125/311] reorganize table into separate sections --- goal/README.md | 93 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/goal/README.md b/goal/README.md index 02cb8b136e..b78db24b23 100644 --- a/goal/README.md +++ b/goal/README.md @@ -233,24 +233,31 @@ The _view_ versions of `Tensor` include `Sliced`, `Reshaped`, `Masked`, `Indexe * `Indexed` uses a tensor of indexes where the final, innermost dimension is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. - The current NumPy version of indexed is rather complex and difficult for many people to understand, as articulated in this [NEP 21 proposal](https://numpy.org/neps/nep-0021-advanced-indexing.html). We probably want to fix. + The current NumPy version of indexed is rather complex and difficult for many people to understand, as articulated in this [NEP 21 proposal](https://numpy.org/neps/nep-0021-advanced-indexing.html). The `Indexed` view at least provides a simpler way of representing the indexes into the source tensor, instead of requiring multiple parallel 1D arrays. * `Rows` is an optimized version of `Sliced` with indexes only for the first, outermost, _row_ dimension. -Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html) +The following sections provide a full list of equivalents between the `tensor` Go code, Goal, NumPy, and MATLAB, based on the table in [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html). +* The _same:_ in Goal means that the same NumPy syntax works in Goal, minus the `np.` prefix, and likewise for _or:_ (where Goal also has additional syntax). -| tensor | Goal | Python | MATLAB | Notes | +## Tensor shape + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | | `a.NumDim()` | `ndim(a)` or `a.ndim` | `np.ndim(a)` or `a.ndim` | `ndims(a)` | number of dimensions of tensor `a` | | `a.Len()` | `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | | `a.Shape().Sizes` | same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | -| `a.Shape().Sizes[n-1]` | same: | `a.shape[n-1]` | `size(a,n)` | the number of elements of the n-th dimension of tensor `a` | +| `a.Shape().Sizes[1]` | same: | `a.shape[1]` | `size(a,2)` | the number of elements of the 2nd dimension of tensor `a` | | `tensor.Reshape(a, 10, 2)` | `a.reshape([10, 2])` or `reshape(a, [10, 2])` or: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major. | | `tensor.Reshape(a, -1)` or `tensor.As1D(a)` | same: | `a.reshape(-1)` or `np.reshape(a, -1)` | `reshape(a,-1)` | a 1D vector view of `a`; Goal does not support `ravel`, which is nearly identical. | | `tensor.Flatten(a)` | same: | `b = a.flatten()` | `b=a(:)` | turn tensor into a 1D vector, and force a copy | | `b := tensor.Clone(a)` | `b := copy(a)` or: | `b = a.copy()` | `b=a` | direct assignment `b = a` in Goal or NumPy just makes variable b point to tensor a; `copy` is needed to generate new underlying values (MATLAB always makes a copy) | -| | | | | -| **Construction** | | | | + + +## Constructing new tensors + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | | `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | | `zeros([3,4]` or `zeros(3, 4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional tensor of float64 zeros | @@ -262,8 +269,13 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | |`np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | | | |`a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | | | |`a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | -| | | | | -| **Ranges and Grids** [numpy](https://numpy.org/doc/stable/user/how-to-partition.html) | | | | + +## Ranges and grids + +See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for details. + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | | |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector | | | |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector | | | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | @@ -272,8 +284,13 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | | | |`np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | | | |`np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | -| | | | | -| **Basic Indexing** | | | | + +## Basic indexing + +See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.html#basic-indexing) + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | | |`a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | | | |`a[-1]` | `a(end)` | access last element | | | |`a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | @@ -284,9 +301,13 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | |`a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | | | |`a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | | `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | -| | | | | -| **Boolean Tensors and Indexing** | | | | -| (bool tensor of same shape can filter access to other tensor) | | | + +## Boolean tensors and indexing + +See [NumPy boolean indexing](https://numpy.org/doc/stable/user/basics.indexing.html#boolean-array-indexing) + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | | `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | @@ -294,13 +315,14 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | | | |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | | | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | -| | | | | -| **Advanced Indexing** | | | | -| if indexes are themselves an array | then advanced indexing takes place | | indexes are parallel lists of dimension coordinates; not the clearest | + +## Advanced index-based indexing + +See [NumPy integer indexing](https://numpy.org/doc/stable/user/basics.indexing.html#integer-array-indexing). Note that the current NumPy version of indexed is rather complex and difficult for many people to understand, as articulated in this [NEP 21 proposal](https://numpy.org/neps/nep-0021-advanced-indexing.html). + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | | |`a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | -| | | | | -| **Indexed Filtering and Sorting** | | | | -| (indexes only on outer row dim) | index on all elements | | | | | |`np.nonzero(a > 0.5)` | `find(a > 0.5)` | find the indices where (a > 0.5) | | | |`a[:, v.T > 0.5]` | `a(:,find(v>0.5))` | extract the columns of `a` where column vector `v` > 0.5 | | | |`a[:,np.nonzero(v > 0.5)[0]]` | `a(:,find(v > 0.5))` | extract the columns of `a` where vector `v` > 0.5 | @@ -309,15 +331,20 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | |`np.sort(a, axis=1)` or `a.sort(axis=1)` | `sort(a, 2)` | sort the each row of 2D tensor, `a` | | | |`I = np.argsort(a[:, 0]); b = a[I,:]` | `[b,I]=sortrows(a,1)` | save the tensor `a` as tensor `b` with rows sorted by the first column | | | |`np.unique(a)` | `unique(a)` | a vector of unique values in tensor `a` | -| | | | | -| **Math** | | | | + +## Basic math operations (add, multiply, etc) + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | | |`a * b` | `a .* b` | element-wise multiply | | | |`a/b` | `a./b` | element-wise divide | | `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | | `cos(a)` | `cos(a)` | `cos(a)` | element-wise function application | -| | | | | -| **2D Matrix Linear Algebra** | | | | -| (n-dimensional tensors resliced as lists of 2D matricies) | | | | + +## 2D Matrix Linear Algebra + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | | |`a @ b` | `a * b` | matrix multiply | | | |`a.transpose() or a.T` | `a.'` | transpose of a | | | |`a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | @@ -335,18 +362,26 @@ Here's a full list of equivalents, from [numpy-for-matlab-users](https://numpy.o | | |`D,V = eigs(a, k=3)` | `D,V = linalg.eig(a, b)` | `[V,D]=eigs(a,3)` | find the k=3 largest eigenvalues and eigenvectors of 2D tensor, a | | | |`Q,R = linalg.qr(a)` | `[Q,R]=qr(a,0)` | QR decomposition | | |`P,L,U = linalg.lu(a)` where `a == P@L@U` | `[L,U,P]=lu(a)` where `a==P'*L*U` | LU decomposition with partial pivoting (note: P(MATLAB) == transpose(P(NumPy))) | -| | | | | -| **Statistics** | | | | +| | |`x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | + +## Statistics + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | `a.max()` or `max(a)` or `stats.Max(a)` | `a.max()` or `np.nanmax(a)` | `max(max(a))` | maximum element of `a`, Goal always ignores `NaN` as missing data | | | |`a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | | | |`a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | | | |`np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | | `stats.L2Norm(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | | | |`cg` | `conjgrad` | conjugate gradients solver | -| | | | | -| **Misc Functions** | | | | + +## FFT and complex numbers + +todo: huge amount of work needed to support complex numbers throughout! + +| `tensor` Go | Goal | NumPy | MATLAB | Notes | +| ------------ | ----------- | ------ | ------ | ---------------- | | | |`np.fft.fft(a)` | `fft(a)` | Fourier transform of `a` | | | |`np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of `a` | -| | |`x = linalg.lstsq(Z, y)` | `x = Z\y` | perform a linear regression of the form | | | |`signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | From 7e3e899ff0eda23c8993587bab7d2b09d40cd4c6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 15:55:47 -0700 Subject: [PATCH 126/311] ones, full functions --- goal/README.md | 29 +++++---- goal/transpile/math.go | 21 ++++++- goal/transpile/transpile_test.go | 3 + tensor/convert.go | 62 +++++++++++++++++++ tensor/matrix/README.md | 8 +++ .../symbols/cogentcore_org-core-tensor.go | 7 +++ 6 files changed, 116 insertions(+), 14 deletions(-) diff --git a/goal/README.md b/goal/README.md index b78db24b23..ec2ca6101e 100644 --- a/goal/README.md +++ b/goal/README.md @@ -248,27 +248,30 @@ The following sections provide a full list of equivalents between the `tensor` G | `a.Len()` | `len(a)` or `a.len` or: | `np.size(a)` or `a.size` | `numel(a)` | number of elements of tensor `a` | | `a.Shape().Sizes` | same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | | `a.Shape().Sizes[1]` | same: | `a.shape[1]` | `size(a,2)` | the number of elements of the 2nd dimension of tensor `a` | -| `tensor.Reshape(a, 10, 2)` | `a.reshape([10, 2])` or `reshape(a, [10, 2])` or: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major. | +| `tensor.Reshape(a, 10, 2)` | same except no `a.shape = (10,2)`: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major; Goal does _not_ support direct shape assignment version. | +| `tensor.Reshape(a, tensor.AsIntSlice(sh)...)` | same: | `a.reshape(10, sh)` or `np.reshape(a, sh)` | `reshape(a,sh)` | **TODO:** set shape based on list of dimension sizes in tensor `sh` | | `tensor.Reshape(a, -1)` or `tensor.As1D(a)` | same: | `a.reshape(-1)` or `np.reshape(a, -1)` | `reshape(a,-1)` | a 1D vector view of `a`; Goal does not support `ravel`, which is nearly identical. | | `tensor.Flatten(a)` | same: | `b = a.flatten()` | `b=a(:)` | turn tensor into a 1D vector, and force a copy | | `b := tensor.Clone(a)` | `b := copy(a)` or: | `b = a.copy()` | `b=a` | direct assignment `b = a` in Goal or NumPy just makes variable b point to tensor a; `copy` is needed to generate new underlying values (MATLAB always makes a copy) | +| | |`a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | -## Constructing new tensors +## Constructing | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | -| `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | -| `zeros([3,4]` or `zeros(3, 4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 two-dimensional tensor of float64 zeros | -| `zeros([3, 4, 5])` or `zeros(3, 4, 5)` | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional tensor of float64 zeros | -| `ones([3, 4])` or `ones(3, 4)` | `np.ones((3, 4))` | `ones(3,4)` | 3x4 two-dimensional tensor of 64-bit floating point ones | -| `rand([3, 4])` or `slrand([3,4], c, fi)` | `rng.random(3, 4)` | `rand(3,4)` | 3x4 2D float64 tensor with uniform random 0..1 elements; `rand` uses current Go `rand` source, while `slrand` uses [gosl](../gpu/gosl/slrand) GPU-safe call with counter `c` and function index `fi` and key = index of element | -| | |`np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | -| | |`np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | -| | |`np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | -| | |`a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | -| | |`a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | +| `tensor.NewFloat64FromValues([]float64{1, 2, 3})` | `[1., 2., 3.]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | +| TODO: (does reshape of flat literal) | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | +| TODO: | | `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | +| `tensor.NewFloat64(3,4)` | `zeros(3,4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 2D tensor of float64 zeros; Goal does not use "tuple" so no double parens | +| `tensor.NewFloat64(3,4,5)` | `zeros(3, 4, 5)` | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional tensor of float64 zeros | +| `tensor.NewFloat64Ones(3,4)` | `ones(3, 4)` | `np.ones((3, 4))` | `ones(3,4)` | 3x4 2D tensor of 64-bit floating point ones | +| `tensor.NewFloat64Full(5.5, 3,4)` | `full(5.5, 3, 4)` | `np.full((3, 4), 5.5)` | ? | 3x4 2D tensor of 5.5; Goal variadic arg structure requires value to come first | +| TODO: | | `rand([3, 4])` or `slrand([3,4], c, fi)` | `rng.random(3, 4)` | `rand(3,4)` | 3x4 2D float64 tensor with uniform random 0..1 elements; `rand` uses current Go `rand` source, while `slrand` uses [gosl](../gpu/gosl/slrand) GPU-safe call with counter `c` and function index `fi` and key = index of element | +| TODO: | |`np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | +| TODO: | |`np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | +| TODO: | |`np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | +| TODO: | |`a[np.r_[:len(a),0]]` | `a([1:end 1],:)` | `a` with copy of the first row appended to the end | ## Ranges and grids diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 9b1053c6fa..013416892d 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -505,6 +505,11 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { typ = "string" fun = "String" } + // cfun := mp.funcs.Peek() + // if cfun != nil && !cfun.tensorArgs { // we need int values + // mp.exprList(il.Indices) + // return + // } mp.startFunc("tensor.New"+fun+"FromValues", false) mp.out.Add(token.LPAREN) mp.out.Add(token.IDENT, "[]"+typ) @@ -517,7 +522,10 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { } var numpyFuncs = map[string]funWrap{ + "array": {"tensor.NewFloat64", ""}, "zeros": {"tensor.NewFloat64", ""}, + "full": {"tensor.NewFloat64Full", ""}, + "ones": {"tensor.NewFloat64Ones", ""}, "arange": {"tensor.NewSliceInts", ""}, "reshape": {"tensor.Reshape", ""}, "copy": {"tensor.Clone", ""}, @@ -616,6 +624,17 @@ func (mp *mathParse) ident(id *ast.Ident) { if id == nil { return } - // fmt.Println("ident:", x.Name) + /* TODO: this requires tracking of each arg to determine needed type + cfun := mp.funcs.Peek() + if cfun != nil && !cfun.tensorArgs { // we need the numbers from it, usually ints + mp.out.Add(token.IDENT, "tensor.AsIntSlice") + mp.out.Add(token.LPAREN) + mp.addCur() + mp.out.Add(token.RPAREN) + mp.out.Add(token.ELLIPSIS) + } else { + mp.addCur() + } + */ mp.addCur() } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index d54c8bffdb..a6b7a06be9 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -211,9 +211,12 @@ func TestMath(t *testing.T) { {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, + // {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, // TODO: not yet {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + // {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, // TODO: not yet {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, diff --git a/tensor/convert.go b/tensor/convert.go index 8bf8eb8bd0..e8c3303316 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -149,6 +149,68 @@ func NewStringFromValues(vals ...string) *String { return tsr } +// SetAllFloat64 sets all values of given tensor to given value. +func SetAllFloat64(tsr Tensor, val float64) { + VectorizeThreaded(1, func(tsr ...Tensor) int { + return tsr[0].Len() + }, + func(idx int, tsr ...Tensor) { + tsr[0].SetFloat1D(val, idx) + }, tsr) +} + +// SetAllInt sets all values of given tensor to given value. +func SetAllInt(tsr Tensor, val int) { + VectorizeThreaded(1, func(tsr ...Tensor) int { + return tsr[0].Len() + }, + func(idx int, tsr ...Tensor) { + tsr[0].SetInt1D(val, idx) + }, tsr) +} + +// SetAllString sets all values of given tensor to given value. +func SetAllString(tsr Tensor, val string) { + VectorizeThreaded(1, func(tsr ...Tensor) int { + return tsr[0].Len() + }, + func(idx int, tsr ...Tensor) { + tsr[0].SetString1D(val, idx) + }, tsr) +} + +// NewFloat64Full returns a new tensor full of given scalar value, +// of given shape sizes. +func NewFloat64Full(val float64, sizes ...int) *Float64 { + tsr := NewFloat64(sizes...) + SetAllFloat64(tsr, val) + return tsr +} + +// NewFloat64Ones returns a new tensor full of 1s, +// of given shape sizes. +func NewFloat64Ones(sizes ...int) *Float64 { + tsr := NewFloat64(sizes...) + SetAllFloat64(tsr, 1.0) + return tsr +} + +// NewIntFull returns a new tensor full of given scalar value, +// of given shape sizes. +func NewIntFull(val int, sizes ...int) *Int { + tsr := NewInt(sizes...) + SetAllInt(tsr, val) + return tsr +} + +// NewStringFull returns a new tensor full of given scalar value, +// of given shape sizes. +func NewStringFull(val string, sizes ...int) *String { + tsr := NewString(sizes...) + SetAllString(tsr, val) + return tsr +} + // AsFloat64Scalar returns the first value of tensor as a float64 scalar. // Returns 0 if no values. func AsFloat64Scalar(tsr Tensor) float64 { diff --git a/tensor/matrix/README.md b/tensor/matrix/README.md index df190c2731..c27c88a623 100644 --- a/tensor/matrix/README.md +++ b/tensor/matrix/README.md @@ -2,4 +2,12 @@ This package provides interfaces for `Tensor` types to the [gonum](https://github.com/gonum/gonum) functions for linear algebra, defined on the 2D `mat.Matrix` interface. +# TODO + +Add following functions here: + +* `eye` +* `identity` +* `diag` +* `diagonal` diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 94362845ee..23fad658ce 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -71,12 +71,15 @@ func init() { "NewFloat32": reflect.ValueOf(tensor.NewFloat32), "NewFloat64": reflect.ValueOf(tensor.NewFloat64), "NewFloat64FromValues": reflect.ValueOf(tensor.NewFloat64FromValues), + "NewFloat64Full": reflect.ValueOf(tensor.NewFloat64Full), + "NewFloat64Ones": reflect.ValueOf(tensor.NewFloat64Ones), "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), "NewFunc": reflect.ValueOf(tensor.NewFunc), "NewIndexed": reflect.ValueOf(tensor.NewIndexed), "NewInt": reflect.ValueOf(tensor.NewInt), "NewInt32": reflect.ValueOf(tensor.NewInt32), "NewIntFromValues": reflect.ValueOf(tensor.NewIntFromValues), + "NewIntFull": reflect.ValueOf(tensor.NewIntFull), "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), "NewMasked": reflect.ValueOf(tensor.NewMasked), "NewOfType": reflect.ValueOf(tensor.NewOfType), @@ -89,6 +92,7 @@ func init() { "NewSliced": reflect.ValueOf(tensor.NewSliced), "NewString": reflect.ValueOf(tensor.NewString), "NewStringFromValues": reflect.ValueOf(tensor.NewStringFromValues), + "NewStringFull": reflect.ValueOf(tensor.NewStringFull), "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), "NewStringShape": reflect.ValueOf(tensor.NewStringShape), "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), @@ -109,6 +113,9 @@ func init() { "Reslice": reflect.ValueOf(tensor.Reslice), "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), "SaveCSV": reflect.ValueOf(tensor.SaveCSV), + "SetAllFloat64": reflect.ValueOf(tensor.SetAllFloat64), + "SetAllInt": reflect.ValueOf(tensor.SetAllInt), + "SetAllString": reflect.ValueOf(tensor.SetAllString), "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), "SetPrecision": reflect.ValueOf(tensor.SetPrecision), "SetShape": reflect.ValueOf(tensor.SetShape), From 9810245a1d06f466bd58f7bd88a037954d7d6834 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 16:13:42 -0700 Subject: [PATCH 127/311] squeeze --- goal/README.md | 6 +- goal/transpile/math.go | 8 ++- tensor/convert.go | 61 ++++++++++++------- .../symbols/cogentcore_org-core-tensor.go | 1 + 4 files changed, 48 insertions(+), 28 deletions(-) diff --git a/goal/README.md b/goal/README.md index ec2ca6101e..9d9b282f40 100644 --- a/goal/README.md +++ b/goal/README.md @@ -251,16 +251,16 @@ The following sections provide a full list of equivalents between the `tensor` G | `tensor.Reshape(a, 10, 2)` | same except no `a.shape = (10,2)`: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major; Goal does _not_ support direct shape assignment version. | | `tensor.Reshape(a, tensor.AsIntSlice(sh)...)` | same: | `a.reshape(10, sh)` or `np.reshape(a, sh)` | `reshape(a,sh)` | **TODO:** set shape based on list of dimension sizes in tensor `sh` | | `tensor.Reshape(a, -1)` or `tensor.As1D(a)` | same: | `a.reshape(-1)` or `np.reshape(a, -1)` | `reshape(a,-1)` | a 1D vector view of `a`; Goal does not support `ravel`, which is nearly identical. | -| `tensor.Flatten(a)` | same: | `b = a.flatten()` | `b=a(:)` | turn tensor into a 1D vector, and force a copy | +| `tensor.Flatten(a)` | same: | `b = a.flatten()` | `b=a(:)` | returns a 1D copy of a | | `b := tensor.Clone(a)` | `b := copy(a)` or: | `b = a.copy()` | `b=a` | direct assignment `b = a` in Goal or NumPy just makes variable b point to tensor a; `copy` is needed to generate new underlying values (MATLAB always makes a copy) | -| | |`a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. Note that MATLAB will always return tensors of 2D or higher while NumPy will return tensors of 0D or higher | +| `tensor.Squeeze(a)` | same: |`a.squeeze()` | `squeeze(a)` | remove singleton dimensions of tensor `a`. | ## Constructing | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| `tensor.NewFloat64FromValues([]float64{1, 2, 3})` | `[1., 2., 3.]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | +| `tensor.NewFloat64FromValues(` `[]float64{1, 2, 3})` | `[1., 2., 3.]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | | TODO: (does reshape of flat literal) | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | | TODO: | | `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | | `tensor.NewFloat64(3,4)` | `zeros(3,4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 2D tensor of float64 zeros; Goal does not use "tuple" so no double parens | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 013416892d..1cef5e5068 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -521,21 +521,23 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { mp.endFunc() } +// nofun = do not accept a function version, just a method var numpyFuncs = map[string]funWrap{ - "array": {"tensor.NewFloat64", ""}, + // "array": {"tensor.NewFloatFromValues", ""}, // todo: probably not right, maybe don't have? "zeros": {"tensor.NewFloat64", ""}, "full": {"tensor.NewFloat64Full", ""}, "ones": {"tensor.NewFloat64Ones", ""}, "arange": {"tensor.NewSliceInts", ""}, "reshape": {"tensor.Reshape", ""}, "copy": {"tensor.Clone", ""}, - "flatten": {"tensor.Flatten", ""}, + "flatten": {"tensor.Flatten", "nofun"}, + "squeeze": {"tensor.Squeeze", "nofun"}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { switch x := ex.Fun.(type) { case *ast.Ident: - if fw, ok := numpyProps[x.Name]; ok { + if fw, ok := numpyProps[x.Name]; ok && fw.wrap != "nofun" { mp.callPropFun(ex, fw) return } diff --git a/tensor/convert.go b/tensor/convert.go index e8c3303316..2977e2e22b 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -28,6 +28,45 @@ func Flatten(tsr Tensor) Values { return Clone(As1D(tsr)) } +// Squeeze a [Reshaped] view of given tensor with all singleton +// (size = 1) dimensions removed (if none, just returns the tensor). +func Squeeze(tsr Tensor) Tensor { + nd := tsr.NumDims() + sh := tsr.ShapeSizes() + reshape := make([]int, 0, nd) + for _, sz := range sh { + if sz > 1 { + reshape = append(reshape, sz) + } + } + if len(reshape) == nd { + return tsr + } + return NewReshaped(tsr, reshape...) +} + +// As1D returns a 1D tensor, which is either the input tensor if it is +// already 1D, or a new [Reshaped] 1D view of it. +// This can be useful e.g., for stats and metric functions that operate +// on a 1D list of values. See also [Flatten]. +func As1D(tsr Tensor) Tensor { + if tsr.NumDims() == 1 { + return tsr + } + return NewReshaped(tsr, tsr.Len()) +} + +// Cells1D returns a flat 1D view of the innermost cells for given row index. +// For a [RowMajor] tensor, it uses the [RowTensor] subspace directly, +// otherwise it uses [Sliced] to extract the cells. In either case, +// [As1D] is used to ensure the result is a 1D tensor. +func Cells1D(tsr Tensor, row int) Tensor { + if rm, ok := tsr.(RowMajor); ok { + return As1D(rm.RowTensor(row)) + } + return As1D(NewSliced(tsr, []int{row})) +} + // MustBeValues returns the given tensor as a [Values] subtype, or nil and // an error if it is not one. Typically outputs of compute operations must // be values, and are reshaped to hold the results as needed. @@ -84,28 +123,6 @@ func SetShapeSizesMustBeValues(tsr Tensor, sizes ...int) error { return nil } -// As1D returns a 1D tensor, which is either the input tensor if it is -// already 1D, or a new [Reshaped] 1D view of it. -// This can be useful e.g., for stats and metric functions that operate -// on a 1D list of values. See also [Flatten]. -func As1D(tsr Tensor) Tensor { - if tsr.NumDims() == 1 { - return tsr - } - return NewReshaped(tsr, tsr.Len()) -} - -// Cells1D returns a flat 1D view of the innermost cells for given row index. -// For a [RowMajor] tensor, it uses the [RowTensor] subspace directly, -// otherwise it uses [Sliced] to extract the cells. In either case, -// [As1D] is used to ensure the result is a 1D tensor. -func Cells1D(tsr Tensor, row int) Tensor { - if rm, ok := tsr.(RowMajor); ok { - return As1D(rm.RowTensor(row)) - } - return As1D(NewSliced(tsr, []int{row})) -} - // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. func NewFloat64Scalar(val float64) *Float64 { diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 23fad658ce..7f233a7f53 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -129,6 +129,7 @@ func init() { "SlicesMagicValues": reflect.ValueOf(tensor.SlicesMagicValues), "Space": reflect.ValueOf(tensor.Space), "Sprint": reflect.ValueOf(tensor.Sprint), + "Squeeze": reflect.ValueOf(tensor.Squeeze), "StableSort": reflect.ValueOf(tensor.StableSort), "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), "Tab": reflect.ValueOf(tensor.Tab), From 6ef251291015d8f3158e0891a145c095b6d835b3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 25 Sep 2024 16:30:11 -0700 Subject: [PATCH 128/311] slicing docs --- goal/README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/goal/README.md b/goal/README.md index 9d9b282f40..4a207ed458 100644 --- a/goal/README.md +++ b/goal/README.md @@ -277,6 +277,8 @@ The following sections provide a full list of equivalents between the `tensor` G See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for details. +TODO: + | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | | | |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector | @@ -290,20 +292,20 @@ See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for de ## Basic indexing -See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.html#basic-indexing) +See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.html#basic-indexing). Tensor Go uses the `Reslice` function for all cases (repeated `tensor.` prefix replaced with `t.` to take less space). | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| | |`a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | -| | |`a[-1]` | `a(end)` | access last element | -| | |`a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` | -| | |`a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | same as Go slice ranging | -| | |`a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | -| | |`a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | -| | |`a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | -| | |`a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | -| | |`a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | -| `y = x[1, :].copy()` or `y = x[1, :].Clone()` | `y = x[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | +| `t.Reslice(a, 1, 4)` | same: |`a[1, 4]` | `a(2,5)` | access element in second row, fifth column in 2D tensor `a` | +| `t.Reslice(a, -1)` | same: |`a[-1]` | `a(end)` | access last element | +| `t.Reslice(a,` `1, t.FullAxis)` | same: |`a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` (could omit second arg in Reslice too) | +| `t.Reslice(a,` `Slice{Stop:5})` | same: |`a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | 0..4 rows of `a`; uses same Go slice ranging here: (start:stop) where stop is _exclusive_ | +| `t.Reslice(a,` `Slice{Start:-5})` | same: |`a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | +| `t.Reslice(a,` `Slice{Stop:3},` `Slice{Start:4, Stop:9})` | same: |`a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | +| `t.Reslice(a,` `Slice{Start:2,` `Stop:25,` `Step:2}, t.FullAxis)` | same: |`a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | +| `t.Reslice(a,` `Slice{Step:2},` `t.FullAxis)` | same: |`a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | +| `t.Reslice(a,`, `Slice{Step:-1},` `t.FullAxis)` | same: |`a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | +| `t.Clone(t.Reslice(a,` `1, t.FullAxis))` | `b = copy(a[1, :])` or: | `b = a[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | ## Boolean tensors and indexing From 144c40b5ee7f7c2375b3d36eb9133e282dc044f1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 03:14:06 -0700 Subject: [PATCH 129/311] full revamp of tensor function calling: remove call by string, just call directly -- much better for arbitrary method signatures. bad side is need for current Out functions and new version that returns output. --- goal/README.md | 4 +- goal/transpile/math.go | 12 +- goal/transpile/transpile_test.go | 21 +- tensor/README.md | 4 +- tensor/convert.go | 21 -- tensor/funcs.go | 225 ++++---------- tensor/funcs_test.go | 29 +- tensor/matrix/matrix.go | 4 +- tensor/stats/cluster/clust_test.go | 7 +- tensor/stats/cluster/cluster.go | 2 +- tensor/stats/metric/funcs.go | 183 +++++++++--- tensor/stats/metric/matrix.go | 147 ++++++---- tensor/stats/metric/metric_test.go | 59 ++-- tensor/stats/metric/metrics.go | 46 +-- tensor/stats/metric/misc.go | 11 +- tensor/stats/metric/vec.go | 6 +- tensor/stats/stats/describe.go | 3 +- tensor/stats/stats/funcs.go | 209 +++++++++++-- tensor/stats/stats/group.go | 20 +- tensor/stats/stats/norm.go | 61 +++- tensor/stats/stats/quantiles.go | 48 ++- tensor/stats/stats/stats.go | 52 ++-- tensor/stats/stats/stats_test.go | 112 ++++--- tensor/stats/stats/vec.go | 12 +- tensor/tmath/math.go | 454 +++++++++++++++++------------ tensor/tmath/math_test.go | 4 +- tensor/tmath/ops.go | 118 ++++++-- tensor/tmath/ops_test.go | 100 +++++-- 28 files changed, 1157 insertions(+), 817 deletions(-) diff --git a/goal/README.md b/goal/README.md index 4a207ed458..ed86324e3a 100644 --- a/goal/README.md +++ b/goal/README.md @@ -292,7 +292,7 @@ TODO: ## Basic indexing -See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.html#basic-indexing). Tensor Go uses the `Reslice` function for all cases (repeated `tensor.` prefix replaced with `t.` to take less space). +See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.html#basic-indexing). Tensor Go uses the `Reslice` function for all cases (repeated `tensor.` prefix replaced with `t.` to take less space). Here you can clearly see the advantage of Goal in allowing significantly more succinct expressions to be written for accomplishing critical tensor functionality. | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | @@ -306,6 +306,8 @@ See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.htm | `t.Reslice(a,` `Slice{Step:2},` `t.FullAxis)` | same: |`a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | | `t.Reslice(a,`, `Slice{Step:-1},` `t.FullAxis)` | same: |`a[::-1,:]` | `a(end:-1:1,:) or flipud(a)` | `a` with rows in reverse order | | `t.Clone(t.Reslice(a,` `1, t.FullAxis))` | `b = copy(a[1, :])` or: | `b = a[1, :].copy()` | `y=x(2,:)` | without the copy, `y` would point to a view of values in `x`; `copy` creates distinct values, in this case of _only_ the 2nd row of `x` -- i.e., it "concretizes" a given view into a literal, memory-continuous set of values for that view. | +| `tmath.Assign(` `t.Reslice(a,` `Slice{Stop:5}),` `t.NewIntScalar(2))` | same: |`a[:5] = 2` | `a(1:5,:) = 2` | assign the value 2 to 0..4 rows of `a` | +| (you get the idea) | same: |`a[:5] = b[:, :5]` | `a(1:5,:) = b(:, 1:5)` | assign the values in the first 5 columns of `b` to the first 5 rows of `a` | ## Boolean tensors and indexing diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 1cef5e5068..71b9984649 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -279,10 +279,8 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { case token.QUO: fn = "Div" } - mp.startFunc("tensor.CallOut", true) // yes tensor args + mp.startFunc("tmath."+fn, true) // yes tensor args mp.out.Add(token.LPAREN) - mp.out.Add(token.STRING, `"`+fn+`"`) - mp.out.Add(token.COMMA) mp.expr(ex.X) mp.out.Add(token.COMMA) mp.idx++ @@ -326,10 +324,8 @@ func (mp *mathParse) assignStmt(as *ast.AssignStmt) { case token.QUO_ASSIGN: fn = "DivAssign" } - mp.startFunc("tensor.Call", true) // yes tensor args + mp.startFunc("tmath."+fn, true) // yes tensor args mp.out.Add(token.LPAREN) - mp.out.Add(token.STRING, `"`+fn+`"`) - mp.out.Add(token.COMMA) mp.exprList(as.Lhs) mp.out.Add(token.COMMA) mp.idx++ @@ -613,13 +609,11 @@ func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { mp.idx++ return } - mp.startFunc("tensor.CallOut", true) // tensors + mp.startFunc(funName, true) // tensors mp.addToken(token.LPAREN) if pkgName != "" { mp.idx += 2 // . and selector } - mp.out.Add(token.IDENT, `"`+funName+`"`) - mp.addToken(token.COMMA) // use the name -- need more } func (mp *mathParse) ident(id *ast.Ident) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index a6b7a06be9..118c3f915d 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,9 +204,9 @@ func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tensor.CallOut("Add", a, tensor.NewIntScalar(1))`}, - {"# x = x * 4", `x = tensor.CallOut("Mul", x, tensor.NewIntScalar(4))`}, - {"# a = x + y", `a = tensor.CallOut("Add", x, y)`}, + {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tmath.Add(x, y)`}, {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, @@ -225,13 +225,14 @@ func TestMath(t *testing.T) { {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - {"# c := cos(a)", `c := tensor.CallOut("Cos", a)`}, - {"# m := stats.Mean(a)", `m := tensor.CallOut("stats.Mean", a)`}, - {"# m := (stats.Mean(a))", `m := (tensor.CallOut("stats.Mean", a))`}, - {"# m := stats.Mean(reshape(a,36))", `m := tensor.CallOut("stats.Mean", tensor.Reshape(a, 36))`}, - {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tensor.CallOut("Sub", tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), tensor.CallOut("stats.Mean", ra))`}, - {"# a[:, 2] = b", `tensor.Call("Assign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - {"# a[:, 2] += b", `tensor.Call("AddAssign", tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# c := cos(a)", `c := tmath.Cos(a)`}, + {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, + {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, + {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, + {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, + {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, } st := NewState() diff --git a/tensor/README.md b/tensor/README.md index 0fdb28e410..7a0a8a7eb7 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -28,7 +28,9 @@ There is also a `RowMajor` sub-interface for tensors (implemented by the `Values The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, and a varargs list of tensors. In general it is completely up to the compute function how to interpret the index, although we also support the "broadcasting" principles from NumPy for binary functions operating on two tensors, as discussed below. There is a Threaded version of this for parallelizable functions, and a GPU version in the [gosl](../gpu/gosl) Go-as-a-shading-language package. -All tensor package functions are registered using a global name-to-function map (`Funcs`), and can be called by name via `tensor.Call` or `tensor.CallOut` (creates the appropriate output tensor for you). Standard enumerated functions in `stats` and `metrics` have a `FuncName` method that appends the package name, which is how they are registered and called. These functions are the equivalent of the [ufunc](https://numpy.org/doc/stable/user/basics.ufuncs.html) universal functions in NumPy. +All tensor package functions are registered using a global name-to-function map (`Funcs`), which can be used to retrieve the function by name, along with relevant metadata used by [goal](../goal). This allows functions to be called via enums, in the `stats` and `metrics` packages, for example. These functions are the equivalent of the [ufunc](https://numpy.org/doc/stable/user/basics.ufuncs.html) universal functions in NumPy. + +To support the best possible performance in compute-intensive code, we have written all the core tensor functions in an `Out` suffixed version that takes the output tensor as an additional input argument (it must be a `Values` type), which allows an appropriately sized tensor to be used to hold the outputs on repeated function calls, instead of requiring new memory allocations every time. These versions are used in other calls where appropriate. The function without the `Out` suffix just wraps the `Out` version, and is what is called directly by Goal, where the output return value is essential for proper chaining of operations. * [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. diff --git a/tensor/convert.go b/tensor/convert.go index 2977e2e22b..fb0a17d8a2 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -102,27 +102,6 @@ func SetShapeFrom(vals Values, from Tensor) { vals.SetShapeSizes(from.ShapeSizes()...) } -// SetShapeFromMustBeValues sets shape of given tensor from a source tensor, -// calling [MustBeValues] on the destination tensor to ensure it is a [Values], -// type, returning an error if not. This is used extensively for output -// tensors in functions, and all such output tensors _must_ be Values tensors. -func SetShapeFromMustBeValues(tsr Tensor, from Tensor) error { - return SetShapeSizesMustBeValues(tsr, from.ShapeSizes()...) -} - -// SetShapeSizesMustBeValues sets shape of given tensor from given sizes, -// calling [MustBeValues] on the destination tensor to ensure it is a [Values], -// type, returning an error if not. This is used extensively for output -// tensors in functions, and all such output tensors _must_ be Values tensors. -func SetShapeSizesMustBeValues(tsr Tensor, sizes ...int) error { - vals, err := MustBeValues(tsr) - if err != nil { - return err - } - vals.SetShapeSizes(sizes...) - return nil -} - // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. func NewFloat64Scalar(val float64) *Float64 { diff --git a/tensor/funcs.go b/tensor/funcs.go index 10a63eb954..e0121d3081 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -6,22 +6,16 @@ package tensor import ( "fmt" - "reflect" - "slices" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" ) -// AnyFirstArg should be used to set AnyFirst functions -const AnyFirstArg = true - // Func represents a registered tensor function, which has // In number of input Tensor arguments, and Out number of output // arguments (typically 1). There can also be an 'any' first // argument to support other kinds of parameters. -// This is used to make tensor functions available to the Goal -// language. +// This is used to make tensor functions available to the Goal language. type Func struct { // Name is the original CamelCase Go name for function Name string @@ -30,14 +24,11 @@ type Func struct { // args, with an optional any first arg. Fun any - // In is number of input args + // In is number of input tensor args In int - // Out is number of output args + // Out is number of output tensor args Out int - - // AnyFirst indicates if there is an 'any' first argument. - AnyFirst bool } // NewFunc creates a new Func desciption of the given @@ -51,16 +42,10 @@ type Func struct { // and an optional arg indicating an 'any' first argument. // The remaining arguments in the function (automatically // determined) are classified as input arguments. -func NewFunc(name string, fun any, out int, anyFirst ...bool) (*Func, error) { - fn := &Func{Name: name, Fun: fun, Out: out} - if len(anyFirst) == 1 && anyFirst[0] { - fn.AnyFirst = true - } - nargs := fn.ArgCount() - if out > nargs { - return nil, fmt.Errorf("tensor.NewFunc: too many output args for function %q, which takes %d (-1 means function signature is not recognized)", name, nargs) - } - fn.In = 1 - out +func NewFunc(name string, fun any) (*Func, error) { + fn := &Func{Name: name, Fun: fun} + // fn.In = 1 - out + // todo: get signature return fn, nil } @@ -78,7 +63,7 @@ var Funcs map[string]*Func // AddFunc adds given named function to the global tensor named function // registry, which is used by Goal to call functions by name. // See [NewFunc] for more information. -func AddFunc(name string, fun any, out int, anyFirst ...bool) error { +func AddFunc(name string, fun any) error { if Funcs == nil { Funcs = make(map[string]*Func) } @@ -86,7 +71,7 @@ func AddFunc(name string, fun any, out int, anyFirst ...bool) error { if ok { return errors.Log(fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name)) } - fn, err := NewFunc(name, fun, out, anyFirst...) + fn, err := NewFunc(name, fun) if errors.Log(err) != nil { return err } @@ -105,6 +90,8 @@ func FuncByName(name string) (*Func, error) { return fn, nil } +// todo: definitely switch over to reflection here: + // ArgCount returns the number of tensor arguments the function takes, // using a type switch. func (fn *Func) ArgCount() int { @@ -144,170 +131,60 @@ func (fn *Func) ArgCheck(n int, tsr ...Tensor) error { return nil } -// Call calls function with given set of input & output arguments -// appropriate for the given function (error if not). -func (fn *Func) Call(tsr ...Tensor) error { - if fn.AnyFirst { - return fmt.Errorf("tensor.Call: function %q: requires a first string argument", fn.Name) - } - switch f := fn.Fun.(type) { - case func(a Tensor) error: - if err := fn.ArgCheck(1, tsr...); err != nil { - return err - } - return f(tsr[0]) - case func(a, b Tensor) error: - if err := fn.ArgCheck(2, tsr...); err != nil { - return err - } - return f(tsr[0], tsr[1]) - case func(a, b, c Tensor) error: - if err := fn.ArgCheck(3, tsr...); err != nil { - return err - } - return f(tsr[0], tsr[1], tsr[2]) - case func(a, b, c, d Tensor) error: - if err := fn.ArgCheck(4, tsr...); err != nil { - return err - } - return f(tsr[0], tsr[1], tsr[2], tsr[3]) - case func(a, b, c, d, e Tensor) error: - if err := fn.ArgCheck(5, tsr...); err != nil { - return err - } - return f(tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) - } - return nil +// These generic functions provide a one liner for wrapping functions +// that take an output Tensor as the last argument, which is important +// for memory re-use of the output in performance-critical cases. +// The names indicate the number of input tensor arguments. +// Additional generic non-Tensor inputs are supported up to 2, +// with Gen1 and Gen2 versions. + +// CallOut1 adds output [Values] tensor for function. +func CallOut1(fun func(a Tensor, out Values) error, a Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(a, out)) + return out } -// CallAny calls function with given set of input & output arguments -// appropriate for the given function (error if not), -// with a first 'any' argument. -func (fn *Func) CallAny(first any, tsr ...Tensor) error { - if !fn.AnyFirst { - return fmt.Errorf("tensor.CallAny: function %q: does not take a first 'any' argument", fn.Name) - } - switch f := fn.Fun.(type) { - case func(first any, a Tensor) error: - if err := fn.ArgCheck(1, tsr...); err != nil { - return err - } - return f(first, tsr[0]) - case func(first any, a, b Tensor) error: - if err := fn.ArgCheck(2, tsr...); err != nil { - return err - } - return f(first, tsr[0], tsr[1]) - case func(first any, a, b, c Tensor) error: - if err := fn.ArgCheck(3, tsr...); err != nil { - return err - } - return f(first, tsr[0], tsr[1], tsr[2]) - case func(first any, a, b, c, d Tensor) error: - if err := fn.ArgCheck(4, tsr...); err != nil { - return err - } - return f(first, tsr[0], tsr[1], tsr[2], tsr[3]) - case func(first any, a, b, c, d, e Tensor) error: - if err := fn.ArgCheck(5, tsr...); err != nil { - return err - } - return f(first, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4]) - } - return nil +func CallOut2(fun func(a, b Tensor, out Values) error, a, b Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(a, b, out)) + return out } -// CallOut is like [Call] but it automatically creates an output -// tensor of the same type as the first input tensor passed, -// and returns the output as return values, along with any error. -func (fn *Func) CallOut(tsr ...Tensor) (Tensor, error) { - if fn.Out == 0 { - err := fn.Call(tsr...) - return nil, err - } - typ := reflect.Float64 - if fn.In > 0 { - typ = tsr[0].DataType() - } - out := NewOfType(typ) - tlist := slices.Clone(tsr) - tlist = append(tlist, out) - err := fn.Call(tlist...) - return out, err +func CallOut3(fun func(a, b, c Tensor, out Values) error, a, b, c Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(a, b, c, out)) + return out } -// CallOutMulti is like [CallOut] but deals with multiple output tensors. -func (fn *Func) CallOutMulti(tsr ...Tensor) ([]Tensor, error) { - if fn.Out == 0 { - err := fn.Call(tsr...) - return nil, err - } - typ := reflect.Float64 - if fn.In > 0 { - typ = tsr[0].DataType() - } - outs := make([]Tensor, fn.Out) - for i := range outs { - outs[i] = NewOfType(typ) - } - tsr = append(tsr, outs...) - err := fn.Call(tsr...) - return outs, err +func CallOut2Bool(fun func(a, b Tensor, out *Bool) error, a, b Tensor) *Bool { + out := NewBool() + errors.Log(fun(a, b, out)) + return out } -// Call calls function of given name, with given set of _input_ -// and output arguments appropriate for the given function. -// An error is logged if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match, or an error is returned by the function. -func Call(name string, tsr ...Tensor) error { - fn, err := FuncByName(name) - if err != nil { - return err - } - return fn.Call(tsr...) +func CallOut1Gen1[T any](fun func(g T, a Tensor, out Values) error, g T, a Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(g, a, out)) + return out } -// CallOut calls function of given name, with given set of _input_ -// arguments appropriate for the given function, returning a created -// output tensor, for the common case with just one return value. -// An error is logged if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match, or an error is returned by the function. -func CallOut(name string, tsr ...Tensor) Tensor { - fn, err := FuncByName(name) - if errors.Log(err) != nil { - return nil - } - return errors.Log1(fn.CallOut(tsr...)) +func CallOut1Gen2[T any, S any](fun func(g T, h S, a Tensor, out Values) error, g T, h S, a Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(g, h, a, out)) + return out } -// CallAny calls function of given name, with given set of arguments -// (any, input and output) appropriate for the given function. -// An error is returned if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match. This version of [Call] is for functions that -// have an initial string argument -func CallAny(name string, first any, tsr ...Tensor) error { - fn, err := FuncByName(name) - if err != nil { - return err - } - return fn.CallAny(first, tsr...) +func CallOut2Gen1[T any](fun func(g T, a, b Tensor, out Values) error, g T, a, b Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(g, a, b, out)) + return out } -// CallOutMulti calls function of given name, with given set of _input_ -// arguments appropriate for the given function, returning newly created -// output tensors, for the rare case of multiple return values. -// An error is logged if the function name has not been registered -// in the Funcs global function registry, or the argument count -// does not match, or an error is returned by the function. -func CallOutMulti(name string, tsr ...Tensor) []Tensor { - fn, err := FuncByName(name) - if errors.Log(err) != nil { - return nil - } - return errors.Log1(fn.CallOutMulti(tsr...)) +func CallOut2Gen2[T any, S any](fun func(g T, h S, a, b Tensor, out Values) error, g T, h S, a, b Tensor) Values { + out := NewOfType(a.DataType()) + errors.Log(fun(g, h, a, b, out)) + return out } // SetCalcFunc sets a function to calculate updated value for given tensor, diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 186e8b73f1..b77892e7fc 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -11,10 +11,9 @@ import ( "github.com/stretchr/testify/assert" ) -func abs(in, out Tensor) error { - if err := SetShapeFromMustBeValues(out, in); err != nil { - return err - } +// prototype of a simple compute function: +func absout(in Tensor, out Values) error { + SetShapeFrom(out, in) VectorizeThreaded(1, NFirstLen, func(idx int, tsr ...Tensor) { tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) @@ -22,33 +21,23 @@ func abs(in, out Tensor) error { } func TestFuncs(t *testing.T) { - err := AddFunc("Abs", abs, 1) + err := AddFunc("Abs", absout) assert.NoError(t, err) - err = AddFunc("Abs", abs, 1) - assert.Error(t, err) - - err = AddFunc("Abs3", abs, 3) - assert.Error(t, err) + err = AddFunc("Abs", absout) + assert.Error(t, err) // already vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} oned := NewNumberFromValues(vals...) oneout := oned.Clone() - fn, err := FuncByName("Abs") - assert.NoError(t, err) - err = fn.Call(oned, oneout) + _, err = FuncByName("Abs") assert.NoError(t, err) - assert.Equal(t, 1.507556722888818, oneout.Float1D(0)) + absout(oned, oneout) - err = fn.Call(oned) - assert.Error(t, err) - - out, err := fn.CallOut(oned) - assert.NoError(t, err) - assert.Equal(t, AsFloat64Scalar(oneout), AsFloat64Scalar(out)) + assert.Equal(t, 1.507556722888818, oneout.Float1D(0)) } func TestAlign(t *testing.T) { diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go index 87fda82fc0..796f3db14c 100644 --- a/tensor/matrix/matrix.go +++ b/tensor/matrix/matrix.go @@ -87,9 +87,9 @@ func (sy *Symmetric) SymmetricDim() (r int) { // CopyDense copies a gonum mat.Dense matrix into given Tensor // using standard Float64 interface -func CopyDense(to tensor.Tensor, dm *mat.Dense) { +func CopyDense(to tensor.Values, dm *mat.Dense) { nr, nc := dm.Dims() - tensor.SetShapeSizesMustBeValues(to, nr, nc) + to.SetShapeSizes(nr, nc) idx := 0 for ri := 0; ri < nr; ri++ { for ci := 0; ci < nc; ci++ { diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 47222c170b..2338df4b02 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -34,8 +34,7 @@ func TestClust(t *testing.T) { t.Error(err) } in := dt.Column("Input") - out := tensor.NewFloat64() - metric.Matrix(metric.Euclidean, in, out) + out := metric.Matrix(metric.Euclidean, in) cl := Cluster(Avg.String(), out, dt.Column("Name")) @@ -50,7 +49,7 @@ func TestClust(t *testing.T) { } gather(cl) - exdists := []float64{0, 9.181170003996987, 5.534356399283667, 4.859933131085473, 3.4641016151377544, 0, 0, 3.4641016151377544, 0, 0, 3.4641016151377544, 0, 0, 5.111664626761644, 4.640135790634417, 4, 0, 0, 3.4641016151377544, 0, 0, 3.605551275463989, 0, 0} + exdists := []float64{0, 9.181170119179619, 5.534356355667114, 4.859933137893677, 3.464101552963257, 0, 0, 3.464101552963257, 0, 0, 3.464101552963257, 0, 0, 5.111664593219757, 4.640135824680328, 4, 0, 0, 3.464101552963257, 0, 0, 3.605551242828369, 0, 0} - tolassert.EqualTolSlice(t, exdists, dists, 1.0e-8) + tolassert.EqualTolSlice(t, exdists, dists, 1.0e-7) } diff --git a/tensor/stats/cluster/cluster.go b/tensor/stats/cluster/cluster.go index 274ca79190..1a77daed82 100644 --- a/tensor/stats/cluster/cluster.go +++ b/tensor/stats/cluster/cluster.go @@ -114,7 +114,7 @@ func InitAllLeaves(ntot int) *Node { func Glom(root *Node, funcName string, dmat tensor.Tensor) *Node { ntot := dmat.DimSize(0) // number of leaves mout := tensor.NewFloat64Scalar(0) - stats.Max(tensor.As1D(dmat), mout) + stats.MaxOut(tensor.As1D(dmat), mout) maxd := mout.Float1D(0) // indexes in each group aidx := make([]int, ntot) diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 1afeb9652f..62da1e7f94 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -7,6 +7,7 @@ package metric import ( "math" + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/stats/stats" @@ -23,11 +24,17 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type MetricFunc = func(a, b, out tensor.Tensor) error +type MetricFunc = func(a, b tensor.Tensor) tensor.Values + +// MetricOutFunc is the function signature for a metric function, +// that takes output values as the final argument. See [MetricFunc]. +// This version is for computationally demanding cases and saves +// reallocation of output. +type MetricOutFunc = func(a, b tensor.Tensor, out tensor.Values) error // SumSquaresScaleOut64 computes the sum of squares differences between tensor values, // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. -func SumSquaresScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { +func SumSquaresScaleOut64(a, b tensor.Tensor, out tensor.Values) (scale64, ss64 tensor.Tensor, err error) { if err = tensor.MustBeSameShape(a, b); err != nil { return } @@ -41,7 +48,7 @@ func SumSquaresScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, // SumSquaresOut64 computes the sum of squares differences between tensor values, // and returns the Float64 output values for use in subsequent computations. -func SumSquaresOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { +func SumSquaresOut64(a, b tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { scale64, ss64, err := SumSquaresScaleOut64(a, b, out) if err != nil { return nil, err @@ -62,16 +69,24 @@ func SumSquaresOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { return scale64, err } -// SumSquares computes the sum of squares differences between tensor values, -// See [MetricFunc] for general information. -func SumSquares(a, b, out tensor.Tensor) error { +// SumSquaresOut computes the sum of squares differences between tensor values, +// See [MetricOutFunc] for general information. +func SumSquaresOut(a, b tensor.Tensor, out tensor.Values) error { _, err := SumSquaresOut64(a, b, out) return err } -// Euclidean computes the Euclidean square root of the sum of squares +// SumSquares computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func SumSquares(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(SumSquaresOut(a, b, out)) + return out +} + +// EuclideanOut computes the Euclidean square root of the sum of squares // differences between tensor values, aka the L2 Norm. -func Euclidean(a, b, out tensor.Tensor) error { +func EuclideanOut(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresScaleOut64(a, b, out) if err != nil { return err @@ -92,10 +107,18 @@ func Euclidean(a, b, out tensor.Tensor) error { return nil } -// Abs computes the sum of the absolute value of differences between the -// tensor values, aka the L1 Norm. +// Euclidean computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. -func Abs(a, b, out tensor.Tensor) error { +func Euclidean(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(EuclideanOut(a, b, out)) + return out +} + +// AbsOut computes the sum of the absolute value of differences between the +// tensor values, aka the L1 Norm. +// See [MetricOutFunc] for general information. +func AbsOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -107,10 +130,18 @@ func Abs(a, b, out tensor.Tensor) error { return err } -// Hamming computes the sum of 1s for every element that is different, -// i.e., "city block" distance. +// Abs computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. -func Hamming(a, b, out tensor.Tensor) error { +func Abs(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(AbsOut(a, b, out)) + return out +} + +// HammingOut computes the sum of 1s for every element that is different, +// i.e., "city block" distance. +// See [MetricOutFunc] for general information. +func HammingOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -125,10 +156,18 @@ func Hamming(a, b, out tensor.Tensor) error { return err } +// Hamming computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func Hamming(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(HammingOut(a, b, out)) + return out +} + // SumSquaresBinTolScaleOut64 computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. // returning scale and ss factors aggregated separately for better numerical stability, per BLAS. -func SumSquaresBinTolScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { +func SumSquaresBinTolScaleOut64(a, b tensor.Tensor, out tensor.Values) (scale64, ss64 tensor.Tensor, err error) { if err = tensor.MustBeSameShape(a, b); err != nil { return } @@ -144,10 +183,10 @@ func SumSquaresBinTolScaleOut64(a, b, out tensor.Tensor) (scale64, ss64 tensor.T return } -// EuclideanBinTol computes the Euclidean square root of the sum of squares +// EuclideanBinTolOut computes the Euclidean square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. -func EuclideanBinTol(a, b, out tensor.Tensor) error { +func EuclideanBinTolOut(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) if err != nil { return err @@ -168,9 +207,17 @@ func EuclideanBinTol(a, b, out tensor.Tensor) error { return nil } -// SumSquaresBinTol computes the sum of squares differences between tensor values, +// EuclideanBinTol computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func EuclideanBinTol(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(EuclideanBinTolOut(a, b, out)) + return out +} + +// SumSquaresBinTolOut computes the sum of squares differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. -func SumSquaresBinTol(a, b, out tensor.Tensor) error { +func SumSquaresBinTolOut(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) if err != nil { return err @@ -191,14 +238,22 @@ func SumSquaresBinTol(a, b, out tensor.Tensor) error { return nil } -// CrossEntropy is a standard measure of the difference between two +// SumSquaresBinTol computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func SumSquaresBinTol(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(SumSquaresBinTolOut(a, b, out)) + return out +} + +// CrossEntropyOut is a standard measure of the difference between two // probabilty distributions, reflecting the additional entropy (uncertainty) associated // with measuring probabilities under distribution b when in fact they come from // distribution a. It is also the entropy of a plus the divergence between a from b, // using Kullback-Leibler (KL) divergence. It is computed as: // a * log(a/b) + (1-a) * log(1-a/1-b). -// See [MetricFunc] for general information. -func CrossEntropy(a, b, out tensor.Tensor) error { +// See [MetricOutFunc] for general information. +func CrossEntropyOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -218,9 +273,17 @@ func CrossEntropy(a, b, out tensor.Tensor) error { return err } -// InnerProduct computes the sum of the co-products of the two on-NaN tensor values. +// CrossEntropy computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. -func InnerProduct(a, b, out tensor.Tensor) error { +func CrossEntropy(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(CrossEntropyOut(a, b, out)) + return out +} + +// InnerProductOut computes the sum of the co-products of the two on-NaN tensor values. +// See [MetricOutFunc] for general information. +func InnerProductOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -232,10 +295,18 @@ func InnerProduct(a, b, out tensor.Tensor) error { return err } -// Covariance computes the co-variance between two vectors, +// InnerProduct computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func InnerProduct(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(InnerProductOut(a, b, out)) + return out +} + +// CovarianceOut computes the co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus // the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. -func Covariance(a, b, out tensor.Tensor) error { +func CovarianceOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -261,6 +332,14 @@ func Covariance(a, b, out tensor.Tensor) error { return nil } +// Covariance computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func Covariance(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(CovarianceOut(a, b, out)) + return out +} + // CorrelationOut64 computes the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their @@ -268,7 +347,7 @@ func Covariance(a, b, out tensor.Tensor) error { // (i.e., the standardized covariance). // Equivalent to the cosine of mean-normalized vectors. // Returns the Float64 output values for subsequent use. -func CorrelationOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { +func CorrelationOut64(a, b tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { if err := tensor.MustBeSameShape(a, b); err != nil { return nil, err } @@ -304,18 +383,26 @@ func CorrelationOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { return ss64, nil } -// Correlation computes the correlation between two vectors, +// CorrelationOut computes the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized [CovarianceFunc]). // Equivalent to the [CosineFunc] of mean-normalized vectors. -func Correlation(a, b, out tensor.Tensor) error { +func CorrelationOut(a, b tensor.Tensor, out tensor.Values) error { _, err := CorrelationOut64(a, b, out) return err } -// InvCorrelation computes 1 minus the correlation between two vectors, +// Correlation computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func Correlation(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(CorrelationOut(a, b, out)) + return out +} + +// InvCorrelationOut computes 1 minus the correlation between two vectors, // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). @@ -323,7 +410,7 @@ func Correlation(a, b, out tensor.Tensor) error { // Equivalent to the [CosineFunc] of mean-normalized vectors. // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCorrelation(a, b, out tensor.Tensor) error { +func InvCorrelationOut(a, b tensor.Tensor, out tensor.Values) error { cor64, err := CorrelationOut64(a, b, out) if err != nil { return err @@ -336,10 +423,18 @@ func InvCorrelation(a, b, out tensor.Tensor) error { return nil } +// InvCorrelation computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func InvCorrelation(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(InvCorrelationOut(a, b, out)) + return out +} + // CosineOut64 computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized [InnerProductFunc]: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc]. -func CosineOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { +func CosineOut64(a, b tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { ss64, avar64, bvar64, err := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { ss += a * b @@ -364,20 +459,28 @@ func CosineOut64(a, b, out tensor.Tensor) (tensor.Tensor, error) { return ss64, nil } -// Cosine computes the high-dimensional angle between two vectors, +// CosineOut computes the high-dimensional angle between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). See also [CorrelationFunc] -func Cosine(a, b, out tensor.Tensor) error { +func CosineOut(a, b tensor.Tensor, out tensor.Values) error { _, err := CosineOut64(a, b, out) return err } -// InvCosine computes 1 minus the cosine between two vectors, +// Cosine computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func Cosine(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(CosineOut(a, b, out)) + return out +} + +// InvCosineOut computes 1 minus the cosine between two vectors, // in range (-1..1) as the normalized inner product: // inner product / sqrt(ssA * ssB). // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. -func InvCosine(a, b, out tensor.Tensor) error { +func InvCosineOut(a, b tensor.Tensor, out tensor.Values) error { cos64, err := CosineOut64(a, b, out) if err != nil { return err @@ -389,3 +492,11 @@ func InvCosine(a, b, out tensor.Tensor) error { } return nil } + +// InvCosine computes the sum of squares differences between tensor values, +// See [MetricFunc] for general information. +func InvCosine(a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(InvCosineOut(a, b, out)) + return out +} diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index f1de5e8fba..55811c619f 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -5,8 +5,7 @@ package metric import ( - "errors" - + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32/vecint" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/matrix" @@ -16,15 +15,15 @@ import ( ) func init() { - tensor.AddFunc("metric.Matrix", Matrix, 1, tensor.AnyFirstArg) - tensor.AddFunc("metric.CrossMatrix", CrossMatrix, 1, tensor.AnyFirstArg) - tensor.AddFunc("metric.CovarianceMatrix", CovarianceMatrix, 1, tensor.AnyFirstArg) - tensor.AddFunc("metric.PCA", PCA, 2) - tensor.AddFunc("metric.SVD", SVD, 2) - tensor.AddFunc("metric.ProjectOnMatrixColumn", ProjectOnMatrixColumn, 1) + tensor.AddFunc("metric.Matrix", Matrix) + tensor.AddFunc("metric.CrossMatrix", CrossMatrix) + tensor.AddFunc("metric.CovarianceMatrix", CovarianceMatrix) + tensor.AddFunc("metric.PCA", PCA) + tensor.AddFunc("metric.SVD", SVD) + tensor.AddFunc("metric.ProjectOnMatrixColumn", ProjectOnMatrixColumn) } -// Matrix computes the rows x rows square distance / similarity matrix +// MatrixOut computes the rows x rows square distance / similarity matrix // between the patterns for each row of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns (cells). Use [tensor.NewRowCellsView] @@ -34,7 +33,7 @@ func init() { // The results fill in the elements of the output matrix, which is symmetric, // and only the lower triangular part is computed, with results copied // to the upper triangular region, for maximum efficiency. -func Matrix(fun any, in, out tensor.Tensor) error { +func MatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { mfun, err := AsMetricFunc(fun) if err != nil { return err @@ -43,10 +42,7 @@ func Matrix(fun any, in, out tensor.Tensor) error { if rows == 0 || cells == 0 { return nil } - if err := tensor.SetShapeSizesMustBeValues(out, rows, rows); err != nil { - return err - } - mout := tensor.NewFloat64Scalar(0.0) + out.SetShapeSizes(rows, rows) coords := TriangularLIndicies(rows) nc := len(coords) // note: flops estimating 3 per item on average -- different for different metrics. @@ -55,7 +51,7 @@ func Matrix(fun any, in, out tensor.Tensor) error { c := coords[idx] sa := tensor.Cells1D(tsr[0], c.X) sb := tensor.Cells1D(tsr[0], c.Y) - mfun(sa, sb, mout) + mout := mfun(sa, sb) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, in, out) for _, c := range coords { // copy to upper @@ -67,7 +63,23 @@ func Matrix(fun any, in, out tensor.Tensor) error { return nil } -// CrossMatrix computes the distance / similarity matrix between +// Matrix computes the rows x rows square distance / similarity matrix +// between the patterns for each row of the given higher dimensional input tensor, +// which must have at least 2 dimensions: the outermost rows, +// and within that, 1+dimensional patterns (cells). Use [tensor.NewRowCellsView] +// to organize data into the desired split between a 1D outermost Row dimension +// and the remaining Cells dimension. +// The metric function must have the [MetricFunc] signature. +// The results fill in the elements of the output matrix, which is symmetric, +// and only the lower triangular part is computed, with results copied +// to the upper triangular region, for maximum efficiency. +func Matrix(fun any, in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log(MatrixOut(fun, in, out)) + return out +} + +// CrossMatrixOut computes the distance / similarity matrix between // two different sets of patterns in the two input tensors, where // the patterns are in the sub-space cells of the tensors, // which must have at least 2 dimensions: the outermost rows, @@ -76,7 +88,7 @@ func Matrix(fun any, in, out tensor.Tensor) error { // The metric function must have the [MetricFunc] signature. // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. -func CrossMatrix(fun any, a, b, out tensor.Tensor) error { +func CrossMatrixOut(fun any, a, b tensor.Tensor, out tensor.Values) error { mfun, err := AsMetricFunc(fun) if err != nil { return err @@ -89,10 +101,7 @@ func CrossMatrix(fun any, a, b, out tensor.Tensor) error { if brows == 0 || bcells == 0 { return nil } - if err := tensor.SetShapeSizesMustBeValues(out, arows, brows); err != nil { - return err - } - mout := tensor.NewFloat64Scalar(0.0) + out.SetShapeSizes(arows, brows) // note: flops estimating 3 per item on average -- different for different metrics. flops := min(acells, bcells) * 3 nc := arows * brows @@ -102,13 +111,28 @@ func CrossMatrix(fun any, a, b, out tensor.Tensor) error { br := idx % brows sa := tensor.Cells1D(tsr[0], ar) sb := tensor.Cells1D(tsr[1], br) - mfun(sa, sb, mout) + mout := mfun(sa, sb) tsr[2].SetFloat(mout.Float1D(0), ar, br) }, a, b, out) return nil } -// CovarianceMatrix generates the cells x cells square covariance matrix +// CrossMatrix computes the distance / similarity matrix between +// two different sets of patterns in the two input tensors, where +// the patterns are in the sub-space cells of the tensors, +// which must have at least 2 dimensions: the outermost rows, +// and within that, 1+dimensional patterns that the given distance metric +// function is applied to, with the results filling in the cells of the output matrix. +// The metric function must have the [MetricFunc] signature. +// The rows of the output matrix are the rows of the first input tensor, +// and the columns of the output are the rows of the second input tensor. +func CrossMatrix(fun any, a, b tensor.Tensor) tensor.Values { + out := tensor.NewOfType(a.DataType()) + errors.Log(CrossMatrixOut(fun, a, b, out)) + return out +} + +// CovarianceMatrixOut generates the cells x cells square covariance matrix // for all per-row cells of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, // and within that, 1+dimensional patterns (cells). @@ -121,7 +145,7 @@ func CrossMatrix(fun any, a, b, out tensor.Tensor) error { // which is typical in neural network models, and use // Correlation if they are on very different scales, because it effectively rescales). // The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. -func CovarianceMatrix(fun any, in, out tensor.Tensor) error { +func CovarianceMatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { mfun, err := AsMetricFunc(fun) if err != nil { return err @@ -130,10 +154,7 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { if rows == 0 || cells == 0 { return nil } - mout := tensor.NewFloat64Scalar(0.0) - if err := tensor.SetShapeSizesMustBeValues(out, cells, cells); err != nil { - return err - } + out.SetShapeSizes(cells, cells) flatvw := tensor.NewReshaped(in, rows, cells) var av, bv tensor.Tensor @@ -153,7 +174,7 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { bv = tensor.Reslice(tsr[0], tensor.FullAxis, c.Y) curCoords.Y = c.Y } - mfun(av, bv, mout) + mout := mfun(av, bv) tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) }, flatvw, out) for _, c := range coords { // copy to upper @@ -165,6 +186,25 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { return nil } +// CovarianceMatrix generates the cells x cells square covariance matrix +// for all per-row cells of the given higher dimensional input tensor, +// which must have at least 2 dimensions: the outermost rows, +// and within that, 1+dimensional patterns (cells). +// Each value in the resulting matrix represents the extent to which the +// value of a given cell covaries across the rows of the tensor with the +// value of another cell. +// Uses the given metric function, typically [Covariance] or [Correlation], +// The metric function must have the [MetricFunc] signature. +// Use Covariance if vars have similar overall scaling, +// which is typical in neural network models, and use +// Correlation if they are on very different scales, because it effectively rescales). +// The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. +func CovarianceMatrix(fun any, in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log(CovarianceMatrixOut(fun, in, out)) + return out +} + // PCA performs the eigen decomposition of the given CovarianceMatrix, // using principal components analysis (PCA), which is slower than [SVD]. // The eigenvectors are same size as Covar. Each eigenvector is a column @@ -172,18 +212,14 @@ func CovarianceMatrix(fun any, in, out tensor.Tensor) error { // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *lowest* to *highest*. // Note that PCA produces results in the *opposite* order of [SVD]. -func PCA(covar, eigenvecs, vals tensor.Tensor) error { +func PCA(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { n := covar.DimSize(0) cv, err := matrix.NewSymmetric(tensor.AsFloat64Tensor(covar)) if err != nil { return err } - if err := tensor.SetShapeSizesMustBeValues(eigenvecs, n, n); err != nil { - return err - } - if err := tensor.SetShapeSizesMustBeValues(vals, n); err != nil { - return err - } + eigenvecs.SetShapeSizes(n, n) + vals.SetShapeSizes(n) var eig mat.EigenSym ok := eig.Factorize(cv, true) if !ok { @@ -207,18 +243,14 @@ func PCA(covar, eigenvecs, vals tensor.Tensor) error { // i.e., maximum eigenvector is the last column. // The eigenvalues are the size of one row, ordered *highest* to *lowest*. // Note that SVD produces results in the *opposite* order of [PCA]. -func SVD(covar, eigenvecs, vals tensor.Tensor) error { +func SVD(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { n := covar.DimSize(0) cv, err := matrix.NewSymmetric(tensor.AsFloat64Tensor(covar)) if err != nil { return err } - if err := tensor.SetShapeSizesMustBeValues(eigenvecs, n, n); err != nil { - return err - } - if err := tensor.SetShapeSizesMustBeValues(vals, n); err != nil { - return err - } + eigenvecs.SetShapeSizes(n, n) + vals.SetShapeSizes(n) var eig mat.SVD ok := eig.Factorize(cv, mat.SVDFull) // todo: benchmark different versions if !ok { @@ -235,40 +267,51 @@ func SVD(covar, eigenvecs, vals tensor.Tensor) error { return nil } -// ProjectOnMatrixColumn is a convenience function for projecting given vector +// ProjectOnMatrixColumnOut is a convenience function for projecting given vector // of values along a specific column (2nd dimension) of the given 2D matrix, // specified by the scalar colindex, putting results into out. // If the vec is more than 1 dimensional, then it is treated as rows x cells, // and each row of cells is projected through the matrix column, producing a // 1D output with the number of rows. Otherwise a single number is produced. // This is typically done with results from SVD or PCA. -func ProjectOnMatrixColumn(mtx, vec, colindex, out tensor.Tensor) error { +func ProjectOnMatrixColumnOut(mtx, vec, colindex tensor.Tensor, out tensor.Values) error { ci := int(colindex.Float1D(0)) col := tensor.As1D(tensor.Reslice(mtx, tensor.Slice{}, ci)) // fmt.Println(mtx.String(), col.String()) rows, cells := vec.Shape().RowCellSize() if rows > 0 && cells > 0 { msum := tensor.NewFloat64Scalar(0) - if err := tensor.SetShapeSizesMustBeValues(out, rows); err != nil { - return err - } + out.SetShapeSizes(rows) mout := tensor.NewFloat64(cells) for i := range rows { - err := tmath.Mul(tensor.Cells1D(vec, i), col, mout) + err := tmath.MulOut(tensor.Cells1D(vec, i), col, mout) if err != nil { return err } - stats.Sum(mout, msum) + stats.SumOut(mout, msum) out.SetFloat1D(msum.Float1D(0), i) } } else { mout := tensor.NewFloat64(1) - tmath.Mul(vec, col, mout) - stats.Sum(mout, out) + tmath.MulOut(vec, col, mout) + stats.SumOut(mout, out) } return nil } +// ProjectOnMatrixColumn is a convenience function for projecting given vector +// of values along a specific column (2nd dimension) of the given 2D matrix, +// specified by the scalar colindex, putting results into out. +// If the vec is more than 1 dimensional, then it is treated as rows x cells, +// and each row of cells is projected through the matrix column, producing a +// 1D output with the number of rows. Otherwise a single number is produced. +// This is typically done with results from SVD or PCA. +func ProjectOnMatrixColumn(mtx, vec, colindex tensor.Tensor) tensor.Values { + out := tensor.NewOfType(vec.DataType()) + errors.Log(ProjectOnMatrixColumnOut(mtx, vec, colindex, out)) + return out +} + //////////////////////////////////////////// // Triangular square matrix functions diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index c66a81188b..f3d0c23dea 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -25,75 +25,66 @@ func TestFuncs(t *testing.T) { btsr := tensor.NewNumberFromValues(b64...) out := tensor.NewFloat64(1) - Euclidean(atsr, btsr, out) + EuclideanOut(atsr, btsr, out) assert.InDelta(t, results[MetricEuclidean], out.Values[0], tol) - SumSquares(atsr, btsr, out) + SumSquaresOut(atsr, btsr, out) assert.InDelta(t, results[MetricSumSquares], out.Values[0], tol) - EuclideanBinTol(atsr, btsr, out) + EuclideanBinTolOut(atsr, btsr, out) assert.InDelta(t, results[MetricEuclideanBinTol], out.Values[0], tol) - Abs(atsr, btsr, out) + AbsOut(atsr, btsr, out) assert.InDelta(t, results[MetricAbs], out.Values[0], tol) - Hamming(atsr, btsr, out) + HammingOut(atsr, btsr, out) assert.Equal(t, results[MetricHamming], out.Values[0]) - SumSquaresBinTol(atsr, btsr, out) + SumSquaresBinTolOut(atsr, btsr, out) assert.InDelta(t, results[MetricSumSquaresBinTol], out.Values[0], tol) - Covariance(atsr, btsr, out) + CovarianceOut(atsr, btsr, out) assert.InDelta(t, results[MetricCovariance], out.Values[0], tol) - Correlation(atsr, btsr, out) + CorrelationOut(atsr, btsr, out) assert.InDelta(t, results[MetricCorrelation], out.Values[0], tol) - InvCorrelation(atsr, btsr, out) + InvCorrelationOut(atsr, btsr, out) assert.InDelta(t, results[MetricInvCorrelation], out.Values[0], tol) - CrossEntropy(atsr, btsr, out) + CrossEntropyOut(atsr, btsr, out) assert.InDelta(t, results[MetricCrossEntropy], out.Values[0], tol) - InnerProduct(atsr, btsr, out) + InnerProductOut(atsr, btsr, out) assert.InDelta(t, results[MetricInnerProduct], out.Values[0], tol) - Cosine(atsr, btsr, out) + CosineOut(atsr, btsr, out) assert.InDelta(t, results[MetricCosine], out.Values[0], tol) - InvCosine(atsr, btsr, out) + InvCosineOut(atsr, btsr, out) assert.InDelta(t, results[MetricInvCosine], out.Values[0], tol) for met := MetricEuclidean; met < MetricsN; met++ { - met.Call(atsr, btsr, out) - assert.InDelta(t, results[met], out.Values[0], tol) + out := met.Call(atsr, btsr) + assert.InDelta(t, results[met], out.Float1D(0), tol) } } func TestMatrix(t *testing.T) { - var simres = `[12, 12] -[0]: 0 3.4641016151377544 8.831760866327848 9.273618495495704 8.717797887081348 9.38083151964686 4.69041575982343 5.830951894845301 8.12403840463596 8.54400374531753 5.291502622129181 6.324555320336759 -[1]: 3.4641016151377544 0 9.38083151964686 8.717797887081348 9.273618495495704 8.831760866327848 5.830951894845301 4.69041575982343 8.717797887081348 7.937253933193772 6.324555320336759 5.291502622129181 -[2]: 8.831760866327848 9.38083151964686 0 3.4641016151377544 4.242640687119285 5.0990195135927845 9.38083151964686 9.899494936611665 4.47213595499958 5.744562646538029 9.38083151964686 9.899494936611665 -[3]: 9.273618495495704 8.717797887081348 3.4641016151377544 0 5.477225575051661 3.7416573867739413 9.797958971132712 9.273618495495704 5.656854249492381 4.58257569495584 9.797958971132712 9.273618495495704 -[4]: 8.717797887081348 9.273618495495704 4.242640687119285 5.477225575051661 0 4 8.831760866327848 9.38083151964686 4.242640687119285 5.5677643628300215 8.831760866327848 9.38083151964686 -[5]: 9.38083151964686 8.831760866327848 5.0990195135927845 3.7416573867739413 4 0 9.486832980505138 8.94427190999916 5.830951894845301 4.795831523312719 9.486832980505138 8.94427190999916 -[6]: 4.69041575982343 5.830951894845301 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 0 3.4641016151377544 9.16515138991168 9.539392014169456 4.242640687119285 5.477225575051661 -[7]: 5.830951894845301 4.69041575982343 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 3.4641016151377544 0 9.695359714832659 9 5.477225575051661 4.242640687119285 -[8]: 8.12403840463596 8.717797887081348 4.47213595499958 5.656854249492381 4.242640687119285 5.830951894845301 9.16515138991168 9.695359714832659 0 3.605551275463989 9.16515138991168 9.695359714832659 -[9]: 8.54400374531753 7.937253933193772 5.744562646538029 4.58257569495584 5.5677643628300215 4.795831523312719 9.539392014169456 9 3.605551275463989 0 9.539392014169456 9 -[10]: 5.291502622129181 6.324555320336759 9.38083151964686 9.797958971132712 8.831760866327848 9.486832980505138 4.242640687119285 5.477225575051661 9.16515138991168 9.539392014169456 0 3.4641016151377544 -[11]: 6.324555320336759 5.291502622129181 9.899494936611665 9.273618495495704 9.38083151964686 8.94427190999916 5.477225575051661 4.242640687119285 9.695359714832659 9 3.4641016151377544 0 -` + + simres := []float64{0, 3.464101552963257, 8.83176040649414, 9.273618698120117, 8.717798233032227, 9.380831718444824, 4.690415859222412, 5.830951690673828, 8.124038696289062, 8.5440034866333, 5.291502475738525, 6.324555397033691} + dt := table.New() err := dt.OpenCSV("../cluster/testdata/faces.dat", tensor.Tab) assert.NoError(t, err) in := dt.Column("Input") out := tensor.NewFloat64() - err = Matrix(Euclidean, in, out) + err = MatrixOut(Euclidean, in, out) assert.NoError(t, err) // fmt.Println(out.Tensor) - assert.Equal(t, simres, out.String()) + for i, v := range simres { + assert.InDelta(t, v, out.Float1D(i), 1.0e-8) + } } func TestPCAIris(t *testing.T) { @@ -109,7 +100,7 @@ func TestPCAIris(t *testing.T) { } data := dt.Column("data") covar := tensor.NewFloat64() - err = CovarianceMatrix(Correlation, data, covar) + err = CovarianceMatrixOut(Correlation, data, covar) assert.NoError(t, err) // fmt.Printf("correl: %s\n", covar.String()) @@ -127,7 +118,7 @@ func TestPCAIris(t *testing.T) { colidx := tensor.NewFloat64Scalar(3) // strongest at end prjns := tensor.NewFloat64() - err = ProjectOnMatrixColumn(vecs, data, colidx, prjns) + err = ProjectOnMatrixColumnOut(vecs, data, colidx, prjns) assert.NoError(t, err) // tensor.SaveCSV(prjns, "testdata/pca_projection.csv", tensor.Comma) trgprjns := []float64{ @@ -158,7 +149,7 @@ func TestPCAIris(t *testing.T) { } colidx.SetFloat1D(0, 0) // strongest at start - err = ProjectOnMatrixColumn(vecs, data, colidx, prjns) + err = ProjectOnMatrixColumnOut(vecs, data, colidx, prjns) assert.NoError(t, err) // tensor.SaveCSV(prjns, "testdata/svd_projection.csv", tensor.Comma) trgprjns = []float64{ diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index a77fa521f1..c87f371577 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -12,19 +12,19 @@ import ( ) func init() { - tensor.AddFunc(MetricEuclidean.FuncName(), Euclidean, 1) - tensor.AddFunc(MetricSumSquares.FuncName(), SumSquares, 1) - tensor.AddFunc(MetricAbs.FuncName(), Abs, 1) - tensor.AddFunc(MetricHamming.FuncName(), Hamming, 1) - tensor.AddFunc(MetricEuclideanBinTol.FuncName(), EuclideanBinTol, 1) - tensor.AddFunc(MetricSumSquaresBinTol.FuncName(), SumSquaresBinTol, 1) - tensor.AddFunc(MetricInvCosine.FuncName(), InvCosine, 1) - tensor.AddFunc(MetricInvCorrelation.FuncName(), InvCorrelation, 1) - tensor.AddFunc(MetricInnerProduct.FuncName(), InnerProduct, 1) - tensor.AddFunc(MetricCrossEntropy.FuncName(), CrossEntropy, 1) - tensor.AddFunc(MetricCovariance.FuncName(), Covariance, 1) - tensor.AddFunc(MetricCorrelation.FuncName(), Correlation, 1) - tensor.AddFunc(MetricCosine.FuncName(), Cosine, 1) + tensor.AddFunc(MetricEuclidean.FuncName(), Euclidean) + tensor.AddFunc(MetricSumSquares.FuncName(), SumSquares) + tensor.AddFunc(MetricAbs.FuncName(), Abs) + tensor.AddFunc(MetricHamming.FuncName(), Hamming) + tensor.AddFunc(MetricEuclideanBinTol.FuncName(), EuclideanBinTol) + tensor.AddFunc(MetricSumSquaresBinTol.FuncName(), SumSquaresBinTol) + tensor.AddFunc(MetricInvCosine.FuncName(), InvCosine) + tensor.AddFunc(MetricInvCorrelation.FuncName(), InvCorrelation) + tensor.AddFunc(MetricInnerProduct.FuncName(), InnerProduct) + tensor.AddFunc(MetricCrossEntropy.FuncName(), CrossEntropy) + tensor.AddFunc(MetricCovariance.FuncName(), Covariance) + tensor.AddFunc(MetricCorrelation.FuncName(), Correlation) + tensor.AddFunc(MetricCosine.FuncName(), Cosine) } // Metrics are standard metric functions @@ -109,14 +109,8 @@ func (m Metrics) Func() MetricFunc { // Call calls a standard Metrics enum function on given tensors. // Output results are in the out tensor. -func (m Metrics) Call(a, b, out tensor.Tensor) error { - return tensor.Call(m.FuncName(), a, b, out) -} - -// CallOut calls a standard Metrics enum function on given tensors, -// returning output as a newly created tensor. -func (m Metrics) CallOut(a, b tensor.Tensor) tensor.Tensor { - return tensor.CallOut(m.FuncName(), a, b) +func (m Metrics) Call(a, b tensor.Tensor) tensor.Values { + return m.Func()(a, b) } // Increasing returns true if the distance metric is such that metric @@ -139,3 +133,13 @@ func AsMetricFunc(fun any) (MetricFunc, error) { } return mfun, nil } + +// AsMetricOutFunc returns given function as a [MetricFunc] function, +// or an error if it does not fit that signature. +func AsMetricOutFunc(fun any) (MetricOutFunc, error) { + mfun, ok := fun.(MetricOutFunc) + if !ok { + return nil, errors.New("metric.AsMetricOutFunc: function does not fit the MetricOutFunc signature") + } + return mfun, nil +} diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 8f8e156e9b..3a559dc345 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -11,7 +11,7 @@ import ( ) func init() { - tensor.AddFunc("metric.ClosestRow", ClosestRow, 1, tensor.AnyFirstArg) + tensor.AddFunc("metric.ClosestRow", ClosestRow) } // ClosestRow returns the closest fit between probe pattern and patterns in @@ -22,22 +22,19 @@ func init() { // Note: this does _not_ use any existing Indexes for the probe, // but does for the vocab, and the returned index is the logical index // into any existing Indexes. -func ClosestRow(fun any, probe, vocab, out tensor.Tensor) error { - if err := tensor.SetShapeSizesMustBeValues(out, 2); err != nil { - return err - } +func ClosestRow(fun any, probe, vocab tensor.Tensor, out tensor.Values) error { + out.SetShapeSizes(2) mfun, err := AsMetricFunc(fun) if err != nil { return err } rows, _ := vocab.Shape().RowCellSize() mi := -1 - mout := tensor.NewFloat64Scalar(0.0) mind := math.MaxFloat64 pr1d := tensor.As1D(probe) for ri := range rows { sub := tensor.Cells1D(vocab, ri) - mfun(pr1d, sub, mout) + mout := mfun(pr1d, sub) d := mout.Float1D(0) if d < mind { mi = ri diff --git a/tensor/stats/metric/vec.go b/tensor/stats/metric/vec.go index 004cca9bf6..832425d858 100644 --- a/tensor/stats/metric/vec.go +++ b/tensor/stats/metric/vec.go @@ -24,10 +24,8 @@ func Vectorize3Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr } nt := len(tsr) osz := tensor.CellsSize(tsr[0].ShapeSizes()) - out := tsr[nt-1] - if err := tensor.SetShapeSizesMustBeValues(out, osz...); err != nil { - return nil, nil, nil, err - } + out := tsr[nt-1].(tensor.Values) + out.SetShapeSizes(osz...) out1 = tensor.NewFloat64(osz...) out2 = tensor.NewFloat64(osz...) out3 = tensor.NewFloat64(osz...) diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go index ca9df612d7..fd69fa9424 100644 --- a/tensor/stats/stats/describe.go +++ b/tensor/stats/stats/describe.go @@ -41,7 +41,8 @@ func Describe(dir *datafs.Data, tsrs ...tensor.Tensor) { for _, st := range DescriptiveStats { stnm := st.String() sv := datafs.NewValue[float64](td, stnm, 1) - st.Call(tsr, sv) + stout := st.Call(tsr) + sv.CopyFrom(stout) } } } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index c79407bc00..b821c909bc 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -7,11 +7,15 @@ package stats import ( "math" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) -// StatsFunc is the function signature for a stats function, -// which is computed over the outermost row dimension and the +// StatsFunc is the function signature for a stats function that +// returns a new output vector. This can be less efficient for repeated +// computations where the output can be re-used: see [StatsOutFunc]. +// But this version can be directly chained with other function calls. +// Function is computed over the outermost row dimension and the // output is the shape of the remaining inner cells (a scalar for 1D inputs). // Use [tensor.As1D], [tensor.NewRowCellsView], [tensor.Cells1D] etc // to reshape and reslice the data as needed. @@ -20,11 +24,17 @@ import ( // e.g., using VectorizeThreaded or GPU, due to shared writing // to the same output values. Special implementations are required // if that is needed. -type StatsFunc = func(in, out tensor.Tensor) error +type StatsFunc = func(in tensor.Tensor) tensor.Values + +// StatsOutFunc is the function signature for a stats function, +// that takes output values as final argument. See [StatsFunc] +// This version is for computationally demanding cases and saves +// reallocation of output. +type StatsOutFunc = func(in tensor.Tensor, out tensor.Values) error // CountOut64 computes the count of non-NaN tensor values, // and returns the Float64 output values for subsequent use. -func CountOut64(in, out tensor.Tensor) (tensor.Tensor, error) { +func CountOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { return VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + 1 @@ -34,14 +44,22 @@ func CountOut64(in, out tensor.Tensor) (tensor.Tensor, error) { // Count computes the count of non-NaN tensor values. // See [StatsFunc] for general information. -func Count(in, out tensor.Tensor) error { +func Count(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log1(CountOut64(in, out)) + return out +} + +// CountOut computes the count of non-NaN tensor values. +// See [StatsOutFunc] for general information. +func CountOut(in tensor.Tensor, out tensor.Values) error { _, err := CountOut64(in, out) return err } // SumOut64 computes the sum of tensor values, // and returns the Float64 output values for subsequent use. -func SumOut64(in, out tensor.Tensor) (tensor.Tensor, error) { +func SumOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { return VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + val @@ -49,17 +67,32 @@ func SumOut64(in, out tensor.Tensor) (tensor.Tensor, error) { }, in, out) } -// Sum computes the sum of tensor values. -// See [StatsFunc] for general information. -func Sum(in, out tensor.Tensor) error { +// SumOut computes the sum of tensor values. +// See [StatsOutFunc] for general information. +func SumOut(in tensor.Tensor, out tensor.Values) error { _, err := SumOut64(in, out) return err } +// Sum computes the sum of tensor values. +// See [StatsFunc] for general information. +func Sum(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log1(SumOut64(in, out)) + return out +} + // SumAbs computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. -func SumAbs(in, out tensor.Tensor) error { +func SumAbs(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(SumAbsOut, in) +} + +// SumAbsOut computes the sum of absolute-value-of tensor values. +// This is also known as the L1 norm. +// See [StatsFunc] for general information. +func SumAbsOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) @@ -70,7 +103,13 @@ func SumAbs(in, out tensor.Tensor) error { // Prod computes the product of tensor values. // See [StatsFunc] for general information. -func Prod(in, out tensor.Tensor) error { +func Prod(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(ProdOut, in) +} + +// ProdOut computes the product of tensor values. +// See [StatsOutFunc] for general information. +func ProdOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 1, func(val, agg float64) float64 { return agg * val @@ -81,7 +120,13 @@ func Prod(in, out tensor.Tensor) error { // Min computes the min of tensor values. // See [StatsFunc] for general information. -func Min(in, out tensor.Tensor) error { +func Min(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(MinOut, in) +} + +// MinOut computes the min of tensor values. +// See [StatsOutFunc] for general information. +func MinOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, val) @@ -92,7 +137,13 @@ func Min(in, out tensor.Tensor) error { // Max computes the max of tensor values. // See [StatsFunc] for general information. -func Max(in, out tensor.Tensor) error { +func Max(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(MaxOut, in) +} + +// MaxOut computes the max of tensor values. +// See [StatsOutFunc] for general information. +func MaxOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, val) @@ -103,7 +154,13 @@ func Max(in, out tensor.Tensor) error { // MinAbs computes the min of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MinAbs(in, out tensor.Tensor) error { +func MinAbs(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(MinAbsOut, in) +} + +// MinAbsOut computes the min of absolute-value-of tensor values. +// See [StatsOutFunc] for general information. +func MinAbsOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], math.MaxFloat64, func(val, agg float64) float64 { return math.Min(agg, math.Abs(val)) @@ -114,7 +171,13 @@ func MinAbs(in, out tensor.Tensor) error { // MaxAbs computes the max of absolute-value-of tensor values. // See [StatsFunc] for general information. -func MaxAbs(in, out tensor.Tensor) error { +func MaxAbs(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(MaxAbsOut, in) +} + +// MaxAbsOut computes the max of absolute-value-of tensor values. +// See [StatsOutFunc] for general information. +func MaxAbsOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], -math.MaxFloat64, func(val, agg float64) float64 { return math.Max(agg, math.Abs(val)) @@ -125,7 +188,7 @@ func MaxAbs(in, out tensor.Tensor) error { // MeanOut64 computes the mean of tensor values, // and returns the Float64 output values for subsequent use. -func MeanOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor, err error) { +func MeanOut64(in tensor.Tensor, out tensor.Values) (mean64, count64 tensor.Tensor, err error) { sum64, err := SumOut64(in, out) if err != nil { return @@ -146,14 +209,22 @@ func MeanOut64(in, out tensor.Tensor) (mean64, count64 tensor.Tensor, err error) // Mean computes the mean of tensor values. // See [StatsFunc] for general information. -func Mean(in, out tensor.Tensor) error { +func Mean(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log2(MeanOut64(in, out)) + return out +} + +// MeanOut computes the mean of tensor values. +// See [StatsOutFunc] for general information. +func MeanOut(in tensor.Tensor, out tensor.Values) error { _, _, err := MeanOut64(in, out) return err } // SumSqDevOut64 computes the sum of squared mean deviates of tensor values, // and returns the Float64 output values for subsequent use. -func SumSqDevOut64(in, out tensor.Tensor) (ssd64, mean64, count64 tensor.Tensor, err error) { +func SumSqDevOut64(in tensor.Tensor, out tensor.Values) (ssd64, mean64, count64 tensor.Tensor, err error) { mean64, count64, err = MeanOut64(in, out) if err != nil { return @@ -169,7 +240,7 @@ func SumSqDevOut64(in, out tensor.Tensor) (ssd64, mean64, count64 tensor.Tensor, // VarOut64 computes the sample variance of tensor values, // and returns the Float64 output values for subsequent use. -func VarOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, err error) { +func VarOut64(in tensor.Tensor, out tensor.Values) (var64, mean64, count64 tensor.Tensor, err error) { var64, mean64, count64, err = SumSqDevOut64(in, out) if err != nil { return @@ -190,14 +261,24 @@ func VarOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, err // Var computes the sample variance of tensor values. // Squared deviations from mean, divided by n-1. See also [VarPopFunc]. // See [StatsFunc] for general information. -func Var(in, out tensor.Tensor) error { +func Var(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + _, _, _, err := VarOut64(in, out) + errors.Log(err) + return out +} + +// VarOut computes the sample variance of tensor values. +// Squared deviations from mean, divided by n-1. See also [VarPopFunc]. +// See [StatsOutFunc] for general information. +func VarOut(in tensor.Tensor, out tensor.Values) error { _, _, _, err := VarOut64(in, out) return err } // StdOut64 computes the sample standard deviation of tensor values. // and returns the Float64 output values for subsequent use. -func StdOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor, err error) { +func StdOut64(in tensor.Tensor, out tensor.Values) (std64, mean64, count64 tensor.Tensor, err error) { std64, mean64, count64, err = VarOut64(in, out) if err != nil { return @@ -214,7 +295,17 @@ func StdOut64(in, out tensor.Tensor) (std64, mean64, count64 tensor.Tensor, err // Std computes the sample standard deviation of tensor values. // Sqrt of variance from [VarFunc]. See also [StdPopFunc]. // See [StatsFunc] for general information. -func Std(in, out tensor.Tensor) error { +func Std(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + _, _, _, err := StdOut64(in, out) + errors.Log(err) + return out +} + +// StdOut computes the sample standard deviation of tensor values. +// Sqrt of variance from [VarFunc]. See also [StdPopFunc]. +// See [StatsOutFunc] for general information. +func StdOut(in tensor.Tensor, out tensor.Values) error { _, _, _, err := StdOut64(in, out) return err } @@ -222,7 +313,16 @@ func Std(in, out tensor.Tensor) error { // Sem computes the sample standard error of the mean of tensor values. // Standard deviation [StdFunc] / sqrt(n). See also [SemPopFunc]. // See [StatsFunc] for general information. -func Sem(in, out tensor.Tensor) error { +func Sem(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log(SemOut(in, out)) + return out +} + +// SemOut computes the sample standard error of the mean of tensor values. +// Standard deviation [StdFunc] / sqrt(n). See also [SemPopFunc]. +// See [StatsOutFunc] for general information. +func SemOut(in tensor.Tensor, out tensor.Values) error { var64, _, count64, err := VarOut64(in, out) if err != nil { return err @@ -241,7 +341,7 @@ func Sem(in, out tensor.Tensor) error { // VarPopOut64 computes the population variance of tensor values. // and returns the Float64 output values for subsequent use. -func VarPopOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, err error) { +func VarPopOut64(in tensor.Tensor, out tensor.Values) (var64, mean64, count64 tensor.Tensor, err error) { var64, mean64, count64, err = SumSqDevOut64(in, out) if err != nil { return @@ -261,7 +361,17 @@ func VarPopOut64(in, out tensor.Tensor) (var64, mean64, count64 tensor.Tensor, e // VarPop computes the population variance of tensor values. // Squared deviations from mean, divided by n. See also [VarFunc]. // See [StatsFunc] for general information. -func VarPop(in, out tensor.Tensor) error { +func VarPop(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + _, _, _, err := VarPopOut64(in, out) + errors.Log(err) + return out +} + +// VarPopOut computes the population variance of tensor values. +// Squared deviations from mean, divided by n. See also [VarFunc]. +// See [StatsOutFunc] for general information. +func VarPopOut(in tensor.Tensor, out tensor.Values) error { _, _, _, err := VarPopOut64(in, out) return err } @@ -269,7 +379,16 @@ func VarPop(in, out tensor.Tensor) error { // StdPop computes the population standard deviation of tensor values. // Sqrt of variance from [VarPopFunc]. See also [StdFunc]. // See [StatsFunc] for general information. -func StdPop(in, out tensor.Tensor) error { +func StdPop(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log(StdPopOut(in, out)) + return out +} + +// StdPopOut computes the population standard deviation of tensor values. +// Sqrt of variance from [VarPopFunc]. See also [StdFunc]. +// See [StatsOutFunc] for general information. +func StdPopOut(in tensor.Tensor, out tensor.Values) error { var64, _, _, err := VarPopOut64(in, out) if err != nil { return err @@ -284,7 +403,16 @@ func StdPop(in, out tensor.Tensor) error { // SemPop computes the population standard error of the mean of tensor values. // Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. // See [StatsFunc] for general information. -func SemPop(in, out tensor.Tensor) error { +func SemPop(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log(SemPopOut(in, out)) + return out +} + +// SemPopOut computes the population standard error of the mean of tensor values. +// Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. +// See [StatsOutFunc] for general information. +func SemPopOut(in tensor.Tensor, out tensor.Values) error { var64, _, count64, err := VarPopOut64(in, out) if err != nil { return err @@ -304,7 +432,7 @@ func SemPop(in, out tensor.Tensor) error { // SumSqScaleOut64 is a helper for sum-of-squares, returning scale and ss // factors aggregated separately for better numerical stability, per BLAS. // Returns the Float64 output values for subsequent use. -func SumSqScaleOut64(in, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err error) { +func SumSqScaleOut64(in tensor.Tensor, out tensor.Values) (scale64, ss64 tensor.Tensor, err error) { scale64, ss64, err = Vectorize2Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec2outFunc(idx, tsr[0], tsr[1], tsr[2], 0, 1, func(val, scale, ss float64) (float64, float64) { if val == 0 { @@ -325,7 +453,7 @@ func SumSqScaleOut64(in, out tensor.Tensor) (scale64, ss64 tensor.Tensor, err er // SumSqOut64 computes the sum of squares of tensor values, // and returns the Float64 output values for subsequent use. -func SumSqOut64(in, out tensor.Tensor) (tensor.Tensor, error) { +func SumSqOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { scale64, ss64, err := SumSqScaleOut64(in, out) if err != nil { return nil, err @@ -348,7 +476,15 @@ func SumSqOut64(in, out tensor.Tensor) (tensor.Tensor, error) { // SumSq computes the sum of squares of tensor values, // See [StatsFunc] for general information. -func SumSq(in, out tensor.Tensor) error { +func SumSq(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log1(SumSqOut64(in, out)) + return out +} + +// SumSqOut computes the sum of squares of tensor values, +// See [StatsOutFunc] for general information. +func SumSqOut(in tensor.Tensor, out tensor.Values) error { _, err := SumSqOut64(in, out) return err } @@ -356,7 +492,7 @@ func SumSq(in, out tensor.Tensor) error { // L2NormOut64 computes the square root of the sum of squares of tensor values, // known as the L2 norm, and returns the Float64 output values for // use in subsequent computations. -func L2NormOut64(in, out tensor.Tensor) (tensor.Tensor, error) { +func L2NormOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { scale64, ss64, err := SumSqScaleOut64(in, out) if err != nil { return nil, err @@ -380,7 +516,16 @@ func L2NormOut64(in, out tensor.Tensor) (tensor.Tensor, error) { // L2Norm computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsFunc] for general information. -func L2Norm(in, out tensor.Tensor) error { +func L2Norm(in tensor.Tensor) tensor.Values { + out := tensor.NewOfType(in.DataType()) + errors.Log1(L2NormOut64(in, out)) + return out +} + +// L2NormOut computes the square root of the sum of squares of tensor values, +// known as the L2 norm. +// See [StatsOutFunc] for general information. +func L2NormOut(in tensor.Tensor, out tensor.Values) error { _, err := L2NormOut64(in, out) return err } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 94dfb3746b..d1ad8e49d5 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -6,7 +6,6 @@ package stats import ( "strconv" - "strings" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/datafs" @@ -126,10 +125,6 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { // todo: GroupCombined -// note: we have to pass stat as a string here because we need the name -// to record the results in the datafs, and we can't get the name directly. -// also we need _2_ anys, and varargs! - // GroupStats computes the given stats function on the unique grouped indexes // produced by the [Groups] function, in the given [datafs] directory, // applied to each of the tensors passed here. @@ -140,7 +135,7 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { // a String tensor with the unique values of each source [Groups] tensor, // and a aligned Float64 tensor with the statistics results for each such // unique group value. See the README.md file for a diagram of the results. -func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) error { +func GroupStats(dir *datafs.Data, stat Stats, tsrs ...tensor.Tensor) error { gd, err := dir.RecycleDir("Groups") if err != nil { return err @@ -149,12 +144,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) error { if err != nil { return err } - stnm := StripPackage(stat) - spl := strings.Split(stat, ".") - if len(spl) == 2 { - stnm = spl[1] - } - stout := tensor.NewFloat64Scalar(0) + stnm := StripPackage(stat.String()) groups := gd.ItemsFunc(nil) for _, gp := range groups { gpnm := gp.Name() @@ -178,7 +168,7 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) error { for i, v := range vals { idx := tensor.AsIntSlice(v) sg := tensor.NewRows(tsr.AsValues(), idx...) - tensor.Call(stat, sg, stout) + stout := stat.Call(sg) sv.SetFloatRow(stout.Float1D(0), i) } } @@ -189,14 +179,14 @@ func GroupStats(dir *datafs.Data, stat string, tsrs ...tensor.Tensor) error { // TableGroupStats runs [GroupStats] using standard [Stats] // on the given columns from given [table.Table]. func TableGroupStats(dir *datafs.Data, stat Stats, dt *table.Table, columns ...string) error { - return GroupStats(dir, stat.FuncName(), dt.ColumnList(columns...)...) + return GroupStats(dir, stat, dt.ColumnList(columns...)...) } // GroupDescribe runs standard descriptive statistics on given tensor data // using [GroupStats] function, with [DescriptiveStats] list of stats. func GroupDescribe(dir *datafs.Data, tsrs ...tensor.Tensor) error { for _, st := range DescriptiveStats { - err := GroupStats(dir, st.FuncName(), tsrs...) + err := GroupStats(dir, st, tsrs...) if err != nil { return err } diff --git a/tensor/stats/stats/norm.go b/tensor/stats/stats/norm.go index cd7181f3a0..1146ebef8d 100644 --- a/tensor/stats/stats/norm.go +++ b/tensor/stats/stats/norm.go @@ -10,40 +10,64 @@ import ( "cogentcore.org/core/tensor/tmath" ) +func init() { + tensor.AddFunc("stats.ZScore", ZScore) + tensor.AddFunc("stats.UnitNorm", UnitNorm) + tensor.AddFunc("stats.Clamp", Clamp) + tensor.AddFunc("stats.Binarize", Binarize) +} + +// ZScore computes Z-normalized values into given output tensor, +// subtracting the Mean and dividing by the standard deviation. +func ZScore(a tensor.Tensor) tensor.Values { + return tensor.CallOut1(ZScoreOut, a) +} + // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. -func ZScore(a, out tensor.Tensor) error { +func ZScoreOut(a tensor.Tensor, out tensor.Values) error { mout := tensor.NewFloat64() std, mean, _, err := StdOut64(a, mout) if err != nil { return err } - tmath.Sub(a, mean, out) - tmath.Div(out, std, out) + tmath.SubOut(a, mean, out) + tmath.DivOut(out, std, out) return nil } // UnitNorm computes unit normalized values into given output tensor, // subtracting the Min value and dividing by the Max of the remaining numbers. -func UnitNorm(a, out tensor.Tensor) error { +func UnitNorm(a tensor.Tensor) tensor.Values { + return tensor.CallOut1(UnitNormOut, a) +} + +// UnitNormOut computes unit normalized values into given output tensor, +// subtracting the Min value and dividing by the Max of the remaining numbers. +func UnitNormOut(a tensor.Tensor, out tensor.Values) error { mout := tensor.NewFloat64() - err := Min(a, mout) + err := MinOut(a, mout) if err != nil { return err } - tmath.Sub(a, mout, out) - Max(out, mout) - tmath.Div(out, mout, out) + tmath.SubOut(a, mout, out) + MaxOut(out, mout) + tmath.DivOut(out, mout, out) return nil } // Clamp ensures that all values are within min, max limits, clamping // values to those bounds if they exceed them. min and max args are // treated as scalars (first value used). -func Clamp(in, minv, maxv, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Clamp(in, minv, maxv tensor.Tensor) tensor.Values { + return tensor.CallOut3(ClampOut, in, minv, minv) +} + +// ClampOut ensures that all values are within min, max limits, clamping +// values to those bounds if they exceed them. min and max args are +// treated as scalars (first value used). +func ClampOut(in, minv, maxv tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) mn := minv.Float1D(0) mx := maxv.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { @@ -55,10 +79,15 @@ func Clamp(in, minv, maxv, out tensor.Tensor) error { // Binarize results in a binary-valued output by setting // values >= the threshold to 1, else 0. threshold is // treated as a scalar (first value used). -func Binarize(in, threshold, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Binarize(in, threshold tensor.Tensor) tensor.Values { + return tensor.CallOut2(BinarizeOut, in, threshold) +} + +// BinarizeOut results in a binary-valued output by setting +// values >= the threshold to 1, else 0. threshold is +// treated as a scalar (first value used). +func BinarizeOut(in, threshold tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) thr := threshold.Float1D(0) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { v := tsr[0].Float1D(idx) diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 1978155654..02ea725447 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -14,21 +14,31 @@ import ( // Quantiles returns the given quantile(s) of non-NaN elements in given // 1D tensor. Because sorting uses indexes, this only works for 1D case. // If needed for a sub-space of values, that can be extracted through slicing +// and then used. Logs an error if not 1D. +// qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. +// Uses linear interpolation. +// Because this requires a sort, it is more efficient to get as many quantiles +// as needed in one pass. +func Quantiles(in, qs tensor.Tensor) tensor.Values { + return tensor.CallOut2(QuantilesOut, in, qs) +} + +// QuantilesOut returns the given quantile(s) of non-NaN elements in given +// 1D tensor. Because sorting uses indexes, this only works for 1D case. +// If needed for a sub-space of values, that can be extracted through slicing // and then used. Returns and logs an error if not 1D. // qs are 0-1 values, 0 = min, 1 = max, .5 = median, etc. // Uses linear interpolation. // Because this requires a sort, it is more efficient to get as many quantiles // as needed in one pass. -func Quantiles(in, qs, out tensor.Tensor) error { +func QuantilesOut(in, qs tensor.Tensor, out tensor.Values) error { if in.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D input tensors allowed")) } if qs.NumDims() != 1 { return errors.Log(errors.New("stats.QuantilesFunc: only 1D quantile tensors allowed")) } - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } + tensor.SetShapeFrom(out, in) sin := tensor.AsRows(in.AsValues()) sin.ExcludeMissing() sin.Sort(tensor.Ascending) @@ -62,18 +72,36 @@ func Quantiles(in, qs, out tensor.Tensor) error { // Median computes the median (50% quantile) of tensor values. // See [StatsFunc] for general information. -func Median(in, out tensor.Tensor) error { - return Quantiles(in, tensor.NewFloat64Scalar(.5), out) +func Median(in tensor.Tensor) tensor.Values { + return Quantiles(in, tensor.NewFloat64Scalar(.5)) } // Q1 computes the first quantile (25%) of tensor values. // See [StatsFunc] for general information. -func Q1(in, out tensor.Tensor) error { - return Quantiles(in, tensor.NewFloat64Scalar(.25), out) +func Q1(in tensor.Tensor) tensor.Values { + return Quantiles(in, tensor.NewFloat64Scalar(.25)) } // Q3 computes the third quantile (75%) of tensor values. // See [StatsFunc] for general information. -func Q3(in, out tensor.Tensor) error { - return Quantiles(in, tensor.NewFloat64Scalar(.75), out) +func Q3(in tensor.Tensor) tensor.Values { + return Quantiles(in, tensor.NewFloat64Scalar(.75)) +} + +// MedianOut computes the median (50% quantile) of tensor values. +// See [StatsFunc] for general information. +func MedianOut(in tensor.Tensor, out tensor.Values) error { + return QuantilesOut(in, tensor.NewFloat64Scalar(.5), out) +} + +// Q1Out computes the first quantile (25%) of tensor values. +// See [StatsFunc] for general information. +func Q1Out(in tensor.Tensor, out tensor.Values) error { + return QuantilesOut(in, tensor.NewFloat64Scalar(.25), out) +} + +// Q3Out computes the third quantile (75%) of tensor values. +// See [StatsFunc] for general information. +func Q3Out(in tensor.Tensor, out tensor.Values) error { + return QuantilesOut(in, tensor.NewFloat64Scalar(.75), out) } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 1115bfdda5..6994bb1326 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -14,27 +14,27 @@ import ( //go:generate core generate func init() { - tensor.AddFunc(StatCount.FuncName(), Count, 1) - tensor.AddFunc(StatSum.FuncName(), Sum, 1) - tensor.AddFunc(StatSumAbs.FuncName(), SumAbs, 1) - tensor.AddFunc(StatL1Norm.FuncName(), SumAbs, 1) - tensor.AddFunc(StatProd.FuncName(), Prod, 1) - tensor.AddFunc(StatMin.FuncName(), Min, 1) - tensor.AddFunc(StatMax.FuncName(), Max, 1) - tensor.AddFunc(StatMinAbs.FuncName(), MinAbs, 1) - tensor.AddFunc(StatMaxAbs.FuncName(), MaxAbs, 1) - tensor.AddFunc(StatMean.FuncName(), Mean, 1) - tensor.AddFunc(StatVar.FuncName(), Var, 1) - tensor.AddFunc(StatStd.FuncName(), Std, 1) - tensor.AddFunc(StatSem.FuncName(), Sem, 1) - tensor.AddFunc(StatSumSq.FuncName(), SumSq, 1) - tensor.AddFunc(StatL2Norm.FuncName(), L2Norm, 1) - tensor.AddFunc(StatVarPop.FuncName(), VarPop, 1) - tensor.AddFunc(StatStdPop.FuncName(), StdPop, 1) - tensor.AddFunc(StatSemPop.FuncName(), SemPop, 1) - tensor.AddFunc(StatMedian.FuncName(), Median, 1) - tensor.AddFunc(StatQ1.FuncName(), Q1, 1) - tensor.AddFunc(StatQ3.FuncName(), Q3, 1) + tensor.AddFunc(StatCount.FuncName(), Count) + tensor.AddFunc(StatSum.FuncName(), Sum) + tensor.AddFunc(StatSumAbs.FuncName(), SumAbs) + tensor.AddFunc(StatL1Norm.FuncName(), SumAbs) + tensor.AddFunc(StatProd.FuncName(), Prod) + tensor.AddFunc(StatMin.FuncName(), Min) + tensor.AddFunc(StatMax.FuncName(), Max) + tensor.AddFunc(StatMinAbs.FuncName(), MinAbs) + tensor.AddFunc(StatMaxAbs.FuncName(), MaxAbs) + tensor.AddFunc(StatMean.FuncName(), Mean) + tensor.AddFunc(StatVar.FuncName(), Var) + tensor.AddFunc(StatStd.FuncName(), Std) + tensor.AddFunc(StatSem.FuncName(), Sem) + tensor.AddFunc(StatSumSq.FuncName(), SumSq) + tensor.AddFunc(StatL2Norm.FuncName(), L2Norm) + tensor.AddFunc(StatVarPop.FuncName(), VarPop) + tensor.AddFunc(StatStdPop.FuncName(), StdPop) + tensor.AddFunc(StatSemPop.FuncName(), SemPop) + tensor.AddFunc(StatMedian.FuncName(), Median) + tensor.AddFunc(StatQ1.FuncName(), Q1) + tensor.AddFunc(StatQ3.FuncName(), Q3) } // Stats is a list of different standard aggregation functions, which can be used @@ -119,15 +119,9 @@ func (s Stats) Func() StatsFunc { } // Call calls this statistic function on given tensors. -// Output results are in the out tensor. -func (s Stats) Call(in, out tensor.Tensor) error { - return tensor.Call(s.FuncName(), in, out) -} - -// CallOut calls a standard Stats enum function on given tensor, // returning output as a newly created tensor. -func (s Stats) CallOut(in tensor.Tensor) tensor.Tensor { - return tensor.CallOut(s.FuncName(), in) +func (s Stats) Call(in tensor.Tensor) tensor.Values { + return s.Func()(in) } // StripPackage removes any package name from given string, diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index 4d7039552d..8c16304d48 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -21,74 +21,70 @@ func TestFuncs64(t *testing.T) { tol := 1.0e-8 - Count(ix, out) + CountOut(ix, out) assert.Equal(t, results[StatCount], out.Values[0]) - Sum(ix, out) + SumOut(ix, out) assert.Equal(t, results[StatSum], out.Values[0]) - SumAbs(ix, out) + SumAbsOut(ix, out) assert.Equal(t, results[StatSumAbs], out.Values[0]) - Prod(ix, out) + ProdOut(ix, out) assert.Equal(t, results[StatProd], out.Values[0]) - Min(ix, out) + MinOut(ix, out) assert.Equal(t, results[StatMin], out.Values[0]) - Max(ix, out) + MaxOut(ix, out) assert.Equal(t, results[StatMax], out.Values[0]) - MinAbs(ix, out) + MinAbsOut(ix, out) assert.Equal(t, results[StatMinAbs], out.Values[0]) - MaxAbs(ix, out) + MaxAbsOut(ix, out) assert.Equal(t, results[StatMaxAbs], out.Values[0]) - Mean(ix, out) + MeanOut(ix, out) assert.Equal(t, results[StatMean], out.Values[0]) - Var(ix, out) + VarOut(ix, out) assert.InDelta(t, results[StatVar], out.Values[0], tol) - Std(ix, out) + StdOut(ix, out) assert.InDelta(t, results[StatStd], out.Values[0], tol) - Sem(ix, out) + SemOut(ix, out) assert.InDelta(t, results[StatSem], out.Values[0], tol) - VarPop(ix, out) + VarPopOut(ix, out) assert.InDelta(t, results[StatVarPop], out.Values[0], tol) - StdPop(ix, out) + StdPopOut(ix, out) assert.InDelta(t, results[StatStdPop], out.Values[0], tol) - SemPop(ix, out) + SemPopOut(ix, out) assert.InDelta(t, results[StatSemPop], out.Values[0], tol) - SumSq(ix, out) + SumSqOut(ix, out) assert.InDelta(t, results[StatSumSq], out.Values[0], tol) - L2Norm(ix, out) + L2NormOut(ix, out) assert.InDelta(t, results[StatL2Norm], out.Values[0], tol) - Median(ix, out) + MedianOut(ix, out) assert.InDelta(t, results[StatMedian], out.Values[0], tol) - Q1(ix, out) + Q1Out(ix, out) assert.InDelta(t, results[StatQ1], out.Values[0], tol) - Q3(ix, out) + Q3Out(ix, out) assert.InDelta(t, results[StatQ3], out.Values[0], tol) for stat := StatCount; stat < StatsN; stat++ { - err := stat.Call(ix, out) - assert.NoError(t, err) - assert.InDelta(t, results[stat], out.Values[0], tol) + out := stat.Call(ix) + assert.InDelta(t, results[stat], out.Float1D(0), tol) } - err := tensor.Call("stats.Mean", ix, out) // ensure plain name is registered. - assert.NoError(t, err) - assert.InDelta(t, results[StatMean], out.Values[0], tol) } func TestFuncsInt(t *testing.T) { @@ -99,60 +95,60 @@ func TestFuncsInt(t *testing.T) { results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} - Count(ix, out) + CountOut(ix, out) assert.Equal(t, results[StatCount], out.Values[0]) - Sum(ix, out) + SumOut(ix, out) assert.Equal(t, results[StatSum], out.Values[0]) - SumAbs(ix, out) + SumAbsOut(ix, out) assert.Equal(t, results[StatSumAbs], out.Values[0]) - Prod(ix, out) + ProdOut(ix, out) assert.Equal(t, results[StatProd], out.Values[0]) - Min(ix, out) + MinOut(ix, out) assert.Equal(t, results[StatMin], out.Values[0]) - Max(ix, out) + MaxOut(ix, out) assert.Equal(t, results[StatMax], out.Values[0]) - MinAbs(ix, out) + MinAbsOut(ix, out) assert.Equal(t, results[StatMinAbs], out.Values[0]) - MaxAbs(ix, out) + MaxAbsOut(ix, out) assert.Equal(t, results[StatMaxAbs], out.Values[0]) - Mean(ix, out) + MeanOut(ix, out) assert.Equal(t, results[StatMean], out.Values[0]) - Var(ix, out) + VarOut(ix, out) assert.Equal(t, results[StatVar], out.Values[0]) - Std(ix, out) + StdOut(ix, out) assert.Equal(t, results[StatStd], out.Values[0]) - Sem(ix, out) + SemOut(ix, out) assert.Equal(t, results[StatSem], out.Values[0]) - VarPop(ix, out) + VarPopOut(ix, out) assert.Equal(t, results[StatVarPop], out.Values[0]) - StdPop(ix, out) + StdPopOut(ix, out) assert.Equal(t, results[StatStdPop], out.Values[0]) - SemPop(ix, out) + SemPopOut(ix, out) assert.Equal(t, results[StatSemPop], out.Values[0]) - SumSq(ix, out) + SumSqOut(ix, out) assert.Equal(t, results[StatSumSq], out.Values[0]) - L2Norm(ix, out) + L2NormOut(ix, out) assert.Equal(t, results[StatL2Norm], out.Values[0]) for stat := StatCount; stat <= StatSemPop; stat++ { - stat.Call(ix, out) - assert.Equal(t, results[stat], out.Values[0]) + out := stat.Call(ix) + assert.Equal(t, results[stat], out.Int1D(0)) } } @@ -169,16 +165,16 @@ func TestFuncsCell(t *testing.T) { ix := tensor.NewRows(tsr) out := tensor.NewFloat32(20, 10) - Count(ix, out) + CountOut(ix, out) nsub := out.Len() for i := range nsub { assert.Equal(t, 20.0, out.FloatRowCell(0, i)) } - Mean(ix, out) + MeanOut(ix, out) for i := range nsub { assert.InDelta(t, vals[i], out.FloatRowCell(0, i), 1.0e-7) // lower tol, using float32 } - Var(ix, out) + VarOut(ix, out) for i := range nsub { assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) } @@ -190,34 +186,34 @@ func TestNorm(t *testing.T) { oned := tensor.NewNumberFromValues(vals...) oneout := oned.Clone() - ZScore(oned, oneout) + ZScoreOut(oned, oneout) mout := tensor.NewFloat64() std, mean, _, err := StdOut64(oneout, mout) assert.NoError(t, err) assert.InDelta(t, 1.0, std.Float1D(0), 1.0e-6) assert.InDelta(t, 0.0, mean.Float1D(0), 1.0e-6) - UnitNorm(oned, oneout) - Min(oneout, mout) + UnitNormOut(oned, oneout) + MinOut(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - Max(oneout, mout) + MaxOut(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) minv := tensor.NewFloat64Scalar(0) maxv := tensor.NewFloat64Scalar(1) - Clamp(oned, minv, maxv, oneout) - Min(oneout, mout) + ClampOut(oned, minv, maxv, oneout) + MinOut(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - Max(oneout, mout) + MaxOut(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) thr := tensor.NewFloat64Scalar(0.5) - Binarize(oned, thr, oneout) - Min(oneout, mout) + BinarizeOut(oned, thr, oneout) + MinOut(oneout, mout) assert.InDelta(t, 0.0, mout.Float1D(0), 1.0e-6) - Max(oneout, mout) + MaxOut(oneout, mout) assert.InDelta(t, 1.0, mout.Float1D(0), 1.0e-6) // fmt.Println(oneout) } diff --git a/tensor/stats/stats/vec.go b/tensor/stats/stats/vec.go index 2a561ce8df..98cd3c1054 100644 --- a/tensor/stats/stats/vec.go +++ b/tensor/stats/stats/vec.go @@ -26,10 +26,8 @@ func VectorizeOut64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr } nt := len(tsr) osz := tensor.CellsSize(tsr[0].ShapeSizes()) - out := tsr[nt-1] - if err := tensor.SetShapeSizesMustBeValues(out, osz...); err != nil { - return nil, err - } + out := tsr[nt-1].(tensor.Values) + out.SetShapeSizes(osz...) o64 := tensor.NewFloat64(osz...) etsr := slices.Clone(tsr) etsr[nt-1] = o64 @@ -53,10 +51,8 @@ func Vectorize2Out64(nfunc func(tsr ...tensor.Tensor) int, fun func(idx int, tsr } nt := len(tsr) osz := tensor.CellsSize(tsr[0].ShapeSizes()) - out := tsr[nt-1] - if err = tensor.SetShapeSizesMustBeValues(out, osz...); err != nil { - return - } + out := tsr[nt-1].(tensor.Values) + out.SetShapeSizes(osz...) out1 = tensor.NewFloat64(osz...) out2 = tensor.NewFloat64(osz...) tsrs := slices.Clone(tsr[:nt-1]) diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index b1bf340f27..c918b1243e 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -11,409 +11,483 @@ import ( ) func init() { - tensor.AddFunc("Abs", Abs, 1) - tensor.AddFunc("Acos", Acos, 1) - tensor.AddFunc("Acosh", Acosh, 1) - tensor.AddFunc("Asin", Asin, 1) - tensor.AddFunc("Asinh", Asinh, 1) - tensor.AddFunc("Atan", Atan, 1) - tensor.AddFunc("Atanh", Atanh, 1) - tensor.AddFunc("Cbrt", Cbrt, 1) - tensor.AddFunc("Ceil", Ceil, 1) - tensor.AddFunc("Cos", Cos, 1) - tensor.AddFunc("Cosh", Cosh, 1) - tensor.AddFunc("Erf", Erf, 1) - tensor.AddFunc("Erfc", Erfc, 1) - tensor.AddFunc("Erfcinv", Erfcinv, 1) - tensor.AddFunc("Erfinv", Erfinv, 1) - tensor.AddFunc("Exp", Exp, 1) - tensor.AddFunc("Exp2", Exp2, 1) - tensor.AddFunc("Expm1", Expm1, 1) - tensor.AddFunc("Floor", Floor, 1) - tensor.AddFunc("Gamma", Gamma, 1) - tensor.AddFunc("J0", J0, 1) - tensor.AddFunc("J1", J1, 1) - tensor.AddFunc("Log", Log, 1) - tensor.AddFunc("Log10", Log10, 1) - tensor.AddFunc("Log1p", Log1p, 1) - tensor.AddFunc("Log2", Log2, 1) - tensor.AddFunc("Logb", Logb, 1) - tensor.AddFunc("Round", Round, 1) - tensor.AddFunc("RoundToEven", RoundToEven, 1) - tensor.AddFunc("Sin", Sin, 1) - tensor.AddFunc("Sinh", Sinh, 1) - tensor.AddFunc("Sqrt", Sqrt, 1) - tensor.AddFunc("Tan", Tan, 1) - tensor.AddFunc("Tanh", Tanh, 1) - tensor.AddFunc("Trunc", Trunc, 1) - tensor.AddFunc("Y0", Y0, 1) - tensor.AddFunc("Y1", Y1, 1) -} - -func Abs(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } + tensor.AddFunc("Abs", Abs) + tensor.AddFunc("Acos", Acos) + tensor.AddFunc("Acosh", Acosh) + tensor.AddFunc("Asin", Asin) + tensor.AddFunc("Asinh", Asinh) + tensor.AddFunc("Atan", Atan) + tensor.AddFunc("Atanh", Atanh) + tensor.AddFunc("Cbrt", Cbrt) + tensor.AddFunc("Ceil", Ceil) + tensor.AddFunc("Cos", Cos) + tensor.AddFunc("Cosh", Cosh) + tensor.AddFunc("Erf", Erf) + tensor.AddFunc("Erfc", Erfc) + tensor.AddFunc("Erfcinv", Erfcinv) + tensor.AddFunc("Erfinv", Erfinv) + tensor.AddFunc("Exp", Exp) + tensor.AddFunc("Exp2", Exp2) + tensor.AddFunc("Expm1", Expm1) + tensor.AddFunc("Floor", Floor) + tensor.AddFunc("Gamma", Gamma) + tensor.AddFunc("J0", J0) + tensor.AddFunc("J1", J1) + tensor.AddFunc("Log", Log) + tensor.AddFunc("Log10", Log10) + tensor.AddFunc("Log1p", Log1p) + tensor.AddFunc("Log2", Log2) + tensor.AddFunc("Logb", Logb) + tensor.AddFunc("Round", Round) + tensor.AddFunc("RoundToEven", RoundToEven) + tensor.AddFunc("Sin", Sin) + tensor.AddFunc("Sinh", Sinh) + tensor.AddFunc("Sqrt", Sqrt) + tensor.AddFunc("Tan", Tan) + tensor.AddFunc("Tanh", Tanh) + tensor.AddFunc("Trunc", Trunc) + tensor.AddFunc("Y0", Y0) + tensor.AddFunc("Y1", Y1) +} + +func Abs(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AbsOut, in) +} + +func AbsOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Acos(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Acos(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AcosOut, in) +} + +func AcosOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Acosh(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Acosh(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AcoshOut, in) +} + +func AcoshOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Asin(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Asin(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AsinOut, in) +} + +func AsinOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Asinh(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Asinh(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AsinhOut, in) +} + +func AsinhOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Atan(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Atan(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AtanOut, in) +} + +func AtanOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Atanh(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Atanh(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(AtanhOut, in) +} + +func AtanhOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Cbrt(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Cbrt(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(CbrtOut, in) +} + +func CbrtOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Ceil(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Ceil(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(CeilOut, in) +} + +func CeilOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Cos(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Cos(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(CosOut, in) +} + +func CosOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Cosh(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Cosh(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(CoshOut, in) +} + +func CoshOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Erf(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Erf(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(ErfOut, in) +} + +func ErfOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Erfc(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Erfc(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(ErfcOut, in) +} + +func ErfcOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Erfcinv(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Erfcinv(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(ErfcinvOut, in) +} + +func ErfcinvOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Erfinv(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Erfinv(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(ErfinvOut, in) +} + +func ErfinvOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Exp(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Exp(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(ExpOut, in) +} + +func ExpOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Exp2(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Exp2(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Exp2Out, in) +} + +func Exp2Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Expm1(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Expm1(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Expm1Out, in) +} + +func Expm1Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Floor(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Floor(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(FloorOut, in) +} + +func FloorOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Gamma(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Gamma(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(GammaOut, in) +} + +func GammaOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func J0(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func J0(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(J0Out, in) +} + +func J0Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func J1(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func J1(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(J1Out, in) +} + +func J1Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Log(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Log(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(LogOut, in) +} + +func LogOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Log10(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Log10(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Log10Out, in) +} + +func Log10Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Log1p(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Log1p(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Log1pOut, in) +} + +func Log1pOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Log2(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Log2(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Log2Out, in) +} + +func Log2Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Logb(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Logb(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(LogbOut, in) +} + +func LogbOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Round(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Round(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(RoundOut, in) +} + +func RoundOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func RoundToEven(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func RoundToEven(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(RoundToEvenOut, in) +} + +func RoundToEvenOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Sin(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Sin(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(SinOut, in) +} + +func SinOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Sinh(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Sinh(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(SinhOut, in) +} + +func SinhOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Sqrt(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Sqrt(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(SqrtOut, in) +} + +func SqrtOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Tan(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Tan(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(TanOut, in) +} + +func TanOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Tanh(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Tanh(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(TanhOut, in) +} + +func TanhOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Trunc(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Trunc(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(TruncOut, in) +} + +func TruncOut(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Y0(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Y0(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Y0Out, in) +} + +func Y0Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(idx)), idx) }, in, out) return nil } -func Y1(in, out tensor.Tensor) error { - if err := tensor.SetShapeFromMustBeValues(out, in); err != nil { - return err - } +func Y1(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(Y1Out, in) +} + +func Y1Out(in tensor.Tensor, out tensor.Values) error { + tensor.SetShapeFrom(out, in) tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(idx)), idx) }, in, out) @@ -421,7 +495,7 @@ func Y1(in, out tensor.Tensor) error { } /* -func Atan2(y, in, out tensor.Tensor) +func Atan2(y, in tensor.Tensor, out tensor.Values) func Copysign(f, sign float64) float64 func Dim(x, y float64) float64 func Hypot(p, q float64) float64 @@ -446,8 +520,8 @@ func Float64frombits(b uint64) float64 func FMA(x, y, z float64) float64 -func Jn(n int, in, out tensor.Tensor) -func Yn(n int, in, out tensor.Tensor) +func Jn(n int, in tensor.Tensor, out tensor.Values) +func Yn(n int, in tensor.Tensor, out tensor.Values) func Ldexp(frac float64, exp int) float64 diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 333ac17076..be58ac0214 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -13,7 +13,7 @@ import ( ) type onef func(x float64) float64 -type tonef func(in, out tensor.Tensor) error +type tonef func(in tensor.Tensor, out tensor.Values) error // Equal does equal testing taking into account NaN func Equal(t *testing.T, trg, val float64) { @@ -45,7 +45,7 @@ func TestMath(t *testing.T) { cellout := cell2d.Clone() mfuncs := []onef{math.Abs, math.Acos, math.Acosh, math.Asin, math.Asinh, math.Atan, math.Atanh, math.Cbrt, math.Ceil, math.Cos, math.Cosh, math.Erf, math.Erfc, math.Erfcinv, math.Erfinv, math.Exp, math.Exp2, math.Expm1, math.Floor, math.Gamma, math.J0, math.J1, math.Log, math.Log10, math.Log1p, math.Log2, math.Logb, math.Round, math.RoundToEven, math.Sin, math.Sinh, math.Sqrt, math.Tan, math.Tanh, math.Trunc, math.Y0, math.Y1} - tfuncs := []tonef{Abs, Acos, Acosh, Asin, Asinh, Atan, Atanh, Cbrt, Ceil, Cos, Cosh, Erf, Erfc, Erfcinv, Erfinv, Exp, Exp2, Expm1, Floor, Gamma, J0, J1, Log, Log10, Log1p, Log2, Logb, Round, RoundToEven, Sin, Sinh, Sqrt, Tan, Tanh, Trunc, Y0, Y1} + tfuncs := []tonef{AbsOut, AcosOut, AcoshOut, AsinOut, AsinhOut, AtanOut, AtanhOut, CbrtOut, CeilOut, CosOut, CoshOut, ErfOut, ErfcOut, ErfcinvOut, ErfinvOut, ExpOut, Exp2Out, Expm1Out, FloorOut, GammaOut, J0Out, J1Out, LogOut, Log10Out, Log1pOut, Log2Out, LogbOut, RoundOut, RoundToEvenOut, SinOut, SinhOut, SqrtOut, TanOut, TanhOut, TruncOut, Y0Out, Y1Out} for i, fun := range mfuncs { tf := tfuncs[i] diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 9fa7b73ba8..7b29f06ebf 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -9,19 +9,19 @@ import ( ) func init() { - tensor.AddFunc("Assign", Assign, 0) - tensor.AddFunc("AddAssign", AddAssign, 0) - tensor.AddFunc("SubAssign", SubAssign, 0) - tensor.AddFunc("MulAssign", MulAssign, 0) - tensor.AddFunc("DivAssign", DivAssign, 0) + tensor.AddFunc("Assign", Assign) + tensor.AddFunc("AddAssign", AddAssign) + tensor.AddFunc("SubAssign", SubAssign) + tensor.AddFunc("MulAssign", MulAssign) + tensor.AddFunc("DivAssign", DivAssign) - tensor.AddFunc("Inc", Inc, 0) - tensor.AddFunc("Dec", Dec, 0) + tensor.AddFunc("Inc", Inc) + tensor.AddFunc("Dec", Dec) - tensor.AddFunc("Add", Add, 1) - tensor.AddFunc("Sub", Sub, 1) - tensor.AddFunc("Mul", Mul, 1) - tensor.AddFunc("Div", Div, 1) + tensor.AddFunc("Add", Add) + tensor.AddFunc("Sub", Sub) + tensor.AddFunc("Mul", Mul) + tensor.AddFunc("Div", Div) } // Assign assigns values from b into a. @@ -139,14 +139,17 @@ func Dec(a tensor.Tensor) error { } // Add adds two tensors into output. -func Add(a, b, out tensor.Tensor) error { +func Add(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2(AddOut, a, b) +} + +// AddOut adds two tensors into output. +func AddOut(a, b tensor.Tensor, out tensor.Values) error { as, bs, os, err := tensor.AlignShapes(a, b) if err != nil { return err } - if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { - return err - } + out.SetShapeSizes(os.Sizes...) olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return olen @@ -160,15 +163,18 @@ func Add(a, b, out tensor.Tensor) error { return nil } -// Sub subtracts two tensors into output. -func Sub(a, b, out tensor.Tensor) error { +// Sub subtracts tensors into output. +func Sub(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2(SubOut, a, b) +} + +// SubOut subtracts two tensors into output. +func SubOut(a, b tensor.Tensor, out tensor.Values) error { as, bs, os, err := tensor.AlignShapes(a, b) if err != nil { return err } - if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { - return err - } + out.SetShapeSizes(os.Sizes...) olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return olen @@ -182,15 +188,43 @@ func Sub(a, b, out tensor.Tensor) error { return nil } -// Mul multiplies two tensors into output. -func Mul(a, b, out tensor.Tensor) error { +// Mul multiplies tensors into output. +func Mul(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2(MulOut, a, b) +} + +// MulOut multiplies two tensors into output. +func MulOut(a, b tensor.Tensor, out tensor.Values) error { as, bs, os, err := tensor.AlignShapes(a, b) if err != nil { return err } - if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return olen + }, + func(idx int, tsr ...tensor.Tensor) { + oi := os.IndexFrom1D(idx) + ai := tensor.WrapIndex1D(as, oi...) + bi := tensor.WrapIndex1D(bs, oi...) + out.SetFloat1D(tsr[0].Float1D(ai)*tsr[1].Float1D(bi), idx) + }, a, b, out) + return nil +} + +// Div divides tensors into output. +func Div(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2(DivOut, a, b) +} + +// DivOut divides two tensors into output. +func DivOut(a, b tensor.Tensor, out tensor.Values) error { + as, bs, os, err := tensor.AlignShapes(a, b) + if err != nil { return err } + out.SetShapeSizes(os.Sizes...) olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return olen @@ -199,20 +233,48 @@ func Mul(a, b, out tensor.Tensor) error { oi := os.IndexFrom1D(idx) ai := tensor.WrapIndex1D(as, oi...) bi := tensor.WrapIndex1D(bs, oi...) - out.SetFloat1D(tsr[0].Float1D(ai)*tsr[1].Float1D(bi), idx) + out.SetFloat1D(tsr[0].Float1D(ai)/tsr[1].Float1D(bi), idx) }, a, b, out) return nil } -// Div divides two tensors into output. -func Div(a, b, out tensor.Tensor) error { +// Greater stores in the output the bool value a > b. +func Greater(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(GreaterOut, a, b) +} + +// GreaterOut stores in the output the bool value a > b. +func GreaterOut(a, b tensor.Tensor, out *tensor.Bool) error { as, bs, os, err := tensor.AlignShapes(a, b) if err != nil { return err } - if err := tensor.SetShapeSizesMustBeValues(out, os.Sizes...); err != nil { + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { + return olen + }, + func(idx int, tsr ...tensor.Tensor) { + oi := os.IndexFrom1D(idx) + ai := tensor.WrapIndex1D(as, oi...) + bi := tensor.WrapIndex1D(bs, oi...) + out.SetBool1D(tsr[0].Float1D(ai) > tsr[1].Float1D(bi), idx) + }, a, b, out) + return nil +} + +// Less stores in the output the bool value a > b. +func Less(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(LessOut, a, b) +} + +// LessOut stores in the output the bool value a > b. +func LessOut(a, b tensor.Tensor, out *tensor.Bool) error { + as, bs, os, err := tensor.AlignShapes(a, b) + if err != nil { return err } + out.SetShapeSizes(os.Sizes...) olen := os.Len() tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return olen @@ -221,7 +283,7 @@ func Div(a, b, out tensor.Tensor) error { oi := os.IndexFrom1D(idx) ai := tensor.WrapIndex1D(as, oi...) bi := tensor.WrapIndex1D(bs, oi...) - out.SetFloat1D(tsr[0].Float1D(ai)/tsr[1].Float1D(bi), idx) + out.SetBool1D(tsr[0].Float1D(ai) < tsr[1].Float1D(bi), idx) }, a, b, out) return nil } diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 70ebce45de..508a7174d1 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -11,17 +11,6 @@ import ( "github.com/stretchr/testify/assert" ) -func TestOpsCall(t *testing.T) { - x := tensor.NewIntScalar(1) - y := tensor.NewIntScalar(4) - - a := tensor.CallOut("Mul", x, tensor.NewIntScalar(2)) - b := tensor.CallOut("Add", x, y) - c := tensor.CallOut("Add", tensor.CallOut("Mul", x, y), tensor.CallOut("Mul", a, b)) - - assert.Equal(t, 14.0, c.Float1D(0)) -} - func TestOps(t *testing.T) { scalar := tensor.NewFloat64Scalar(-5.5) scb := scalar.Clone() @@ -44,100 +33,149 @@ func TestOps(t *testing.T) { cellout := cell2d.Clone() _ = cellout - Add(scalar, scb, scout) + AddOut(scalar, scb, scout) assert.Equal(t, -5.5+-4, scout.Float1D(0)) - Add(scalar, oned, oneout) + AddOut(scalar, oned, oneout) for i, v := range vals { assert.Equal(t, v+-5.5, oneout.Float1D(i)) } - Add(oned, oned, oneout) + AddOut(oned, oned, oneout) for i, v := range vals { assert.Equal(t, v+v, oneout.Float1D(i)) } - Add(cell2d, oned, cellout) + AddOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { assert.InDelta(t, v+v, cellout.FloatRowCell(ri, i), 1.0e-6) } } - Sub(scalar, scb, scout) + SubOut(scalar, scb, scout) assert.Equal(t, -5.5 - -4, scout.Float1D(0)) - Sub(scb, scalar, scout) + SubOut(scb, scalar, scout) assert.Equal(t, -4 - -5.5, scout.Float1D(0)) - Sub(scalar, oned, oneout) + SubOut(scalar, oned, oneout) for i, v := range vals { assert.Equal(t, -5.5-v, oneout.Float1D(i)) } - Sub(oned, scalar, oneout) + SubOut(oned, scalar, oneout) for i, v := range vals { assert.Equal(t, v - -5.5, oneout.Float1D(i)) } - Sub(oned, oned, oneout) + SubOut(oned, oned, oneout) for i, v := range vals { assert.Equal(t, v-v, oneout.Float1D(i)) } - Sub(cell2d, oned, cellout) + SubOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { assert.InDelta(t, v-v, cellout.FloatRowCell(ri, i), 1.0e-6) } } - Mul(scalar, scb, scout) + MulOut(scalar, scb, scout) assert.Equal(t, -5.5*-4, scout.Float1D(0)) - Mul(scalar, oned, oneout) + MulOut(scalar, oned, oneout) for i, v := range vals { assert.Equal(t, v*-5.5, oneout.Float1D(i)) } - Mul(oned, oned, oneout) + MulOut(oned, oned, oneout) for i, v := range vals { assert.Equal(t, v*v, oneout.Float1D(i)) } - Mul(cell2d, oned, cellout) + MulOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { assert.InDelta(t, v*v, cellout.FloatRowCell(ri, i), 1.0e-6) } } - Div(scalar, scb, scout) + DivOut(scalar, scb, scout) assert.Equal(t, -5.5/-4, scout.Float1D(0)) - Div(scb, scalar, scout) + DivOut(scb, scalar, scout) assert.Equal(t, -4/-5.5, scout.Float1D(0)) - Div(scalar, oned, oneout) + DivOut(scalar, oned, oneout) for i, v := range vals { assert.Equal(t, -5.5/v, oneout.Float1D(i)) } - Div(oned, scalar, oneout) + DivOut(oned, scalar, oneout) for i, v := range vals { assert.Equal(t, v/-5.5, oneout.Float1D(i)) } - Div(oned, oned, oneout) + DivOut(oned, oned, oneout) for i, v := range vals { assert.Equal(t, v/v, oneout.Float1D(i)) } - Div(cell2d, oned, cellout) + DivOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { assert.InDelta(t, v/v, cellout.FloatRowCell(ri, i), 1.0e-6) } } + onedc := tensor.Clone(oned) + AddAssign(onedc, scalar) + for i, v := range vals { + assert.Equal(t, v+-5.5, onedc.Float1D(i)) + } + + SubAssign(onedc, scalar) + for i, v := range vals { + assert.InDelta(t, v, onedc.Float1D(i), 1.0e-8) + } + + MulAssign(onedc, scalar) + for i, v := range vals { + assert.InDelta(t, v*-5.5, onedc.Float1D(i), 1.0e-7) + } + + DivAssign(onedc, scalar) + for i, v := range vals { + assert.InDelta(t, v, onedc.Float1D(i), 1.0e-7) + } + + Inc(onedc) + for i, v := range vals { + assert.InDelta(t, v+1, onedc.Float1D(i), 1.0e-7) + } + + Dec(onedc) + for i, v := range vals { + assert.InDelta(t, v, onedc.Float1D(i), 1.0e-7) + } +} + +func TestBoolOps(t *testing.T) { + ar := tensor.NewSliceInts(12) + // fmt.Println(v) + bo := tensor.NewBool() + sc := tensor.NewIntScalar(6) + + GreaterOut(ar, sc, bo) + // fmt.Println(bo) + for i, v := range ar.Values { + assert.Equal(t, v > 6, bo.Bool1D(i)) + } + + LessOut(ar, sc, bo) + // fmt.Println(bo) + for i, v := range ar.Values { + assert.Equal(t, v < 6, bo.Bool1D(i)) + } } From fdcf5747e979970c9578f3029ff8db47f6cc5458 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 12:03:13 -0700 Subject: [PATCH 130/311] update parsing to new function calling, yaegicore update --- goal/transpile/math.go | 8 +- goal/transpile/transpile_test.go | 50 ++-- ...cogentcore_org-core-tensor-stats-metric.go | 23 +- .../cogentcore_org-core-tensor-stats-stats.go | 30 ++- .../cogentcore_org-core-tensor-tmath.go | 141 ++++++---- .../symbols/cogentcore_org-core-tensor.go | 255 +++++++++--------- 6 files changed, 299 insertions(+), 208 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 71b9984649..1bb76499e9 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -596,12 +596,15 @@ func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { if pkgName != "" { funName = pkgName + "." + funName _, err = tensor.FuncByName(funName) - } else { + } else { // non-package qualified names are _only_ in tmath! can be lowercase _, err = tensor.FuncByName(funName) if err != nil { funName = strings.ToUpper(funName[:1]) + funName[1:] // first letter uppercased _, err = tensor.FuncByName(funName) } + if err == nil { // registered, must be in tmath + funName = "tmath." + funName + } } if err != nil { // not a registered tensor function mp.startFunc(funName, false) @@ -610,10 +613,11 @@ func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { return } mp.startFunc(funName, true) // tensors - mp.addToken(token.LPAREN) + mp.idx += 1 if pkgName != "" { mp.idx += 2 // . and selector } + mp.addToken(token.LPAREN) } func (mp *mathParse) ident(id *ast.Ident) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 118c3f915d..21dd2c34a2 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -203,31 +203,31 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, - {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, - {"# a = x + y", `a = tmath.Add(x, y)`}, - {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, - // {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, // TODO: not yet - {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - // {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, // TODO: not yet - {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, - {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, - {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, - {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, - {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, - {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - {"# c := cos(a)", `c := tmath.Cos(a)`}, + // {"# x := 1", `x := tensor.NewIntScalar(1)`}, + // {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, + // {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, + // {"# a = x + y", `a = tmath.Add(x, y)`}, + // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + // {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, + // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + // {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, + // // {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, // TODO: not yet + // {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + // {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + // {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + // // {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, // TODO: not yet + // {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + // {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, + // {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + // {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, + // {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + // {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + // {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + // {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + // {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + // {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + // {"# c := cos(a)", `c := tmath.Cos(a)`}, {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go index 4047cf4fe9..9d2a09cad6 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -11,23 +11,38 @@ func init() { Symbols["cogentcore.org/core/tensor/stats/metric/metric"] = map[string]reflect.Value{ // function, constant and variable definitions "Abs": reflect.ValueOf(metric.Abs), + "AbsOut": reflect.ValueOf(metric.AbsOut), "AsMetricFunc": reflect.ValueOf(metric.AsMetricFunc), + "AsMetricOutFunc": reflect.ValueOf(metric.AsMetricOutFunc), "ClosestRow": reflect.ValueOf(metric.ClosestRow), "Correlation": reflect.ValueOf(metric.Correlation), + "CorrelationOut": reflect.ValueOf(metric.CorrelationOut), "CorrelationOut64": reflect.ValueOf(metric.CorrelationOut64), "Cosine": reflect.ValueOf(metric.Cosine), + "CosineOut": reflect.ValueOf(metric.CosineOut), "CosineOut64": reflect.ValueOf(metric.CosineOut64), "Covariance": reflect.ValueOf(metric.Covariance), "CovarianceMatrix": reflect.ValueOf(metric.CovarianceMatrix), + "CovarianceMatrixOut": reflect.ValueOf(metric.CovarianceMatrixOut), + "CovarianceOut": reflect.ValueOf(metric.CovarianceOut), "CrossEntropy": reflect.ValueOf(metric.CrossEntropy), + "CrossEntropyOut": reflect.ValueOf(metric.CrossEntropyOut), "CrossMatrix": reflect.ValueOf(metric.CrossMatrix), + "CrossMatrixOut": reflect.ValueOf(metric.CrossMatrixOut), "Euclidean": reflect.ValueOf(metric.Euclidean), "EuclideanBinTol": reflect.ValueOf(metric.EuclideanBinTol), + "EuclideanBinTolOut": reflect.ValueOf(metric.EuclideanBinTolOut), + "EuclideanOut": reflect.ValueOf(metric.EuclideanOut), "Hamming": reflect.ValueOf(metric.Hamming), + "HammingOut": reflect.ValueOf(metric.HammingOut), "InnerProduct": reflect.ValueOf(metric.InnerProduct), + "InnerProductOut": reflect.ValueOf(metric.InnerProductOut), "InvCorrelation": reflect.ValueOf(metric.InvCorrelation), + "InvCorrelationOut": reflect.ValueOf(metric.InvCorrelationOut), "InvCosine": reflect.ValueOf(metric.InvCosine), + "InvCosineOut": reflect.ValueOf(metric.InvCosineOut), "Matrix": reflect.ValueOf(metric.Matrix), + "MatrixOut": reflect.ValueOf(metric.MatrixOut), "MetricAbs": reflect.ValueOf(metric.MetricAbs), "MetricCorrelation": reflect.ValueOf(metric.MetricCorrelation), "MetricCosine": reflect.ValueOf(metric.MetricCosine), @@ -46,10 +61,13 @@ func init() { "NFunc": reflect.ValueOf(metric.NFunc), "PCA": reflect.ValueOf(metric.PCA), "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), + "ProjectOnMatrixColumnOut": reflect.ValueOf(metric.ProjectOnMatrixColumnOut), "SVD": reflect.ValueOf(metric.SVD), "SumSquares": reflect.ValueOf(metric.SumSquares), "SumSquaresBinTol": reflect.ValueOf(metric.SumSquaresBinTol), + "SumSquaresBinTolOut": reflect.ValueOf(metric.SumSquaresBinTolOut), "SumSquaresBinTolScaleOut64": reflect.ValueOf(metric.SumSquaresBinTolScaleOut64), + "SumSquaresOut": reflect.ValueOf(metric.SumSquaresOut), "SumSquaresOut64": reflect.ValueOf(metric.SumSquaresOut64), "SumSquaresScaleOut64": reflect.ValueOf(metric.SumSquaresScaleOut64), "TriangularLIndicies": reflect.ValueOf(metric.TriangularLIndicies), @@ -62,7 +80,8 @@ func init() { "Vectorize3Out64": reflect.ValueOf(metric.Vectorize3Out64), // type definitions - "MetricFunc": reflect.ValueOf((*metric.MetricFunc)(nil)), - "Metrics": reflect.ValueOf((*metric.Metrics)(nil)), + "MetricFunc": reflect.ValueOf((*metric.MetricFunc)(nil)), + "MetricOutFunc": reflect.ValueOf((*metric.MetricOutFunc)(nil)), + "Metrics": reflect.ValueOf((*metric.Metrics)(nil)), } } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index 2678c4bfae..ac20b76d89 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -12,8 +12,11 @@ func init() { // function, constant and variable definitions "AsStatsFunc": reflect.ValueOf(stats.AsStatsFunc), "Binarize": reflect.ValueOf(stats.Binarize), + "BinarizeOut": reflect.ValueOf(stats.BinarizeOut), "Clamp": reflect.ValueOf(stats.Clamp), + "ClampOut": reflect.ValueOf(stats.ClampOut), "Count": reflect.ValueOf(stats.Count), + "CountOut": reflect.ValueOf(stats.CountOut), "CountOut64": reflect.ValueOf(stats.CountOut64), "Describe": reflect.ValueOf(stats.Describe), "DescribeTable": reflect.ValueOf(stats.DescribeTable), @@ -24,21 +27,34 @@ func init() { "GroupStats": reflect.ValueOf(stats.GroupStats), "Groups": reflect.ValueOf(stats.Groups), "L2Norm": reflect.ValueOf(stats.L2Norm), + "L2NormOut": reflect.ValueOf(stats.L2NormOut), "L2NormOut64": reflect.ValueOf(stats.L2NormOut64), "Max": reflect.ValueOf(stats.Max), "MaxAbs": reflect.ValueOf(stats.MaxAbs), + "MaxAbsOut": reflect.ValueOf(stats.MaxAbsOut), + "MaxOut": reflect.ValueOf(stats.MaxOut), "Mean": reflect.ValueOf(stats.Mean), + "MeanOut": reflect.ValueOf(stats.MeanOut), "MeanOut64": reflect.ValueOf(stats.MeanOut64), "Median": reflect.ValueOf(stats.Median), + "MedianOut": reflect.ValueOf(stats.MedianOut), "Min": reflect.ValueOf(stats.Min), "MinAbs": reflect.ValueOf(stats.MinAbs), + "MinAbsOut": reflect.ValueOf(stats.MinAbsOut), + "MinOut": reflect.ValueOf(stats.MinOut), "NFunc": reflect.ValueOf(stats.NFunc), "Prod": reflect.ValueOf(stats.Prod), + "ProdOut": reflect.ValueOf(stats.ProdOut), "Q1": reflect.ValueOf(stats.Q1), + "Q1Out": reflect.ValueOf(stats.Q1Out), "Q3": reflect.ValueOf(stats.Q3), + "Q3Out": reflect.ValueOf(stats.Q3Out), "Quantiles": reflect.ValueOf(stats.Quantiles), + "QuantilesOut": reflect.ValueOf(stats.QuantilesOut), "Sem": reflect.ValueOf(stats.Sem), + "SemOut": reflect.ValueOf(stats.SemOut), "SemPop": reflect.ValueOf(stats.SemPop), + "SemPopOut": reflect.ValueOf(stats.SemPopOut), "StatCount": reflect.ValueOf(stats.StatCount), "StatL1Norm": reflect.ValueOf(stats.StatL1Norm), "StatL2Norm": reflect.ValueOf(stats.StatL2Norm), @@ -63,23 +79,31 @@ func init() { "StatsN": reflect.ValueOf(stats.StatsN), "StatsValues": reflect.ValueOf(stats.StatsValues), "Std": reflect.ValueOf(stats.Std), + "StdOut": reflect.ValueOf(stats.StdOut), "StdOut64": reflect.ValueOf(stats.StdOut64), "StdPop": reflect.ValueOf(stats.StdPop), + "StdPopOut": reflect.ValueOf(stats.StdPopOut), "StripPackage": reflect.ValueOf(stats.StripPackage), "Sum": reflect.ValueOf(stats.Sum), "SumAbs": reflect.ValueOf(stats.SumAbs), + "SumAbsOut": reflect.ValueOf(stats.SumAbsOut), + "SumOut": reflect.ValueOf(stats.SumOut), "SumOut64": reflect.ValueOf(stats.SumOut64), "SumSq": reflect.ValueOf(stats.SumSq), "SumSqDevOut64": reflect.ValueOf(stats.SumSqDevOut64), + "SumSqOut": reflect.ValueOf(stats.SumSqOut), "SumSqOut64": reflect.ValueOf(stats.SumSqOut64), "SumSqScaleOut64": reflect.ValueOf(stats.SumSqScaleOut64), "TableGroupDescribe": reflect.ValueOf(stats.TableGroupDescribe), "TableGroupStats": reflect.ValueOf(stats.TableGroupStats), "TableGroups": reflect.ValueOf(stats.TableGroups), "UnitNorm": reflect.ValueOf(stats.UnitNorm), + "UnitNormOut": reflect.ValueOf(stats.UnitNormOut), "Var": reflect.ValueOf(stats.Var), + "VarOut": reflect.ValueOf(stats.VarOut), "VarOut64": reflect.ValueOf(stats.VarOut64), "VarPop": reflect.ValueOf(stats.VarPop), + "VarPopOut": reflect.ValueOf(stats.VarPopOut), "VarPopOut64": reflect.ValueOf(stats.VarPopOut64), "Vec2inFunc": reflect.ValueOf(stats.Vec2inFunc), "Vec2outFunc": reflect.ValueOf(stats.Vec2outFunc), @@ -87,9 +111,11 @@ func init() { "Vectorize2Out64": reflect.ValueOf(stats.Vectorize2Out64), "VectorizeOut64": reflect.ValueOf(stats.VectorizeOut64), "ZScore": reflect.ValueOf(stats.ZScore), + "ZScoreOut": reflect.ValueOf(stats.ZScoreOut), // type definitions - "Stats": reflect.ValueOf((*stats.Stats)(nil)), - "StatsFunc": reflect.ValueOf((*stats.StatsFunc)(nil)), + "Stats": reflect.ValueOf((*stats.Stats)(nil)), + "StatsFunc": reflect.ValueOf((*stats.StatsFunc)(nil)), + "StatsOutFunc": reflect.ValueOf((*stats.StatsOutFunc)(nil)), } } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go index cd596d5bbb..4413bcf6ff 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -10,53 +10,98 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/tmath/tmath"] = map[string]reflect.Value{ // function, constant and variable definitions - "Abs": reflect.ValueOf(tmath.Abs), - "Acos": reflect.ValueOf(tmath.Acos), - "Acosh": reflect.ValueOf(tmath.Acosh), - "Add": reflect.ValueOf(tmath.Add), - "AddAssign": reflect.ValueOf(tmath.AddAssign), - "Asin": reflect.ValueOf(tmath.Asin), - "Asinh": reflect.ValueOf(tmath.Asinh), - "Assign": reflect.ValueOf(tmath.Assign), - "Atan": reflect.ValueOf(tmath.Atan), - "Atanh": reflect.ValueOf(tmath.Atanh), - "Cbrt": reflect.ValueOf(tmath.Cbrt), - "Ceil": reflect.ValueOf(tmath.Ceil), - "Cos": reflect.ValueOf(tmath.Cos), - "Cosh": reflect.ValueOf(tmath.Cosh), - "Dec": reflect.ValueOf(tmath.Dec), - "Div": reflect.ValueOf(tmath.Div), - "DivAssign": reflect.ValueOf(tmath.DivAssign), - "Erf": reflect.ValueOf(tmath.Erf), - "Erfc": reflect.ValueOf(tmath.Erfc), - "Erfcinv": reflect.ValueOf(tmath.Erfcinv), - "Erfinv": reflect.ValueOf(tmath.Erfinv), - "Exp": reflect.ValueOf(tmath.Exp), - "Exp2": reflect.ValueOf(tmath.Exp2), - "Expm1": reflect.ValueOf(tmath.Expm1), - "Floor": reflect.ValueOf(tmath.Floor), - "Gamma": reflect.ValueOf(tmath.Gamma), - "Inc": reflect.ValueOf(tmath.Inc), - "J0": reflect.ValueOf(tmath.J0), - "J1": reflect.ValueOf(tmath.J1), - "Log": reflect.ValueOf(tmath.Log), - "Log10": reflect.ValueOf(tmath.Log10), - "Log1p": reflect.ValueOf(tmath.Log1p), - "Log2": reflect.ValueOf(tmath.Log2), - "Logb": reflect.ValueOf(tmath.Logb), - "Mul": reflect.ValueOf(tmath.Mul), - "MulAssign": reflect.ValueOf(tmath.MulAssign), - "Round": reflect.ValueOf(tmath.Round), - "RoundToEven": reflect.ValueOf(tmath.RoundToEven), - "Sin": reflect.ValueOf(tmath.Sin), - "Sinh": reflect.ValueOf(tmath.Sinh), - "Sqrt": reflect.ValueOf(tmath.Sqrt), - "Sub": reflect.ValueOf(tmath.Sub), - "SubAssign": reflect.ValueOf(tmath.SubAssign), - "Tan": reflect.ValueOf(tmath.Tan), - "Tanh": reflect.ValueOf(tmath.Tanh), - "Trunc": reflect.ValueOf(tmath.Trunc), - "Y0": reflect.ValueOf(tmath.Y0), - "Y1": reflect.ValueOf(tmath.Y1), + "Abs": reflect.ValueOf(tmath.Abs), + "AbsOut": reflect.ValueOf(tmath.AbsOut), + "Acos": reflect.ValueOf(tmath.Acos), + "AcosOut": reflect.ValueOf(tmath.AcosOut), + "Acosh": reflect.ValueOf(tmath.Acosh), + "AcoshOut": reflect.ValueOf(tmath.AcoshOut), + "Add": reflect.ValueOf(tmath.Add), + "AddAssign": reflect.ValueOf(tmath.AddAssign), + "AddOut": reflect.ValueOf(tmath.AddOut), + "Asin": reflect.ValueOf(tmath.Asin), + "AsinOut": reflect.ValueOf(tmath.AsinOut), + "Asinh": reflect.ValueOf(tmath.Asinh), + "AsinhOut": reflect.ValueOf(tmath.AsinhOut), + "Assign": reflect.ValueOf(tmath.Assign), + "Atan": reflect.ValueOf(tmath.Atan), + "AtanOut": reflect.ValueOf(tmath.AtanOut), + "Atanh": reflect.ValueOf(tmath.Atanh), + "AtanhOut": reflect.ValueOf(tmath.AtanhOut), + "Cbrt": reflect.ValueOf(tmath.Cbrt), + "CbrtOut": reflect.ValueOf(tmath.CbrtOut), + "Ceil": reflect.ValueOf(tmath.Ceil), + "CeilOut": reflect.ValueOf(tmath.CeilOut), + "Cos": reflect.ValueOf(tmath.Cos), + "CosOut": reflect.ValueOf(tmath.CosOut), + "Cosh": reflect.ValueOf(tmath.Cosh), + "CoshOut": reflect.ValueOf(tmath.CoshOut), + "Dec": reflect.ValueOf(tmath.Dec), + "Div": reflect.ValueOf(tmath.Div), + "DivAssign": reflect.ValueOf(tmath.DivAssign), + "DivOut": reflect.ValueOf(tmath.DivOut), + "Erf": reflect.ValueOf(tmath.Erf), + "ErfOut": reflect.ValueOf(tmath.ErfOut), + "Erfc": reflect.ValueOf(tmath.Erfc), + "ErfcOut": reflect.ValueOf(tmath.ErfcOut), + "Erfcinv": reflect.ValueOf(tmath.Erfcinv), + "ErfcinvOut": reflect.ValueOf(tmath.ErfcinvOut), + "Erfinv": reflect.ValueOf(tmath.Erfinv), + "ErfinvOut": reflect.ValueOf(tmath.ErfinvOut), + "Exp": reflect.ValueOf(tmath.Exp), + "Exp2": reflect.ValueOf(tmath.Exp2), + "Exp2Out": reflect.ValueOf(tmath.Exp2Out), + "ExpOut": reflect.ValueOf(tmath.ExpOut), + "Expm1": reflect.ValueOf(tmath.Expm1), + "Expm1Out": reflect.ValueOf(tmath.Expm1Out), + "Floor": reflect.ValueOf(tmath.Floor), + "FloorOut": reflect.ValueOf(tmath.FloorOut), + "Gamma": reflect.ValueOf(tmath.Gamma), + "GammaOut": reflect.ValueOf(tmath.GammaOut), + "Greater": reflect.ValueOf(tmath.Greater), + "GreaterOut": reflect.ValueOf(tmath.GreaterOut), + "Inc": reflect.ValueOf(tmath.Inc), + "J0": reflect.ValueOf(tmath.J0), + "J0Out": reflect.ValueOf(tmath.J0Out), + "J1": reflect.ValueOf(tmath.J1), + "J1Out": reflect.ValueOf(tmath.J1Out), + "Less": reflect.ValueOf(tmath.Less), + "LessOut": reflect.ValueOf(tmath.LessOut), + "Log": reflect.ValueOf(tmath.Log), + "Log10": reflect.ValueOf(tmath.Log10), + "Log10Out": reflect.ValueOf(tmath.Log10Out), + "Log1p": reflect.ValueOf(tmath.Log1p), + "Log1pOut": reflect.ValueOf(tmath.Log1pOut), + "Log2": reflect.ValueOf(tmath.Log2), + "Log2Out": reflect.ValueOf(tmath.Log2Out), + "LogOut": reflect.ValueOf(tmath.LogOut), + "Logb": reflect.ValueOf(tmath.Logb), + "LogbOut": reflect.ValueOf(tmath.LogbOut), + "Mul": reflect.ValueOf(tmath.Mul), + "MulAssign": reflect.ValueOf(tmath.MulAssign), + "MulOut": reflect.ValueOf(tmath.MulOut), + "Round": reflect.ValueOf(tmath.Round), + "RoundOut": reflect.ValueOf(tmath.RoundOut), + "RoundToEven": reflect.ValueOf(tmath.RoundToEven), + "RoundToEvenOut": reflect.ValueOf(tmath.RoundToEvenOut), + "Sin": reflect.ValueOf(tmath.Sin), + "SinOut": reflect.ValueOf(tmath.SinOut), + "Sinh": reflect.ValueOf(tmath.Sinh), + "SinhOut": reflect.ValueOf(tmath.SinhOut), + "Sqrt": reflect.ValueOf(tmath.Sqrt), + "SqrtOut": reflect.ValueOf(tmath.SqrtOut), + "Sub": reflect.ValueOf(tmath.Sub), + "SubAssign": reflect.ValueOf(tmath.SubAssign), + "SubOut": reflect.ValueOf(tmath.SubOut), + "Tan": reflect.ValueOf(tmath.Tan), + "TanOut": reflect.ValueOf(tmath.TanOut), + "Tanh": reflect.ValueOf(tmath.Tanh), + "TanhOut": reflect.ValueOf(tmath.TanhOut), + "Trunc": reflect.ValueOf(tmath.Trunc), + "TruncOut": reflect.ValueOf(tmath.TruncOut), + "Y0": reflect.ValueOf(tmath.Y0), + "Y0Out": reflect.ValueOf(tmath.Y0Out), + "Y1": reflect.ValueOf(tmath.Y1), + "Y1Out": reflect.ValueOf(tmath.Y1Out), } } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 7f233a7f53..6932944829 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -11,135 +11,132 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/tensor"] = map[string]reflect.Value{ // function, constant and variable definitions - "AddFunc": reflect.ValueOf(tensor.AddFunc), - "AddShapes": reflect.ValueOf(tensor.AddShapes), - "AlignForAssign": reflect.ValueOf(tensor.AlignForAssign), - "AlignShapes": reflect.ValueOf(tensor.AlignShapes), - "AnyFirstArg": reflect.ValueOf(tensor.AnyFirstArg), - "As1D": reflect.ValueOf(tensor.As1D), - "AsFloat32Tensor": reflect.ValueOf(tensor.AsFloat32Tensor), - "AsFloat64Scalar": reflect.ValueOf(tensor.AsFloat64Scalar), - "AsFloat64Slice": reflect.ValueOf(tensor.AsFloat64Slice), - "AsFloat64Tensor": reflect.ValueOf(tensor.AsFloat64Tensor), - "AsIndexed": reflect.ValueOf(tensor.AsIndexed), - "AsIntScalar": reflect.ValueOf(tensor.AsIntScalar), - "AsIntSlice": reflect.ValueOf(tensor.AsIntSlice), - "AsIntTensor": reflect.ValueOf(tensor.AsIntTensor), - "AsMasked": reflect.ValueOf(tensor.AsMasked), - "AsReshaped": reflect.ValueOf(tensor.AsReshaped), - "AsRows": reflect.ValueOf(tensor.AsRows), - "AsSliced": reflect.ValueOf(tensor.AsSliced), - "AsStringScalar": reflect.ValueOf(tensor.AsStringScalar), - "AsStringSlice": reflect.ValueOf(tensor.AsStringSlice), - "AsStringTensor": reflect.ValueOf(tensor.AsStringTensor), - "Ascending": reflect.ValueOf(tensor.Ascending), - "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), - "BoolToInt": reflect.ValueOf(tensor.BoolToInt), - "Calc": reflect.ValueOf(tensor.Calc), - "Call": reflect.ValueOf(tensor.Call), - "CallAny": reflect.ValueOf(tensor.CallAny), - "CallOut": reflect.ValueOf(tensor.CallOut), - "CallOutMulti": reflect.ValueOf(tensor.CallOutMulti), - "Cells1D": reflect.ValueOf(tensor.Cells1D), - "CellsSize": reflect.ValueOf(tensor.CellsSize), - "Clone": reflect.ValueOf(tensor.Clone), - "ColumnMajorStrides": reflect.ValueOf(tensor.ColumnMajorStrides), - "Comma": reflect.ValueOf(tensor.Comma), - "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), - "DelimsN": reflect.ValueOf(tensor.DelimsN), - "DelimsValues": reflect.ValueOf(tensor.DelimsValues), - "Descending": reflect.ValueOf(tensor.Descending), - "Detect": reflect.ValueOf(tensor.Detect), - "Ellipsis": reflect.ValueOf(tensor.Ellipsis), - "Flatten": reflect.ValueOf(tensor.Flatten), - "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), - "Float64ToString": reflect.ValueOf(tensor.Float64ToString), - "FullAxis": reflect.ValueOf(tensor.FullAxis), - "FuncByName": reflect.ValueOf(tensor.FuncByName), - "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), - "IntToBool": reflect.ValueOf(tensor.IntToBool), - "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), - "MustBeSameShape": reflect.ValueOf(tensor.MustBeSameShape), - "MustBeValues": reflect.ValueOf(tensor.MustBeValues), - "NFirstLen": reflect.ValueOf(tensor.NFirstLen), - "NFirstRows": reflect.ValueOf(tensor.NFirstRows), - "NMinLen": reflect.ValueOf(tensor.NMinLen), - "NewAxis": reflect.ValueOf(tensor.NewAxis), - "NewBool": reflect.ValueOf(tensor.NewBool), - "NewBoolShape": reflect.ValueOf(tensor.NewBoolShape), - "NewByte": reflect.ValueOf(tensor.NewByte), - "NewFloat32": reflect.ValueOf(tensor.NewFloat32), - "NewFloat64": reflect.ValueOf(tensor.NewFloat64), - "NewFloat64FromValues": reflect.ValueOf(tensor.NewFloat64FromValues), - "NewFloat64Full": reflect.ValueOf(tensor.NewFloat64Full), - "NewFloat64Ones": reflect.ValueOf(tensor.NewFloat64Ones), - "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), - "NewFunc": reflect.ValueOf(tensor.NewFunc), - "NewIndexed": reflect.ValueOf(tensor.NewIndexed), - "NewInt": reflect.ValueOf(tensor.NewInt), - "NewInt32": reflect.ValueOf(tensor.NewInt32), - "NewIntFromValues": reflect.ValueOf(tensor.NewIntFromValues), - "NewIntFull": reflect.ValueOf(tensor.NewIntFull), - "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), - "NewMasked": reflect.ValueOf(tensor.NewMasked), - "NewOfType": reflect.ValueOf(tensor.NewOfType), - "NewReshaped": reflect.ValueOf(tensor.NewReshaped), - "NewRowCellsView": reflect.ValueOf(tensor.NewRowCellsView), - "NewRows": reflect.ValueOf(tensor.NewRows), - "NewShape": reflect.ValueOf(tensor.NewShape), - "NewSlice": reflect.ValueOf(tensor.NewSlice), - "NewSliceInts": reflect.ValueOf(tensor.NewSliceInts), - "NewSliced": reflect.ValueOf(tensor.NewSliced), - "NewString": reflect.ValueOf(tensor.NewString), - "NewStringFromValues": reflect.ValueOf(tensor.NewStringFromValues), - "NewStringFull": reflect.ValueOf(tensor.NewStringFull), - "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), - "NewStringShape": reflect.ValueOf(tensor.NewStringShape), - "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), - "OddColumn": reflect.ValueOf(tensor.OddColumn), - "OddRow": reflect.ValueOf(tensor.OddRow), - "OpenCSV": reflect.ValueOf(tensor.OpenCSV), - "Precision": reflect.ValueOf(tensor.Precision), - "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), - "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), - "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), - "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), - "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), - "Projection2DString": reflect.ValueOf(tensor.Projection2DString), - "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), - "Range": reflect.ValueOf(tensor.Range), - "ReadCSV": reflect.ValueOf(tensor.ReadCSV), - "Reshape": reflect.ValueOf(tensor.Reshape), - "Reslice": reflect.ValueOf(tensor.Reslice), - "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), - "SaveCSV": reflect.ValueOf(tensor.SaveCSV), - "SetAllFloat64": reflect.ValueOf(tensor.SetAllFloat64), - "SetAllInt": reflect.ValueOf(tensor.SetAllInt), - "SetAllString": reflect.ValueOf(tensor.SetAllString), - "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), - "SetPrecision": reflect.ValueOf(tensor.SetPrecision), - "SetShape": reflect.ValueOf(tensor.SetShape), - "SetShapeFrom": reflect.ValueOf(tensor.SetShapeFrom), - "SetShapeFromMustBeValues": reflect.ValueOf(tensor.SetShapeFromMustBeValues), - "SetShapeNames": reflect.ValueOf(tensor.SetShapeNames), - "SetShapeSizesFromTensor": reflect.ValueOf(tensor.SetShapeSizesFromTensor), - "SetShapeSizesMustBeValues": reflect.ValueOf(tensor.SetShapeSizesMustBeValues), - "ShapeNames": reflect.ValueOf(tensor.ShapeNames), - "SlicesMagicN": reflect.ValueOf(tensor.SlicesMagicN), - "SlicesMagicValues": reflect.ValueOf(tensor.SlicesMagicValues), - "Space": reflect.ValueOf(tensor.Space), - "Sprint": reflect.ValueOf(tensor.Sprint), - "Squeeze": reflect.ValueOf(tensor.Squeeze), - "StableSort": reflect.ValueOf(tensor.StableSort), - "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), - "Tab": reflect.ValueOf(tensor.Tab), - "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), - "UnstableSort": reflect.ValueOf(tensor.UnstableSort), - "Vectorize": reflect.ValueOf(tensor.Vectorize), - "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), - "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), - "WrapIndex1D": reflect.ValueOf(tensor.WrapIndex1D), - "WriteCSV": reflect.ValueOf(tensor.WriteCSV), + "AddFunc": reflect.ValueOf(tensor.AddFunc), + "AddShapes": reflect.ValueOf(tensor.AddShapes), + "AlignForAssign": reflect.ValueOf(tensor.AlignForAssign), + "AlignShapes": reflect.ValueOf(tensor.AlignShapes), + "As1D": reflect.ValueOf(tensor.As1D), + "AsFloat32Tensor": reflect.ValueOf(tensor.AsFloat32Tensor), + "AsFloat64Scalar": reflect.ValueOf(tensor.AsFloat64Scalar), + "AsFloat64Slice": reflect.ValueOf(tensor.AsFloat64Slice), + "AsFloat64Tensor": reflect.ValueOf(tensor.AsFloat64Tensor), + "AsIndexed": reflect.ValueOf(tensor.AsIndexed), + "AsIntScalar": reflect.ValueOf(tensor.AsIntScalar), + "AsIntSlice": reflect.ValueOf(tensor.AsIntSlice), + "AsIntTensor": reflect.ValueOf(tensor.AsIntTensor), + "AsMasked": reflect.ValueOf(tensor.AsMasked), + "AsReshaped": reflect.ValueOf(tensor.AsReshaped), + "AsRows": reflect.ValueOf(tensor.AsRows), + "AsSliced": reflect.ValueOf(tensor.AsSliced), + "AsStringScalar": reflect.ValueOf(tensor.AsStringScalar), + "AsStringSlice": reflect.ValueOf(tensor.AsStringSlice), + "AsStringTensor": reflect.ValueOf(tensor.AsStringTensor), + "Ascending": reflect.ValueOf(tensor.Ascending), + "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), + "BoolToInt": reflect.ValueOf(tensor.BoolToInt), + "Calc": reflect.ValueOf(tensor.Calc), + "CallOut1": reflect.ValueOf(tensor.CallOut1), + "CallOut2": reflect.ValueOf(tensor.CallOut2), + "CallOut2Bool": reflect.ValueOf(tensor.CallOut2Bool), + "CallOut3": reflect.ValueOf(tensor.CallOut3), + "Cells1D": reflect.ValueOf(tensor.Cells1D), + "CellsSize": reflect.ValueOf(tensor.CellsSize), + "Clone": reflect.ValueOf(tensor.Clone), + "ColumnMajorStrides": reflect.ValueOf(tensor.ColumnMajorStrides), + "Comma": reflect.ValueOf(tensor.Comma), + "DefaultNumThreads": reflect.ValueOf(tensor.DefaultNumThreads), + "DelimsN": reflect.ValueOf(tensor.DelimsN), + "DelimsValues": reflect.ValueOf(tensor.DelimsValues), + "Descending": reflect.ValueOf(tensor.Descending), + "Detect": reflect.ValueOf(tensor.Detect), + "Ellipsis": reflect.ValueOf(tensor.Ellipsis), + "Flatten": reflect.ValueOf(tensor.Flatten), + "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), + "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "FullAxis": reflect.ValueOf(tensor.FullAxis), + "FuncByName": reflect.ValueOf(tensor.FuncByName), + "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), + "IntToBool": reflect.ValueOf(tensor.IntToBool), + "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), + "MustBeSameShape": reflect.ValueOf(tensor.MustBeSameShape), + "MustBeValues": reflect.ValueOf(tensor.MustBeValues), + "NFirstLen": reflect.ValueOf(tensor.NFirstLen), + "NFirstRows": reflect.ValueOf(tensor.NFirstRows), + "NMinLen": reflect.ValueOf(tensor.NMinLen), + "NewAxis": reflect.ValueOf(tensor.NewAxis), + "NewBool": reflect.ValueOf(tensor.NewBool), + "NewBoolShape": reflect.ValueOf(tensor.NewBoolShape), + "NewByte": reflect.ValueOf(tensor.NewByte), + "NewFloat32": reflect.ValueOf(tensor.NewFloat32), + "NewFloat64": reflect.ValueOf(tensor.NewFloat64), + "NewFloat64FromValues": reflect.ValueOf(tensor.NewFloat64FromValues), + "NewFloat64Full": reflect.ValueOf(tensor.NewFloat64Full), + "NewFloat64Ones": reflect.ValueOf(tensor.NewFloat64Ones), + "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), + "NewFunc": reflect.ValueOf(tensor.NewFunc), + "NewIndexed": reflect.ValueOf(tensor.NewIndexed), + "NewInt": reflect.ValueOf(tensor.NewInt), + "NewInt32": reflect.ValueOf(tensor.NewInt32), + "NewIntFromValues": reflect.ValueOf(tensor.NewIntFromValues), + "NewIntFull": reflect.ValueOf(tensor.NewIntFull), + "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), + "NewMasked": reflect.ValueOf(tensor.NewMasked), + "NewOfType": reflect.ValueOf(tensor.NewOfType), + "NewReshaped": reflect.ValueOf(tensor.NewReshaped), + "NewRowCellsView": reflect.ValueOf(tensor.NewRowCellsView), + "NewRows": reflect.ValueOf(tensor.NewRows), + "NewShape": reflect.ValueOf(tensor.NewShape), + "NewSlice": reflect.ValueOf(tensor.NewSlice), + "NewSliceInts": reflect.ValueOf(tensor.NewSliceInts), + "NewSliced": reflect.ValueOf(tensor.NewSliced), + "NewString": reflect.ValueOf(tensor.NewString), + "NewStringFromValues": reflect.ValueOf(tensor.NewStringFromValues), + "NewStringFull": reflect.ValueOf(tensor.NewStringFull), + "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), + "NewStringShape": reflect.ValueOf(tensor.NewStringShape), + "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), + "OddColumn": reflect.ValueOf(tensor.OddColumn), + "OddRow": reflect.ValueOf(tensor.OddRow), + "OpenCSV": reflect.ValueOf(tensor.OpenCSV), + "Precision": reflect.ValueOf(tensor.Precision), + "Projection2DCoords": reflect.ValueOf(tensor.Projection2DCoords), + "Projection2DIndex": reflect.ValueOf(tensor.Projection2DIndex), + "Projection2DSet": reflect.ValueOf(tensor.Projection2DSet), + "Projection2DSetString": reflect.ValueOf(tensor.Projection2DSetString), + "Projection2DShape": reflect.ValueOf(tensor.Projection2DShape), + "Projection2DString": reflect.ValueOf(tensor.Projection2DString), + "Projection2DValue": reflect.ValueOf(tensor.Projection2DValue), + "Range": reflect.ValueOf(tensor.Range), + "ReadCSV": reflect.ValueOf(tensor.ReadCSV), + "Reshape": reflect.ValueOf(tensor.Reshape), + "Reslice": reflect.ValueOf(tensor.Reslice), + "RowMajorStrides": reflect.ValueOf(tensor.RowMajorStrides), + "SaveCSV": reflect.ValueOf(tensor.SaveCSV), + "SetAllFloat64": reflect.ValueOf(tensor.SetAllFloat64), + "SetAllInt": reflect.ValueOf(tensor.SetAllInt), + "SetAllString": reflect.ValueOf(tensor.SetAllString), + "SetCalcFunc": reflect.ValueOf(tensor.SetCalcFunc), + "SetPrecision": reflect.ValueOf(tensor.SetPrecision), + "SetShape": reflect.ValueOf(tensor.SetShape), + "SetShapeFrom": reflect.ValueOf(tensor.SetShapeFrom), + "SetShapeNames": reflect.ValueOf(tensor.SetShapeNames), + "SetShapeSizesFromTensor": reflect.ValueOf(tensor.SetShapeSizesFromTensor), + "ShapeNames": reflect.ValueOf(tensor.ShapeNames), + "SlicesMagicN": reflect.ValueOf(tensor.SlicesMagicN), + "SlicesMagicValues": reflect.ValueOf(tensor.SlicesMagicValues), + "Space": reflect.ValueOf(tensor.Space), + "Sprint": reflect.ValueOf(tensor.Sprint), + "Squeeze": reflect.ValueOf(tensor.Squeeze), + "StableSort": reflect.ValueOf(tensor.StableSort), + "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), + "Tab": reflect.ValueOf(tensor.Tab), + "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), + "UnstableSort": reflect.ValueOf(tensor.UnstableSort), + "Vectorize": reflect.ValueOf(tensor.Vectorize), + "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), + "VectorizeThreaded": reflect.ValueOf(tensor.VectorizeThreaded), + "WrapIndex1D": reflect.ValueOf(tensor.WrapIndex1D), + "WriteCSV": reflect.ValueOf(tensor.WriteCSV), // type definitions "Bool": reflect.ValueOf((*tensor.Bool)(nil)), From 4602e4bdf4e1f08e81d28e2edb2c8ada7159dd35 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 14:38:48 -0700 Subject: [PATCH 131/311] all tests working with new arg tracking: now knows when to make things tensors or not; much better --- goal/README.md | 4 +- goal/transpile/math.go | 153 ++++++++++++++++++++++++------- goal/transpile/transpile_test.go | 51 ++++++----- tensor/funcs.go | 112 ++++++++++++---------- tensor/funcs_test.go | 9 +- tensor/stats/metric/funcs.go | 48 +++------- tensor/stats/metric/matrix.go | 12 +-- tensor/stats/stats/funcs.go | 8 +- tensor/tmath/ops.go | 22 ++--- 9 files changed, 246 insertions(+), 173 deletions(-) diff --git a/goal/README.md b/goal/README.md index ed86324e3a..7628ae02bc 100644 --- a/goal/README.md +++ b/goal/README.md @@ -213,7 +213,7 @@ These are the rules used to determine whether a line is Go vs. Shell (word = IDE * Line starts with `word {`: Shell * Otherwise: Go -TODO: update aboven +TODO: update above ## Multiple statements per line @@ -249,7 +249,7 @@ The following sections provide a full list of equivalents between the `tensor` G | `a.Shape().Sizes` | same: | `np.shape(a)` or `a.shape` | `size(a)` | "size" of each dimension in a; `shape` returns a 1D `int` tensor | | `a.Shape().Sizes[1]` | same: | `a.shape[1]` | `size(a,2)` | the number of elements of the 2nd dimension of tensor `a` | | `tensor.Reshape(a, 10, 2)` | same except no `a.shape = (10,2)`: | `a.reshape(10, 2)` or `np.reshape(a, 10, 2)` or `a.shape = (10,2)` | `reshape(a,10,2)` | set the shape of `a` to a new shape that has the same total number of values (len or size); No option to change order in Goal: always row major; Goal does _not_ support direct shape assignment version. | -| `tensor.Reshape(a, tensor.AsIntSlice(sh)...)` | same: | `a.reshape(10, sh)` or `np.reshape(a, sh)` | `reshape(a,sh)` | **TODO:** set shape based on list of dimension sizes in tensor `sh` | +| `tensor.Reshape(a, tensor.AsIntSlice(sh)...)` | same: | `a.reshape(10, sh)` or `np.reshape(a, sh)` | `reshape(a,sh)` | set shape based on list of dimension sizes in tensor `sh` | | `tensor.Reshape(a, -1)` or `tensor.As1D(a)` | same: | `a.reshape(-1)` or `np.reshape(a, -1)` | `reshape(a,-1)` | a 1D vector view of `a`; Goal does not support `ravel`, which is nearly identical. | | `tensor.Flatten(a)` | same: | `b = a.flatten()` | `b=a(:)` | returns a 1D copy of a | | `b := tensor.Clone(a)` | `b := copy(a)` or: | `b = a.copy()` | `b=a` | direct assignment `b = a` in Goal or NumPy just makes variable b point to tensor a; `copy` is needed to generate new underlying values (MATLAB always makes a copy) | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 1bb76499e9..0891701745 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -14,6 +14,18 @@ import ( "cogentcore.org/core/tensor" ) +func init() { + tensor.AddFunc("tensor.Reshape", tensor.Reshape) + tensor.AddFunc("tensor.Reslice", tensor.Reslice) + tensor.AddFunc("tensor.NewFloat64", tensor.NewFloat64) + tensor.AddFunc("tensor.NewIntScalar", tensor.NewIntScalar) + tensor.AddFunc("tensor.NewFloat64Scalar", tensor.NewFloat64Scalar) + tensor.AddFunc("tensor.NewStringScalar", tensor.NewStringScalar) + tensor.AddFunc("tensor.NewIntFromValues", tensor.NewIntFromValues) + tensor.AddFunc("tensor.NewFloat64FromValues", tensor.NewFloat64FromValues) + tensor.AddFunc("tensor.NewStringFromValues", tensor.NewStringFromValues) +} + func MathParse(toks Tokens, code string, fullLine bool) Tokens { nt := len(toks) if nt == 0 { @@ -60,8 +72,8 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { type funcInfo struct { tensor.Func - // true if this function takes tensor args - tensorArgs bool + // current arg index we are processing + curArg int } // mathParse has the parsing state @@ -76,16 +88,69 @@ type mathParse struct { funcs stack.Stack[*funcInfo] } -// startFunc is called when starting a new function -- sets context -func (mp *mathParse) startFunc(name string, tensorArgs bool) *funcInfo { - fn := &funcInfo{} - fn.Name = name - fn.tensorArgs = tensorArgs - mp.funcs.Push(fn) +// returns the current argument for current function +func (mp *mathParse) curArg() *tensor.Arg { + cfun := mp.funcs.Peek() + if cfun == nil { + return nil + } + if cfun.curArg < len(cfun.Args) { + return cfun.Args[cfun.curArg] + } + return nil +} + +func (mp *mathParse) nextArg() { + cfun := mp.funcs.Peek() + if cfun == nil || len(cfun.Args) == 0 { + // fmt.Println("next arg no fun or no args") + return + } + n := len(cfun.Args) + if cfun.curArg == n-1 { + carg := cfun.Args[n-1] + if !carg.IsVariadic { + fmt.Println("math transpile: args exceed registered function number:", cfun) + } + return + } + cfun.curArg++ +} + +func (mp *mathParse) curArgIsTensor() bool { + carg := mp.curArg() + if carg == nil { + return false + } + return carg.IsTensor +} + +func (mp *mathParse) curArgIsInts() bool { + carg := mp.curArg() + if carg == nil { + return false + } + return carg.IsInt && carg.IsVariadic +} + +// startFunc is called when starting a new function. +// empty is "dummy" assign case using Inc +func (mp *mathParse) startFunc(name string) *funcInfo { + fi := &funcInfo{} + sname := name + if name == "" { + sname = "tmath.Inc" + } + if tf, err := tensor.FuncByName(sname); err == nil { + fi.Func = *tf + } else { + fi.Name = name // not clear what point is + } + mp.funcs.Push(fi) if name != "" { mp.out.Add(token.IDENT, name) } - return fn + return fi } func (mp *mathParse) endFunc() { @@ -267,6 +332,28 @@ func (mp *mathParse) exprList(ex []ast.Expr) { } } +func (mp *mathParse) argsList(ex []ast.Expr) { + n := len(ex) + if n == 0 { + return + } + if n == 1 { + mp.expr(ex[0]) + return + } + for i := range n { + // cfun := mp.funcs.Peek() + // if i != cfun.curArg { + // fmt.Println(cfun, "arg should be:", i, "is:", cfun.curArg) + // } + mp.expr(ex[i]) + if i < n-1 { + mp.nextArg() + mp.addToken(token.COMMA) + } + } +} + func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn := "" switch ex.Op { @@ -279,7 +366,7 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { case token.QUO: fn = "Div" } - mp.startFunc("tmath."+fn, true) // yes tensor args + mp.startFunc("tmath." + fn) mp.out.Add(token.LPAREN) mp.expr(ex.X) mp.out.Add(token.COMMA) @@ -297,7 +384,7 @@ func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { func (mp *mathParse) defineStmt(as *ast.AssignStmt) { mp.exprList(as.Lhs) mp.addToken(as.Tok) - mp.startFunc("", true) // just to trigger tensor args + mp.startFunc("") // dummy single arg tensor function mp.exprList(as.Rhs) mp.endFunc() } @@ -306,7 +393,7 @@ func (mp *mathParse) assignStmt(as *ast.AssignStmt) { if _, ok := as.Lhs[0].(*ast.Ident); ok { mp.exprList(as.Lhs) mp.addToken(as.Tok) - mp.startFunc("", true) // just to trigger tensor args + mp.startFunc("") mp.exprList(as.Rhs) mp.endFunc() return @@ -324,7 +411,7 @@ func (mp *mathParse) assignStmt(as *ast.AssignStmt) { case token.QUO_ASSIGN: fn = "DivAssign" } - mp.startFunc("tmath."+fn, true) // yes tensor args + mp.startFunc("tmath." + fn) mp.out.Add(token.LPAREN) mp.exprList(as.Lhs) mp.out.Add(token.COMMA) @@ -335,8 +422,7 @@ func (mp *mathParse) assignStmt(as *ast.AssignStmt) { } func (mp *mathParse) basicLit(lit *ast.BasicLit) { - cfun := mp.funcs.Peek() - if cfun != nil && cfun.tensorArgs { + if mp.curArgIsTensor() { mp.tensorLit(lit) return } @@ -397,7 +483,7 @@ func (fw *funWrap) wrapFunc(mp *mathParse) bool { wrapFun = "tensor.NewStringFromValues" ellip = true } - mp.startFunc(wrapFun, false) + mp.startFunc(wrapFun) mp.out.Add(token.LPAREN) return ellip } @@ -435,9 +521,10 @@ func (mp *mathParse) indexExpr(il *ast.IndexExpr) { func (mp *mathParse) basicSlicingExpr(il *ast.IndexExpr) { iil := il.Index.(*ast.IndexListExpr) - mp.startFunc("tensor.Reslice", false) + mp.startFunc("tensor.Reslice") mp.out.Add(token.LPAREN) mp.expr(il.X) + mp.nextArg() mp.addToken(token.COMMA) // use the [ -- can't use ( to preserve X mp.exprList(iil.Indices) mp.addToken(token.RPAREN) // replaces ] @@ -501,12 +588,13 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { typ = "string" fun = "String" } - // cfun := mp.funcs.Peek() - // if cfun != nil && !cfun.tensorArgs { // we need int values - // mp.exprList(il.Indices) - // return - // } - mp.startFunc("tensor.New"+fun+"FromValues", false) + if mp.curArgIsInts() { + mp.idx++ // opening brace we're not using + mp.exprList(il.Indices) + mp.idx++ // closing brace we're not using + return + } + mp.startFunc("tensor.New" + fun + "FromValues") mp.out.Add(token.LPAREN) mp.out.Add(token.IDENT, "[]"+typ) mp.addToken(token.LBRACE) @@ -553,7 +641,7 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { default: mp.expr(ex.Fun) } - mp.exprList(ex.Args) + mp.argsList(ex.Args) // todo: ellipsis mp.addToken(token.RPAREN) mp.endFunc() @@ -575,19 +663,20 @@ func (mp *mathParse) callPropFun(ex *ast.CallExpr, fw funWrap) { // this calls global function through selector like: a.reshape() func (mp *mathParse) callPropSelFun(ex *ast.CallExpr, obj string, fw funWrap) { - mp.startFunc(fw.fun, false) + mp.startFunc(fw.fun) mp.addToken(token.LPAREN) // use the ( mp.out.Add(token.IDENT, obj) + mp.nextArg() // did first mp.idx += 2 mp.addToken(token.COMMA) - mp.exprList(ex.Args) + mp.argsList(ex.Args) mp.addToken(token.RPAREN) mp.endFunc() } func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { if fw, ok := numpyFuncs[funName]; ok { - mp.startFunc(fw.fun, false) + mp.startFunc(fw.fun) mp.addToken(token.LPAREN) // use the ( mp.idx++ // paren too return @@ -607,12 +696,12 @@ func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { } } if err != nil { // not a registered tensor function - mp.startFunc(funName, false) + mp.startFunc(funName) mp.addToken(token.LPAREN) // use the ( mp.idx++ return } - mp.startFunc(funName, true) // tensors + mp.startFunc(funName) mp.idx += 1 if pkgName != "" { mp.idx += 2 // . and selector @@ -624,9 +713,7 @@ func (mp *mathParse) ident(id *ast.Ident) { if id == nil { return } - /* TODO: this requires tracking of each arg to determine needed type - cfun := mp.funcs.Peek() - if cfun != nil && !cfun.tensorArgs { // we need the numbers from it, usually ints + if mp.curArgIsInts() { mp.out.Add(token.IDENT, "tensor.AsIntSlice") mp.out.Add(token.LPAREN) mp.addCur() @@ -635,6 +722,4 @@ func (mp *mathParse) ident(id *ast.Ident) { } else { mp.addCur() } - */ - mp.addCur() } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 21dd2c34a2..ace411c3cd 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -203,31 +203,32 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - // {"# x := 1", `x := tensor.NewIntScalar(1)`}, - // {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, - // {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, - // {"# a = x + y", `a = tmath.Add(x, y)`}, - // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - // {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, - // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - // {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, - // // {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, // TODO: not yet - // {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - // {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - // {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - // // {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, // TODO: not yet - // {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, - // {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, - // {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - // {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, - // {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - // {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, - // {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, - // {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - // {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - // {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - // {"# c := cos(a)", `c := tmath.Cos(a)`}, + {"# x := 1", `x := tensor.NewIntScalar(1)`}, + {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tmath.Add(x, y)`}, + {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, + {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, + {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, + {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, + {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, + {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, + {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# c := cos(a)", `c := tmath.Cos(a)`}, {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, diff --git a/tensor/funcs.go b/tensor/funcs.go index e0121d3081..04fa517077 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -6,6 +6,7 @@ package tensor import ( "fmt" + "reflect" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" @@ -24,11 +25,24 @@ type Func struct { // args, with an optional any first arg. Fun any - // In is number of input tensor args - In int + // Args has parsed information about the function args, for Goal. + Args []*Arg +} + +// Arg has key information that Goal needs about each arg, for converting +// expressions into the appropriate type. +type Arg struct { + // Type has full reflection type info. + Type reflect.Type + + // IsTensor is true if it satisfies the Tensor interface. + IsTensor bool - // Out is number of output tensor args - Out int + // IsInt is true if Kind = Int, for shape, slice etc params. + IsInt bool + + // IsVariadic is true if this is the last arg and has ...; type will be an array. + IsVariadic bool } // NewFunc creates a new Func desciption of the given @@ -44,11 +58,54 @@ type Func struct { // determined) are classified as input arguments. func NewFunc(name string, fun any) (*Func, error) { fn := &Func{Name: name, Fun: fun} - // fn.In = 1 - out - // todo: get signature + fn.GetArgs() return fn, nil } +// GetArgs gets key info about each arg, for use by Goal transpiler. +func (fn *Func) GetArgs() { + ft := reflect.TypeOf(fn.Fun) + n := ft.NumIn() + if n == 0 { + return + } + fn.Args = make([]*Arg, n) + tsrt := reflect.TypeFor[Tensor]() + for i := range n { + at := ft.In(i) + ag := &Arg{Type: at} + if ft.IsVariadic() && i == n-1 { + ag.IsVariadic = true + } + if at.Kind() == reflect.Int || (at.Kind() == reflect.Slice && at.Elem().Kind() == reflect.Int) { + ag.IsInt = true + } else if at.Implements(tsrt) { + ag.IsTensor = true + } + fn.Args[i] = ag + } +} + +func (fn *Func) String() string { + s := fn.Name + "(" + na := len(fn.Args) + for i, a := range fn.Args { + if a.IsVariadic { + s += "..." + } + ts := a.Type.String() + if ts == "interface {}" { + ts = "any" + } + s += ts + if i < na-1 { + s += ", " + } + } + s += ")" + return s +} + // Funcs is the global tensor named function registry. // All functions must have a signature like this: // func([opt any,] a, b, out tensor.Tensor) error @@ -62,7 +119,7 @@ var Funcs map[string]*Func // AddFunc adds given named function to the global tensor named function // registry, which is used by Goal to call functions by name. -// See [NewFunc] for more information. +// See [NewFunc] for more informa.tion. func AddFunc(name string, fun any) error { if Funcs == nil { Funcs = make(map[string]*Func) @@ -90,47 +147,6 @@ func FuncByName(name string) (*Func, error) { return fn, nil } -// todo: definitely switch over to reflection here: - -// ArgCount returns the number of tensor arguments the function takes, -// using a type switch. -func (fn *Func) ArgCount() int { - nargs := -1 - switch fn.Fun.(type) { - case func(a Tensor) error: - nargs = 1 - case func(a, b Tensor) error: - nargs = 2 - case func(a, b, c Tensor) error: - nargs = 3 - case func(a, b, c, d Tensor) error: - nargs = 4 - case func(a, b, c, d, e Tensor) error: - nargs = 5 - // any cases: - case func(first any, a Tensor) error: - nargs = 1 - case func(first any, a, b Tensor) error: - nargs = 2 - case func(first any, a, b, c Tensor) error: - nargs = 3 - case func(first any, a, b, c, d Tensor) error: - nargs = 4 - case func(first any, a, b, c, d, e Tensor) error: - nargs = 5 - } - return nargs -} - -// ArgCheck returns an error if the number of args in list does not -// match the number required as specified. -func (fn *Func) ArgCheck(n int, tsr ...Tensor) error { - if len(tsr) != n { - return fmt.Errorf("tensor.Call: args passed to %q: %d does not match required: %d", fn.Name, len(tsr), n) - } - return nil -} - // These generic functions provide a one liner for wrapping functions // that take an output Tensor as the last argument, which is important // for memory re-use of the output in performance-critical cases. diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index b77892e7fc..e2465ab132 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -32,11 +32,16 @@ func TestFuncs(t *testing.T) { oned := NewNumberFromValues(vals...) oneout := oned.Clone() - _, err = FuncByName("Abs") + fn, err := FuncByName("Abs") assert.NoError(t, err) - absout(oned, oneout) + // fmt.Println(fn.Args[0], fn.Args[1]) + assert.Equal(t, true, fn.Args[0].IsTensor) + assert.Equal(t, true, fn.Args[1].IsTensor) + assert.Equal(t, false, fn.Args[0].IsInt) + assert.Equal(t, false, fn.Args[1].IsInt) + absout(oned, oneout) assert.Equal(t, 1.507556722888818, oneout.Float1D(0)) } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index 62da1e7f94..d0158cfa0a 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -110,9 +110,7 @@ func EuclideanOut(a, b tensor.Tensor, out tensor.Values) error { // Euclidean computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func Euclidean(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(EuclideanOut(a, b, out)) - return out + return tensor.CallOut2(EuclideanOut, a, b) } // AbsOut computes the sum of the absolute value of differences between the @@ -133,9 +131,7 @@ func AbsOut(a, b tensor.Tensor, out tensor.Values) error { // Abs computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func Abs(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(AbsOut(a, b, out)) - return out + return tensor.CallOut2(AbsOut, a, b) } // HammingOut computes the sum of 1s for every element that is different, @@ -159,9 +155,7 @@ func HammingOut(a, b tensor.Tensor, out tensor.Values) error { // Hamming computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func Hamming(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(HammingOut(a, b, out)) - return out + return tensor.CallOut2(HammingOut, a, b) } // SumSquaresBinTolScaleOut64 computes the sum of squares differences between tensor values, @@ -210,9 +204,7 @@ func EuclideanBinTolOut(a, b tensor.Tensor, out tensor.Values) error { // EuclideanBinTol computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func EuclideanBinTol(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(EuclideanBinTolOut(a, b, out)) - return out + return tensor.CallOut2(EuclideanBinTolOut, a, b) } // SumSquaresBinTolOut computes the sum of squares differences between tensor values, @@ -241,9 +233,7 @@ func SumSquaresBinTolOut(a, b tensor.Tensor, out tensor.Values) error { // SumSquaresBinTol computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func SumSquaresBinTol(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(SumSquaresBinTolOut(a, b, out)) - return out + return tensor.CallOut2(SumSquaresBinTolOut, a, b) } // CrossEntropyOut is a standard measure of the difference between two @@ -276,9 +266,7 @@ func CrossEntropyOut(a, b tensor.Tensor, out tensor.Values) error { // CrossEntropy computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func CrossEntropy(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(CrossEntropyOut(a, b, out)) - return out + return tensor.CallOut2(CrossEntropyOut, a, b) } // InnerProductOut computes the sum of the co-products of the two on-NaN tensor values. @@ -298,9 +286,7 @@ func InnerProductOut(a, b tensor.Tensor, out tensor.Values) error { // InnerProduct computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func InnerProduct(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(InnerProductOut(a, b, out)) - return out + return tensor.CallOut2(InnerProductOut, a, b) } // CovarianceOut computes the co-variance between two vectors, @@ -335,9 +321,7 @@ func CovarianceOut(a, b tensor.Tensor, out tensor.Values) error { // Covariance computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func Covariance(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(CovarianceOut(a, b, out)) - return out + return tensor.CallOut2(CovarianceOut, a, b) } // CorrelationOut64 computes the correlation between two vectors, @@ -397,9 +381,7 @@ func CorrelationOut(a, b tensor.Tensor, out tensor.Values) error { // Correlation computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func Correlation(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(CorrelationOut(a, b, out)) - return out + return tensor.CallOut2(CorrelationOut, a, b) } // InvCorrelationOut computes 1 minus the correlation between two vectors, @@ -426,9 +408,7 @@ func InvCorrelationOut(a, b tensor.Tensor, out tensor.Values) error { // InvCorrelation computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func InvCorrelation(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(InvCorrelationOut(a, b, out)) - return out + return tensor.CallOut2(InvCorrelationOut, a, b) } // CosineOut64 computes the high-dimensional angle between two vectors, @@ -470,9 +450,7 @@ func CosineOut(a, b tensor.Tensor, out tensor.Values) error { // Cosine computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func Cosine(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(CosineOut(a, b, out)) - return out + return tensor.CallOut2(CosineOut, a, b) } // InvCosineOut computes 1 minus the cosine between two vectors, @@ -496,7 +474,5 @@ func InvCosineOut(a, b tensor.Tensor, out tensor.Values) error { // InvCosine computes the sum of squares differences between tensor values, // See [MetricFunc] for general information. func InvCosine(a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(InvCosineOut(a, b, out)) - return out + return tensor.CallOut2(InvCosineOut, a, b) } diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 55811c619f..3cec07ab02 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -74,9 +74,7 @@ func MatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { // and only the lower triangular part is computed, with results copied // to the upper triangular region, for maximum efficiency. func Matrix(fun any, in tensor.Tensor) tensor.Values { - out := tensor.NewOfType(in.DataType()) - errors.Log(MatrixOut(fun, in, out)) - return out + return tensor.CallOut1Gen1(MatrixOut, fun, in) } // CrossMatrixOut computes the distance / similarity matrix between @@ -127,9 +125,7 @@ func CrossMatrixOut(fun any, a, b tensor.Tensor, out tensor.Values) error { // The rows of the output matrix are the rows of the first input tensor, // and the columns of the output are the rows of the second input tensor. func CrossMatrix(fun any, a, b tensor.Tensor) tensor.Values { - out := tensor.NewOfType(a.DataType()) - errors.Log(CrossMatrixOut(fun, a, b, out)) - return out + return tensor.CallOut2Gen1(CrossMatrixOut, fun, a, b) } // CovarianceMatrixOut generates the cells x cells square covariance matrix @@ -200,9 +196,7 @@ func CovarianceMatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { // Correlation if they are on very different scales, because it effectively rescales). // The resulting matrix can be used as the input to PCA or SVD eigenvalue decomposition. func CovarianceMatrix(fun any, in tensor.Tensor) tensor.Values { - out := tensor.NewOfType(in.DataType()) - errors.Log(CovarianceMatrixOut(fun, in, out)) - return out + return tensor.CallOut1Gen1(CovarianceMatrixOut, fun, in) } // PCA performs the eigen decomposition of the given CovarianceMatrix, diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index b821c909bc..1bc5134d2d 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -380,9 +380,7 @@ func VarPopOut(in tensor.Tensor, out tensor.Values) error { // Sqrt of variance from [VarPopFunc]. See also [StdFunc]. // See [StatsFunc] for general information. func StdPop(in tensor.Tensor) tensor.Values { - out := tensor.NewOfType(in.DataType()) - errors.Log(StdPopOut(in, out)) - return out + return tensor.CallOut1(StdPopOut, in) } // StdPopOut computes the population standard deviation of tensor values. @@ -404,9 +402,7 @@ func StdPopOut(in tensor.Tensor, out tensor.Values) error { // Standard deviation [StdPopFunc] / sqrt(n). See also [SemFunc]. // See [StatsFunc] for general information. func SemPop(in tensor.Tensor) tensor.Values { - out := tensor.NewOfType(in.DataType()) - errors.Log(SemPopOut(in, out)) - return out + return tensor.CallOut1(SemPopOut, in) } // SemPopOut computes the population standard error of the mean of tensor values. diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 7b29f06ebf..93c40b43f0 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -9,19 +9,19 @@ import ( ) func init() { - tensor.AddFunc("Assign", Assign) - tensor.AddFunc("AddAssign", AddAssign) - tensor.AddFunc("SubAssign", SubAssign) - tensor.AddFunc("MulAssign", MulAssign) - tensor.AddFunc("DivAssign", DivAssign) + tensor.AddFunc("tmath.Assign", Assign) + tensor.AddFunc("tmath.AddAssign", AddAssign) + tensor.AddFunc("tmath.SubAssign", SubAssign) + tensor.AddFunc("tmath.MulAssign", MulAssign) + tensor.AddFunc("tmath.DivAssign", DivAssign) - tensor.AddFunc("Inc", Inc) - tensor.AddFunc("Dec", Dec) + tensor.AddFunc("tmath.Inc", Inc) + tensor.AddFunc("tmath.Dec", Dec) - tensor.AddFunc("Add", Add) - tensor.AddFunc("Sub", Sub) - tensor.AddFunc("Mul", Mul) - tensor.AddFunc("Div", Div) + tensor.AddFunc("tmath.Add", Add) + tensor.AddFunc("tmath.Sub", Sub) + tensor.AddFunc("tmath.Mul", Mul) + tensor.AddFunc("tmath.Div", Div) } // Assign assigns values from b into a. From 30288f0242f046feb67da75cbf53b52e424eafaf Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 15:52:32 -0700 Subject: [PATCH 132/311] use helper funcs in align for all ops -- key for more general computation too presumably --- goal/transpile/math.go | 6 + goal/transpile/transpile_test.go | 63 +++++----- tensor/align.go | 157 ++++++++++++++++++++++++ tensor/funcs.go | 2 + tensor/tmath/bool.go | 96 +++++++++++++++ tensor/tmath/bool_test.go | 52 ++++++++ tensor/tmath/math_test.go | 10 +- tensor/tmath/ops.go | 204 +++---------------------------- tensor/tmath/ops_test.go | 19 --- 9 files changed, 366 insertions(+), 243 deletions(-) create mode 100644 tensor/tmath/bool.go create mode 100644 tensor/tmath/bool_test.go diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 0891701745..c4061915f4 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -365,6 +365,12 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn = "Mul" case token.QUO: fn = "Div" + case token.LSS: + fn = "Less" + case token.GTR: + fn = "Greater" + default: + fmt.Println("binary token:", ex.Op) } mp.startFunc("tmath." + fn) mp.out.Add(token.LPAREN) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index ace411c3cd..d84614353a 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -203,37 +203,38 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, - {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, - {"# a = x + y", `a = tmath.Add(x, y)`}, - {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, - {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, - {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, - {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, - {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, - {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, - {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, - {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, - {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, - {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - {"# c := cos(a)", `c := tmath.Cos(a)`}, - {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, - {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, - {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, - {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, - {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, + // {"# x := 1", `x := tensor.NewIntScalar(1)`}, + // {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, + // {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, + // {"# a = x + y", `a = tmath.Add(x, y)`}, + // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + // {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, + // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + // {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, + // {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, + // {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + // {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + // {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + // {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, + // {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, + // {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + // {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, + // {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + // {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, + // {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + // {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + // {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + // {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + // {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + // {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + // {"# c := cos(a)", `c := tmath.Cos(a)`}, + // {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, + // {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, + // {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, + // {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, + // {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, + {"# b := a > 5", `b := tmath.Greater(a, tensor.NewIntScalar(5))`}, } st := NewState() diff --git a/tensor/align.go b/tensor/align.go index 12bf4c1e72..04d9752962 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -7,8 +7,19 @@ package tensor import ( "fmt" "slices" + + "cogentcore.org/core/base/errors" ) +func init() { + AddFunc("tensor.FloatAssignFunc", FloatAssignFunc) + AddFunc("tensor.StringAssignFunc", StringAssignFunc) + AddFunc("tensor.FloatBinaryFunc", FloatBinaryFunc) + AddFunc("tensor.StringBinaryFunc", StringBinaryFunc) + AddFunc("tensor.BoolStringsFunc", BoolStringsFunc) + AddFunc("tensor.BoolFloatsFunc", BoolFloatsFunc) +} + // AlignShapes aligns the shapes of two tensors, a and b for a binary // computation producing an output, returning the effective aligned shapes // for a, b, and the output, all with the same number of dimensions. @@ -108,3 +119,149 @@ func AlignForAssign(a, b Tensor) (as, bs *Shape, err error) { bs = NewShape(bsizes...) return } + +// FloatAssignFunc sets a to a binary function of a and b float64 values. +func FloatAssignFunc(fun func(a, b float64) float64, a, b Tensor) error { + as, bs, err := AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + VectorizeThreaded(1, func(tsr ...Tensor) int { + return alen + }, + func(idx int, tsr ...Tensor) { + ai := as.IndexFrom1D(idx) + bi := WrapIndex1D(bs, ai...) + tsr[0].SetFloat1D(fun(tsr[0].Float1D(idx), tsr[1].Float1D(bi)), idx) + }, a, b) + return nil +} + +// StringAssignFunc sets a to a binary function of a and b string values. +func StringAssignFunc(fun func(a, b string) string, a, b Tensor) error { + as, bs, err := AlignForAssign(a, b) + if err != nil { + return err + } + alen := as.Len() + VectorizeThreaded(1, func(tsr ...Tensor) int { + return alen + }, + func(idx int, tsr ...Tensor) { + ai := as.IndexFrom1D(idx) + bi := WrapIndex1D(bs, ai...) + tsr[0].SetString1D(fun(tsr[0].String1D(idx), tsr[1].String1D(bi)), idx) + }, a, b) + return nil +} + +// FloatBinaryFunc sets output to a binary function of a, b float64 values. +func FloatBinaryFunc(fun func(a, b float64) float64, a, b Tensor) Tensor { + return CallOut2Gen1(FloatBinaryFuncOut, fun, a, b) +} + +// FloatBinaryFuncOut sets output to a binary function of a, b float64 values. +func FloatBinaryFuncOut(fun func(a, b float64) float64, a, b Tensor, out Values) error { + as, bs, os, err := AlignShapes(a, b) + if err != nil { + return err + } + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + VectorizeThreaded(1, func(tsr ...Tensor) int { + return olen + }, + func(idx int, tsr ...Tensor) { + oi := os.IndexFrom1D(idx) + ai := WrapIndex1D(as, oi...) + bi := WrapIndex1D(bs, oi...) + out.SetFloat1D(fun(tsr[0].Float1D(ai), tsr[1].Float1D(bi)), idx) + }, a, b, out) + return nil +} + +// StringBinaryFunc sets output to a binary function of a, b string values. +func StringBinaryFunc(fun func(a, b string) string, a, b Tensor) Tensor { + return CallOut2Gen1(StringBinaryFuncOut, fun, a, b) +} + +// StringBinaryFuncOut sets output to a binary function of a, b string values. +func StringBinaryFuncOut(fun func(a, b string) string, a, b Tensor, out Values) error { + as, bs, os, err := AlignShapes(a, b) + if err != nil { + return err + } + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + VectorizeThreaded(1, func(tsr ...Tensor) int { + return olen + }, + func(idx int, tsr ...Tensor) { + oi := os.IndexFrom1D(idx) + ai := WrapIndex1D(as, oi...) + bi := WrapIndex1D(bs, oi...) + out.SetString1D(fun(tsr[0].String1D(ai), tsr[1].String1D(bi)), idx) + }, a, b, out) + return nil +} + +////////////////////////// Bool + +// BoolStringsFunc sets boolean output value based on a function involving +// string values from the two tensors. +func BoolStringsFunc(fun func(a, b string) bool, a, b Tensor) *Bool { + out := NewBool() + errors.Log(BoolStringsFuncOut(fun, a, b, out)) + return out +} + +// BoolStringsFuncOut sets boolean output value based on a function involving +// string values from the two tensors. +func BoolStringsFuncOut(fun func(a, b string) bool, a, b Tensor, out *Bool) error { + as, bs, os, err := AlignShapes(a, b) + if err != nil { + return err + } + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + VectorizeThreaded(5, func(tsr ...Tensor) int { + return olen + }, + func(idx int, tsr ...Tensor) { + oi := os.IndexFrom1D(idx) + ai := WrapIndex1D(as, oi...) + bi := WrapIndex1D(bs, oi...) + out.SetBool1D(fun(tsr[0].String1D(ai), tsr[1].String1D(bi)), idx) + }, a, b, out) + return nil +} + +// BoolFloatsFunc sets boolean output value based on a function involving +// float64 values from the two tensors. +func BoolFloatsFunc(fun func(a, b float64) bool, a, b Tensor) *Bool { + out := NewBool() + errors.Log(BoolFloatsFuncOut(fun, a, b, out)) + return out +} + +// BoolFloatsFuncOut sets boolean output value based on a function involving +// float64 values from the two tensors. +func BoolFloatsFuncOut(fun func(a, b float64) bool, a, b Tensor, out *Bool) error { + as, bs, os, err := AlignShapes(a, b) + if err != nil { + return err + } + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + VectorizeThreaded(5, func(tsr ...Tensor) int { + return olen + }, + func(idx int, tsr ...Tensor) { + oi := os.IndexFrom1D(idx) + ai := WrapIndex1D(as, oi...) + bi := WrapIndex1D(bs, oi...) + out.SetBool1D(fun(tsr[0].Float1D(ai), tsr[1].Float1D(bi)), idx) + }, a, b, out) + return nil +} diff --git a/tensor/funcs.go b/tensor/funcs.go index 04fa517077..94de1f54ca 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -203,6 +203,8 @@ func CallOut2Gen2[T any, S any](fun func(g T, h S, a, b Tensor, out Values) erro return out } +//////////////////// Metadata + // SetCalcFunc sets a function to calculate updated value for given tensor, // storing the function pointer in the Metadata "CalcFunc" key for the tensor. // Can be called by [Calc] function. diff --git a/tensor/tmath/bool.go b/tensor/tmath/bool.go new file mode 100644 index 0000000000..facd1d3be9 --- /dev/null +++ b/tensor/tmath/bool.go @@ -0,0 +1,96 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "cogentcore.org/core/tensor" +) + +func init() { + tensor.AddFunc("tmath.Equal", Equal) + tensor.AddFunc("tmath.Less", Less) + tensor.AddFunc("tmath.Greater", Greater) + tensor.AddFunc("tmath.NotEqual", NotEqual) + tensor.AddFunc("tmath.LessEqual", LessEqual) + tensor.AddFunc("tmath.GreaterEqual", GreaterEqual) +} + +// Equal stores in the output the bool value a == b. +func Equal(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(EqualOut, a, b) +} + +// EqualOut stores in the output the bool value a == b. +func EqualOut(a, b tensor.Tensor, out *tensor.Bool) error { + if a.IsString() { + return tensor.BoolStringsFuncOut(func(a, b string) bool { return a == b }, a, b, out) + } + return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a == b }, a, b, out) +} + +// Less stores in the output the bool value a < b. +func Less(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(LessOut, a, b) +} + +// LessOut stores in the output the bool value a < b. +func LessOut(a, b tensor.Tensor, out *tensor.Bool) error { + if a.IsString() { + return tensor.BoolStringsFuncOut(func(a, b string) bool { return a < b }, a, b, out) + } + return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a < b }, a, b, out) +} + +// Greater stores in the output the bool value a > b. +func Greater(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(GreaterOut, a, b) +} + +// GreaterOut stores in the output the bool value a > b. +func GreaterOut(a, b tensor.Tensor, out *tensor.Bool) error { + if a.IsString() { + return tensor.BoolStringsFuncOut(func(a, b string) bool { return a > b }, a, b, out) + } + return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a > b }, a, b, out) +} + +// NotEqual stores in the output the bool value a != b. +func NotEqual(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(NotEqualOut, a, b) +} + +// NotEqualOut stores in the output the bool value a != b. +func NotEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { + if a.IsString() { + return tensor.BoolStringsFuncOut(func(a, b string) bool { return a != b }, a, b, out) + } + return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a != b }, a, b, out) +} + +// LessEqual stores in the output the bool value a <= b. +func LessEqual(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(LessEqualOut, a, b) +} + +// LessEqualOut stores in the output the bool value a <= b. +func LessEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { + if a.IsString() { + return tensor.BoolStringsFuncOut(func(a, b string) bool { return a <= b }, a, b, out) + } + return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a <= b }, a, b, out) +} + +// GreaterEqual stores in the output the bool value a >= b. +func GreaterEqual(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(GreaterEqualOut, a, b) +} + +// GreaterEqualOut stores in the output the bool value a >= b. +func GreaterEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { + if a.IsString() { + return tensor.BoolStringsFuncOut(func(a, b string) bool { return a >= b }, a, b, out) + } + return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a >= b }, a, b, out) +} diff --git a/tensor/tmath/bool_test.go b/tensor/tmath/bool_test.go new file mode 100644 index 0000000000..b0ec0a77cb --- /dev/null +++ b/tensor/tmath/bool_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tmath + +import ( + "testing" + + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" +) + +func TestBoolOps(t *testing.T) { + ar := tensor.NewSliceInts(12) + // fmt.Println(v) + bo := tensor.NewBool() + sc := tensor.NewIntScalar(6) + + EqualOut(ar, sc, bo) + for i, v := range ar.Values { + assert.Equal(t, v == 6, bo.Bool1D(i)) + } + + LessOut(ar, sc, bo) + for i, v := range ar.Values { + assert.Equal(t, v < 6, bo.Bool1D(i)) + } + + GreaterOut(ar, sc, bo) + // fmt.Println(bo) + for i, v := range ar.Values { + assert.Equal(t, v > 6, bo.Bool1D(i)) + } + + NotEqualOut(ar, sc, bo) + for i, v := range ar.Values { + assert.Equal(t, v != 6, bo.Bool1D(i)) + } + + LessEqualOut(ar, sc, bo) + for i, v := range ar.Values { + assert.Equal(t, v <= 6, bo.Bool1D(i)) + } + + GreaterEqualOut(ar, sc, bo) + // fmt.Println(bo) + for i, v := range ar.Values { + assert.Equal(t, v >= 6, bo.Bool1D(i)) + } + +} diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index be58ac0214..3aee561808 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -15,8 +15,8 @@ import ( type onef func(x float64) float64 type tonef func(in tensor.Tensor, out tensor.Values) error -// Equal does equal testing taking into account NaN -func Equal(t *testing.T, trg, val float64) { +// testEqual does equal testing taking into account NaN +func testEqual(t *testing.T, trg, val float64) { if math.IsNaN(trg) { if !math.IsNaN(val) { t.Error("target is NaN but actual is not") @@ -53,9 +53,9 @@ func TestMath(t *testing.T) { tf(oned, oneout) tf(cell2d, cellout) - Equal(t, fun(scalar.Float1D(0)), scout.Float1D(0)) + testEqual(t, fun(scalar.Float1D(0)), scout.Float1D(0)) for i, v := range vals { - Equal(t, fun(v), oneout.Float1D(i)) + testEqual(t, fun(v), oneout.Float1D(i)) } lv := len(vals) for r := range 5 { @@ -63,7 +63,7 @@ func TestMath(t *testing.T) { si := lv * r for c, v := range vals { ov := tensor.AsFloat32Tensor(cellout).Values[si+c] - Equal(t, fun(v), float64(ov)) + testEqual(t, fun(v), float64(ov)) } } } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 93c40b43f0..8d688fd059 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -22,104 +22,41 @@ func init() { tensor.AddFunc("tmath.Sub", Sub) tensor.AddFunc("tmath.Mul", Mul) tensor.AddFunc("tmath.Div", Div) + } // Assign assigns values from b into a. func Assign(a, b tensor.Tensor) error { - as, bs, err := tensor.AlignForAssign(a, b) - if err != nil { - return err - } - alen := as.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, - func(idx int, tsr ...tensor.Tensor) { - ai := as.IndexFrom1D(idx) - bi := tensor.WrapIndex1D(bs, ai...) - tsr[0].SetFloat1D(tsr[1].Float1D(bi), idx) - }, a, b) - return nil + return tensor.FloatAssignFunc(func(a, b float64) float64 { return b }, a, b) } // AddAssign does += add assign values from b into a. func AddAssign(a, b tensor.Tensor) error { - as, bs, err := tensor.AlignForAssign(a, b) - if err != nil { - return err + if a.IsString() { + return tensor.StringAssignFunc(func(a, b string) string { return a + b }, a, b) } - alen := as.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, - func(idx int, tsr ...tensor.Tensor) { - ai := as.IndexFrom1D(idx) - bi := tensor.WrapIndex1D(bs, ai...) - tsr[0].SetFloat1D(tsr[0].Float1D(idx)+tsr[1].Float1D(bi), idx) - }, a, b) - return nil + return tensor.FloatAssignFunc(func(a, b float64) float64 { return a + b }, a, b) } // SubAssign does -= sub assign values from b into a. func SubAssign(a, b tensor.Tensor) error { - as, bs, err := tensor.AlignForAssign(a, b) - if err != nil { - return err - } - alen := as.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, - func(idx int, tsr ...tensor.Tensor) { - ai := as.IndexFrom1D(idx) - bi := tensor.WrapIndex1D(bs, ai...) - tsr[0].SetFloat1D(tsr[0].Float1D(idx)-tsr[1].Float1D(bi), idx) - }, a, b) - return nil + return tensor.FloatAssignFunc(func(a, b float64) float64 { return a - b }, a, b) } // MulAssign does *= mul assign values from b into a. func MulAssign(a, b tensor.Tensor) error { - as, bs, err := tensor.AlignForAssign(a, b) - if err != nil { - return err - } - alen := as.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, - func(idx int, tsr ...tensor.Tensor) { - ai := as.IndexFrom1D(idx) - bi := tensor.WrapIndex1D(bs, ai...) - tsr[0].SetFloat1D(tsr[0].Float1D(idx)*tsr[1].Float1D(bi), idx) - }, a, b) - return nil + return tensor.FloatAssignFunc(func(a, b float64) float64 { return a * b }, a, b) } // DivAssign does /= divide assign values from b into a. func DivAssign(a, b tensor.Tensor) error { - as, bs, err := tensor.AlignForAssign(a, b) - if err != nil { - return err - } - alen := as.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, - func(idx int, tsr ...tensor.Tensor) { - ai := as.IndexFrom1D(idx) - bi := tensor.WrapIndex1D(bs, ai...) - tsr[0].SetFloat1D(tsr[0].Float1D(idx)/tsr[1].Float1D(bi), idx) - }, a, b) - return nil + return tensor.FloatAssignFunc(func(a, b float64) float64 { return a / b }, a, b) } // Inc increments values in given tensor by 1. func Inc(a tensor.Tensor) error { alen := a.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return alen }, func(idx int, tsr ...tensor.Tensor) { tsr[0].SetFloat1D(tsr[0].Float1D(idx)+1.0, idx) }, a) @@ -129,9 +66,7 @@ func Inc(a tensor.Tensor) error { // Dec decrements values in given tensor by 1. func Dec(a tensor.Tensor) error { alen := a.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return alen - }, + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return alen }, func(idx int, tsr ...tensor.Tensor) { tsr[0].SetFloat1D(tsr[0].Float1D(idx)-1.0, idx) }, a) @@ -145,22 +80,10 @@ func Add(a, b tensor.Tensor) tensor.Tensor { // AddOut adds two tensors into output. func AddOut(a, b tensor.Tensor, out tensor.Values) error { - as, bs, os, err := tensor.AlignShapes(a, b) - if err != nil { - return err + if a.IsString() { + return tensor.StringBinaryFuncOut(func(a, b string) string { return a + b }, a, b, out) } - out.SetShapeSizes(os.Sizes...) - olen := os.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return olen - }, - func(idx int, tsr ...tensor.Tensor) { - oi := os.IndexFrom1D(idx) - ai := tensor.WrapIndex1D(as, oi...) - bi := tensor.WrapIndex1D(bs, oi...) - out.SetFloat1D(tsr[0].Float1D(ai)+tsr[1].Float1D(bi), idx) - }, a, b, out) - return nil + return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a + b }, a, b, out) } // Sub subtracts tensors into output. @@ -170,22 +93,7 @@ func Sub(a, b tensor.Tensor) tensor.Tensor { // SubOut subtracts two tensors into output. func SubOut(a, b tensor.Tensor, out tensor.Values) error { - as, bs, os, err := tensor.AlignShapes(a, b) - if err != nil { - return err - } - out.SetShapeSizes(os.Sizes...) - olen := os.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return olen - }, - func(idx int, tsr ...tensor.Tensor) { - oi := os.IndexFrom1D(idx) - ai := tensor.WrapIndex1D(as, oi...) - bi := tensor.WrapIndex1D(bs, oi...) - out.SetFloat1D(tsr[0].Float1D(ai)-tsr[1].Float1D(bi), idx) - }, a, b, out) - return nil + return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a - b }, a, b, out) } // Mul multiplies tensors into output. @@ -195,22 +103,7 @@ func Mul(a, b tensor.Tensor) tensor.Tensor { // MulOut multiplies two tensors into output. func MulOut(a, b tensor.Tensor, out tensor.Values) error { - as, bs, os, err := tensor.AlignShapes(a, b) - if err != nil { - return err - } - out.SetShapeSizes(os.Sizes...) - olen := os.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return olen - }, - func(idx int, tsr ...tensor.Tensor) { - oi := os.IndexFrom1D(idx) - ai := tensor.WrapIndex1D(as, oi...) - bi := tensor.WrapIndex1D(bs, oi...) - out.SetFloat1D(tsr[0].Float1D(ai)*tsr[1].Float1D(bi), idx) - }, a, b, out) - return nil + return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a * b }, a, b, out) } // Div divides tensors into output. @@ -220,70 +113,5 @@ func Div(a, b tensor.Tensor) tensor.Tensor { // DivOut divides two tensors into output. func DivOut(a, b tensor.Tensor, out tensor.Values) error { - as, bs, os, err := tensor.AlignShapes(a, b) - if err != nil { - return err - } - out.SetShapeSizes(os.Sizes...) - olen := os.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return olen - }, - func(idx int, tsr ...tensor.Tensor) { - oi := os.IndexFrom1D(idx) - ai := tensor.WrapIndex1D(as, oi...) - bi := tensor.WrapIndex1D(bs, oi...) - out.SetFloat1D(tsr[0].Float1D(ai)/tsr[1].Float1D(bi), idx) - }, a, b, out) - return nil -} - -// Greater stores in the output the bool value a > b. -func Greater(a, b tensor.Tensor) tensor.Tensor { - return tensor.CallOut2Bool(GreaterOut, a, b) -} - -// GreaterOut stores in the output the bool value a > b. -func GreaterOut(a, b tensor.Tensor, out *tensor.Bool) error { - as, bs, os, err := tensor.AlignShapes(a, b) - if err != nil { - return err - } - out.SetShapeSizes(os.Sizes...) - olen := os.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return olen - }, - func(idx int, tsr ...tensor.Tensor) { - oi := os.IndexFrom1D(idx) - ai := tensor.WrapIndex1D(as, oi...) - bi := tensor.WrapIndex1D(bs, oi...) - out.SetBool1D(tsr[0].Float1D(ai) > tsr[1].Float1D(bi), idx) - }, a, b, out) - return nil -} - -// Less stores in the output the bool value a > b. -func Less(a, b tensor.Tensor) tensor.Tensor { - return tensor.CallOut2Bool(LessOut, a, b) -} - -// LessOut stores in the output the bool value a > b. -func LessOut(a, b tensor.Tensor, out *tensor.Bool) error { - as, bs, os, err := tensor.AlignShapes(a, b) - if err != nil { - return err - } - out.SetShapeSizes(os.Sizes...) - olen := os.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { - return olen - }, - func(idx int, tsr ...tensor.Tensor) { - oi := os.IndexFrom1D(idx) - ai := tensor.WrapIndex1D(as, oi...) - bi := tensor.WrapIndex1D(bs, oi...) - out.SetBool1D(tsr[0].Float1D(ai) < tsr[1].Float1D(bi), idx) - }, a, b, out) - return nil + return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a / b }, a, b, out) } diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 508a7174d1..9f15c4f40d 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -160,22 +160,3 @@ func TestOps(t *testing.T) { assert.InDelta(t, v, onedc.Float1D(i), 1.0e-7) } } - -func TestBoolOps(t *testing.T) { - ar := tensor.NewSliceInts(12) - // fmt.Println(v) - bo := tensor.NewBool() - sc := tensor.NewIntScalar(6) - - GreaterOut(ar, sc, bo) - // fmt.Println(bo) - for i, v := range ar.Values { - assert.Equal(t, v > 6, bo.Bool1D(i)) - } - - LessOut(ar, sc, bo) - // fmt.Println(bo) - for i, v := range ar.Values { - assert.Equal(t, v < 6, bo.Bool1D(i)) - } -} From 121713f5a1813d8f9b60f3e1f4355fafce5045f5 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 16:33:28 -0700 Subject: [PATCH 133/311] bool masking with simple binary comparison expressions working --- goal/transpile/math.go | 40 +++- goal/transpile/transpile_test.go | 63 +++--- tensor/convert.go | 3 + tensor/masked.go | 12 ++ .../cogentcore_org-core-tensor-tmath.go | 194 +++++++++--------- .../symbols/cogentcore_org-core-tensor.go | 12 ++ 6 files changed, 196 insertions(+), 128 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index c4061915f4..48bfd02047 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -17,6 +17,7 @@ import ( func init() { tensor.AddFunc("tensor.Reshape", tensor.Reshape) tensor.AddFunc("tensor.Reslice", tensor.Reslice) + tensor.AddFunc("tensor.Mask", tensor.Mask) tensor.AddFunc("tensor.NewFloat64", tensor.NewFloat64) tensor.AddFunc("tensor.NewIntScalar", tensor.NewIntScalar) tensor.AddFunc("tensor.NewFloat64Scalar", tensor.NewFloat64Scalar) @@ -307,9 +308,7 @@ func (mp *mathParse) expr(ex ast.Expr) { mp.callExpr(x) case *ast.ArrayType: - // basically at this point we have a bad expression and - // need to do our own parsing. - // it is unclear if perhaps we just need to do that from the start. + // note: shouldn't happen normally: fmt.Println("array type:", x, x.Len) fmt.Printf("%#v\n", x.Len) } @@ -354,6 +353,27 @@ func (mp *mathParse) argsList(ex []ast.Expr) { } } +func (mp *mathParse) exprIsBool(ex ast.Expr) bool { + switch x := ex.(type) { + case *ast.BinaryExpr: + if (x.Op >= token.EQL && x.Op <= token.GTR) || (x.Op >= token.NEQ && x.Op <= token.GEQ) { + return true + } + case *ast.ParenExpr: + return mp.exprIsBool(x.X) + } + return false +} + +func (mp *mathParse) exprsAreBool(ex []ast.Expr) bool { + for _, x := range ex { + if mp.exprIsBool(x) { + return true + } + } + return false +} + func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn := "" switch ex.Op { @@ -365,10 +385,18 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn = "Mul" case token.QUO: fn = "Div" + case token.EQL: + fn = "Equal" case token.LSS: fn = "Less" case token.GTR: fn = "Greater" + case token.NEQ: + fn = "NotEqual" + case token.LEQ: + fn = "LessEqual" + case token.GEQ: + fn = "GreaterEqual" default: fmt.Println("binary token:", ex.Op) } @@ -527,7 +555,11 @@ func (mp *mathParse) indexExpr(il *ast.IndexExpr) { func (mp *mathParse) basicSlicingExpr(il *ast.IndexExpr) { iil := il.Index.(*ast.IndexListExpr) - mp.startFunc("tensor.Reslice") + fun := "tensor.Reslice" + if mp.exprsAreBool(iil.Indices) { + fun = "tensor.Mask" + } + mp.startFunc(fun) mp.out.Add(token.LPAREN) mp.expr(il.X) mp.nextArg() diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index d84614353a..96a7dbb139 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -203,38 +203,39 @@ goal.Run("ls", "-la", "args...") func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - // {"# x := 1", `x := tensor.NewIntScalar(1)`}, - // {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, - // {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, - // {"# a = x + y", `a = tmath.Add(x, y)`}, - // {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - // {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, - // {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, - // {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - // {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, - // {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, - // {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - // {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, - // {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - // {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, - // {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, - // {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, - // {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, - // {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, - // {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, - // {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, - // {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, - // {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, - // {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, - // {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - // {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - // {"# c := cos(a)", `c := tmath.Cos(a)`}, - // {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, - // {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, - // {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, - // {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, - // {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, + {"# x := 1", `x := tensor.NewIntScalar(1)`}, + {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, + {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, + {"# a = x + y", `a = tmath.Add(x, y)`}, + {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, + {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, + {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, + {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, + {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, + {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, + {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, + {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, + {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, + {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, + {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, + {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, + {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, + {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, + {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, + {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, + {"# c := cos(a)", `c := tmath.Cos(a)`}, + {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, + {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, + {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, + {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, + {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, {"# b := a > 5", `b := tmath.Greater(a, tensor.NewIntScalar(5))`}, + {"# b := a[a > 5]", `b := tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5)))`}, } st := NewState() diff --git a/tensor/convert.go b/tensor/convert.go index fb0a17d8a2..c63e7da7e8 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -25,6 +25,9 @@ func Clone(tsr Tensor) Values { // of values, by calling Clone(As1D(tsr)). // It is equivalent to the NumPy flatten function. func Flatten(tsr Tensor) Values { + if msk, ok := tsr.(*Masked); ok { + return msk.AsValues() + } return Clone(As1D(tsr)) } diff --git a/tensor/masked.go b/tensor/masked.go index 0cf1e6f776..86ac1159e1 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -8,6 +8,7 @@ import ( "math" "reflect" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" ) @@ -44,6 +45,17 @@ func NewMasked(tsr Tensor, mask ...*Bool) *Masked { return ms } +// Mask is the general purpose masking function, which checks +// if the mask arg is a Bool and uses if so. +// Otherwise, it logs an error. +func Mask(tsr, mask Tensor) *Masked { + if mb, ok := mask.(*Bool); ok { + return NewMasked(tsr, mb) + } + errors.Log(errors.New("tensor.Mask: provided tensor is not a Bool tensor")) + return NewMasked(tsr) +} + // AsMasked returns the tensor as a [Masked] view. // If it already is one, then it is returned, otherwise it is wrapped // with an initially fully transparent mask. diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go index 4413bcf6ff..d97f23a1e9 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -10,98 +10,106 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/tmath/tmath"] = map[string]reflect.Value{ // function, constant and variable definitions - "Abs": reflect.ValueOf(tmath.Abs), - "AbsOut": reflect.ValueOf(tmath.AbsOut), - "Acos": reflect.ValueOf(tmath.Acos), - "AcosOut": reflect.ValueOf(tmath.AcosOut), - "Acosh": reflect.ValueOf(tmath.Acosh), - "AcoshOut": reflect.ValueOf(tmath.AcoshOut), - "Add": reflect.ValueOf(tmath.Add), - "AddAssign": reflect.ValueOf(tmath.AddAssign), - "AddOut": reflect.ValueOf(tmath.AddOut), - "Asin": reflect.ValueOf(tmath.Asin), - "AsinOut": reflect.ValueOf(tmath.AsinOut), - "Asinh": reflect.ValueOf(tmath.Asinh), - "AsinhOut": reflect.ValueOf(tmath.AsinhOut), - "Assign": reflect.ValueOf(tmath.Assign), - "Atan": reflect.ValueOf(tmath.Atan), - "AtanOut": reflect.ValueOf(tmath.AtanOut), - "Atanh": reflect.ValueOf(tmath.Atanh), - "AtanhOut": reflect.ValueOf(tmath.AtanhOut), - "Cbrt": reflect.ValueOf(tmath.Cbrt), - "CbrtOut": reflect.ValueOf(tmath.CbrtOut), - "Ceil": reflect.ValueOf(tmath.Ceil), - "CeilOut": reflect.ValueOf(tmath.CeilOut), - "Cos": reflect.ValueOf(tmath.Cos), - "CosOut": reflect.ValueOf(tmath.CosOut), - "Cosh": reflect.ValueOf(tmath.Cosh), - "CoshOut": reflect.ValueOf(tmath.CoshOut), - "Dec": reflect.ValueOf(tmath.Dec), - "Div": reflect.ValueOf(tmath.Div), - "DivAssign": reflect.ValueOf(tmath.DivAssign), - "DivOut": reflect.ValueOf(tmath.DivOut), - "Erf": reflect.ValueOf(tmath.Erf), - "ErfOut": reflect.ValueOf(tmath.ErfOut), - "Erfc": reflect.ValueOf(tmath.Erfc), - "ErfcOut": reflect.ValueOf(tmath.ErfcOut), - "Erfcinv": reflect.ValueOf(tmath.Erfcinv), - "ErfcinvOut": reflect.ValueOf(tmath.ErfcinvOut), - "Erfinv": reflect.ValueOf(tmath.Erfinv), - "ErfinvOut": reflect.ValueOf(tmath.ErfinvOut), - "Exp": reflect.ValueOf(tmath.Exp), - "Exp2": reflect.ValueOf(tmath.Exp2), - "Exp2Out": reflect.ValueOf(tmath.Exp2Out), - "ExpOut": reflect.ValueOf(tmath.ExpOut), - "Expm1": reflect.ValueOf(tmath.Expm1), - "Expm1Out": reflect.ValueOf(tmath.Expm1Out), - "Floor": reflect.ValueOf(tmath.Floor), - "FloorOut": reflect.ValueOf(tmath.FloorOut), - "Gamma": reflect.ValueOf(tmath.Gamma), - "GammaOut": reflect.ValueOf(tmath.GammaOut), - "Greater": reflect.ValueOf(tmath.Greater), - "GreaterOut": reflect.ValueOf(tmath.GreaterOut), - "Inc": reflect.ValueOf(tmath.Inc), - "J0": reflect.ValueOf(tmath.J0), - "J0Out": reflect.ValueOf(tmath.J0Out), - "J1": reflect.ValueOf(tmath.J1), - "J1Out": reflect.ValueOf(tmath.J1Out), - "Less": reflect.ValueOf(tmath.Less), - "LessOut": reflect.ValueOf(tmath.LessOut), - "Log": reflect.ValueOf(tmath.Log), - "Log10": reflect.ValueOf(tmath.Log10), - "Log10Out": reflect.ValueOf(tmath.Log10Out), - "Log1p": reflect.ValueOf(tmath.Log1p), - "Log1pOut": reflect.ValueOf(tmath.Log1pOut), - "Log2": reflect.ValueOf(tmath.Log2), - "Log2Out": reflect.ValueOf(tmath.Log2Out), - "LogOut": reflect.ValueOf(tmath.LogOut), - "Logb": reflect.ValueOf(tmath.Logb), - "LogbOut": reflect.ValueOf(tmath.LogbOut), - "Mul": reflect.ValueOf(tmath.Mul), - "MulAssign": reflect.ValueOf(tmath.MulAssign), - "MulOut": reflect.ValueOf(tmath.MulOut), - "Round": reflect.ValueOf(tmath.Round), - "RoundOut": reflect.ValueOf(tmath.RoundOut), - "RoundToEven": reflect.ValueOf(tmath.RoundToEven), - "RoundToEvenOut": reflect.ValueOf(tmath.RoundToEvenOut), - "Sin": reflect.ValueOf(tmath.Sin), - "SinOut": reflect.ValueOf(tmath.SinOut), - "Sinh": reflect.ValueOf(tmath.Sinh), - "SinhOut": reflect.ValueOf(tmath.SinhOut), - "Sqrt": reflect.ValueOf(tmath.Sqrt), - "SqrtOut": reflect.ValueOf(tmath.SqrtOut), - "Sub": reflect.ValueOf(tmath.Sub), - "SubAssign": reflect.ValueOf(tmath.SubAssign), - "SubOut": reflect.ValueOf(tmath.SubOut), - "Tan": reflect.ValueOf(tmath.Tan), - "TanOut": reflect.ValueOf(tmath.TanOut), - "Tanh": reflect.ValueOf(tmath.Tanh), - "TanhOut": reflect.ValueOf(tmath.TanhOut), - "Trunc": reflect.ValueOf(tmath.Trunc), - "TruncOut": reflect.ValueOf(tmath.TruncOut), - "Y0": reflect.ValueOf(tmath.Y0), - "Y0Out": reflect.ValueOf(tmath.Y0Out), - "Y1": reflect.ValueOf(tmath.Y1), - "Y1Out": reflect.ValueOf(tmath.Y1Out), + "Abs": reflect.ValueOf(tmath.Abs), + "AbsOut": reflect.ValueOf(tmath.AbsOut), + "Acos": reflect.ValueOf(tmath.Acos), + "AcosOut": reflect.ValueOf(tmath.AcosOut), + "Acosh": reflect.ValueOf(tmath.Acosh), + "AcoshOut": reflect.ValueOf(tmath.AcoshOut), + "Add": reflect.ValueOf(tmath.Add), + "AddAssign": reflect.ValueOf(tmath.AddAssign), + "AddOut": reflect.ValueOf(tmath.AddOut), + "Asin": reflect.ValueOf(tmath.Asin), + "AsinOut": reflect.ValueOf(tmath.AsinOut), + "Asinh": reflect.ValueOf(tmath.Asinh), + "AsinhOut": reflect.ValueOf(tmath.AsinhOut), + "Assign": reflect.ValueOf(tmath.Assign), + "Atan": reflect.ValueOf(tmath.Atan), + "AtanOut": reflect.ValueOf(tmath.AtanOut), + "Atanh": reflect.ValueOf(tmath.Atanh), + "AtanhOut": reflect.ValueOf(tmath.AtanhOut), + "Cbrt": reflect.ValueOf(tmath.Cbrt), + "CbrtOut": reflect.ValueOf(tmath.CbrtOut), + "Ceil": reflect.ValueOf(tmath.Ceil), + "CeilOut": reflect.ValueOf(tmath.CeilOut), + "Cos": reflect.ValueOf(tmath.Cos), + "CosOut": reflect.ValueOf(tmath.CosOut), + "Cosh": reflect.ValueOf(tmath.Cosh), + "CoshOut": reflect.ValueOf(tmath.CoshOut), + "Dec": reflect.ValueOf(tmath.Dec), + "Div": reflect.ValueOf(tmath.Div), + "DivAssign": reflect.ValueOf(tmath.DivAssign), + "DivOut": reflect.ValueOf(tmath.DivOut), + "Equal": reflect.ValueOf(tmath.Equal), + "EqualOut": reflect.ValueOf(tmath.EqualOut), + "Erf": reflect.ValueOf(tmath.Erf), + "ErfOut": reflect.ValueOf(tmath.ErfOut), + "Erfc": reflect.ValueOf(tmath.Erfc), + "ErfcOut": reflect.ValueOf(tmath.ErfcOut), + "Erfcinv": reflect.ValueOf(tmath.Erfcinv), + "ErfcinvOut": reflect.ValueOf(tmath.ErfcinvOut), + "Erfinv": reflect.ValueOf(tmath.Erfinv), + "ErfinvOut": reflect.ValueOf(tmath.ErfinvOut), + "Exp": reflect.ValueOf(tmath.Exp), + "Exp2": reflect.ValueOf(tmath.Exp2), + "Exp2Out": reflect.ValueOf(tmath.Exp2Out), + "ExpOut": reflect.ValueOf(tmath.ExpOut), + "Expm1": reflect.ValueOf(tmath.Expm1), + "Expm1Out": reflect.ValueOf(tmath.Expm1Out), + "Floor": reflect.ValueOf(tmath.Floor), + "FloorOut": reflect.ValueOf(tmath.FloorOut), + "Gamma": reflect.ValueOf(tmath.Gamma), + "GammaOut": reflect.ValueOf(tmath.GammaOut), + "Greater": reflect.ValueOf(tmath.Greater), + "GreaterEqual": reflect.ValueOf(tmath.GreaterEqual), + "GreaterEqualOut": reflect.ValueOf(tmath.GreaterEqualOut), + "GreaterOut": reflect.ValueOf(tmath.GreaterOut), + "Inc": reflect.ValueOf(tmath.Inc), + "J0": reflect.ValueOf(tmath.J0), + "J0Out": reflect.ValueOf(tmath.J0Out), + "J1": reflect.ValueOf(tmath.J1), + "J1Out": reflect.ValueOf(tmath.J1Out), + "Less": reflect.ValueOf(tmath.Less), + "LessEqual": reflect.ValueOf(tmath.LessEqual), + "LessEqualOut": reflect.ValueOf(tmath.LessEqualOut), + "LessOut": reflect.ValueOf(tmath.LessOut), + "Log": reflect.ValueOf(tmath.Log), + "Log10": reflect.ValueOf(tmath.Log10), + "Log10Out": reflect.ValueOf(tmath.Log10Out), + "Log1p": reflect.ValueOf(tmath.Log1p), + "Log1pOut": reflect.ValueOf(tmath.Log1pOut), + "Log2": reflect.ValueOf(tmath.Log2), + "Log2Out": reflect.ValueOf(tmath.Log2Out), + "LogOut": reflect.ValueOf(tmath.LogOut), + "Logb": reflect.ValueOf(tmath.Logb), + "LogbOut": reflect.ValueOf(tmath.LogbOut), + "Mul": reflect.ValueOf(tmath.Mul), + "MulAssign": reflect.ValueOf(tmath.MulAssign), + "MulOut": reflect.ValueOf(tmath.MulOut), + "NotEqual": reflect.ValueOf(tmath.NotEqual), + "NotEqualOut": reflect.ValueOf(tmath.NotEqualOut), + "Round": reflect.ValueOf(tmath.Round), + "RoundOut": reflect.ValueOf(tmath.RoundOut), + "RoundToEven": reflect.ValueOf(tmath.RoundToEven), + "RoundToEvenOut": reflect.ValueOf(tmath.RoundToEvenOut), + "Sin": reflect.ValueOf(tmath.Sin), + "SinOut": reflect.ValueOf(tmath.SinOut), + "Sinh": reflect.ValueOf(tmath.Sinh), + "SinhOut": reflect.ValueOf(tmath.SinhOut), + "Sqrt": reflect.ValueOf(tmath.Sqrt), + "SqrtOut": reflect.ValueOf(tmath.SqrtOut), + "Sub": reflect.ValueOf(tmath.Sub), + "SubAssign": reflect.ValueOf(tmath.SubAssign), + "SubOut": reflect.ValueOf(tmath.SubOut), + "Tan": reflect.ValueOf(tmath.Tan), + "TanOut": reflect.ValueOf(tmath.TanOut), + "Tanh": reflect.ValueOf(tmath.Tanh), + "TanhOut": reflect.ValueOf(tmath.TanhOut), + "Trunc": reflect.ValueOf(tmath.Trunc), + "TruncOut": reflect.ValueOf(tmath.TruncOut), + "Y0": reflect.ValueOf(tmath.Y0), + "Y0Out": reflect.ValueOf(tmath.Y0Out), + "Y1": reflect.ValueOf(tmath.Y1), + "Y1Out": reflect.ValueOf(tmath.Y1Out), } } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 6932944829..f7e724d784 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -32,6 +32,10 @@ func init() { "AsStringSlice": reflect.ValueOf(tensor.AsStringSlice), "AsStringTensor": reflect.ValueOf(tensor.AsStringTensor), "Ascending": reflect.ValueOf(tensor.Ascending), + "BoolFloatsFunc": reflect.ValueOf(tensor.BoolFloatsFunc), + "BoolFloatsFuncOut": reflect.ValueOf(tensor.BoolFloatsFuncOut), + "BoolStringsFunc": reflect.ValueOf(tensor.BoolStringsFunc), + "BoolStringsFuncOut": reflect.ValueOf(tensor.BoolStringsFuncOut), "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), "BoolToInt": reflect.ValueOf(tensor.BoolToInt), "Calc": reflect.ValueOf(tensor.Calc), @@ -53,10 +57,14 @@ func init() { "Flatten": reflect.ValueOf(tensor.Flatten), "Float64ToBool": reflect.ValueOf(tensor.Float64ToBool), "Float64ToString": reflect.ValueOf(tensor.Float64ToString), + "FloatAssignFunc": reflect.ValueOf(tensor.FloatAssignFunc), + "FloatBinaryFunc": reflect.ValueOf(tensor.FloatBinaryFunc), + "FloatBinaryFuncOut": reflect.ValueOf(tensor.FloatBinaryFuncOut), "FullAxis": reflect.ValueOf(tensor.FullAxis), "FuncByName": reflect.ValueOf(tensor.FuncByName), "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), "IntToBool": reflect.ValueOf(tensor.IntToBool), + "Mask": reflect.ValueOf(tensor.Mask), "MaxSprintLength": reflect.ValueOf(&tensor.MaxSprintLength).Elem(), "MustBeSameShape": reflect.ValueOf(tensor.MustBeSameShape), "MustBeValues": reflect.ValueOf(tensor.MustBeValues), @@ -128,6 +136,9 @@ func init() { "Sprint": reflect.ValueOf(tensor.Sprint), "Squeeze": reflect.ValueOf(tensor.Squeeze), "StableSort": reflect.ValueOf(tensor.StableSort), + "StringAssignFunc": reflect.ValueOf(tensor.StringAssignFunc), + "StringBinaryFunc": reflect.ValueOf(tensor.StringBinaryFunc), + "StringBinaryFuncOut": reflect.ValueOf(tensor.StringBinaryFuncOut), "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), "Tab": reflect.ValueOf(tensor.Tab), "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), @@ -139,6 +150,7 @@ func init() { "WriteCSV": reflect.ValueOf(tensor.WriteCSV), // type definitions + "Arg": reflect.ValueOf((*tensor.Arg)(nil)), "Bool": reflect.ValueOf((*tensor.Bool)(nil)), "Delims": reflect.ValueOf((*tensor.Delims)(nil)), "FilterFunc": reflect.ValueOf((*tensor.FilterFunc)(nil)), From 5a9c58221750bb5ea4b5eb1e96feec30be5a9636 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 16:43:22 -0700 Subject: [PATCH 134/311] table updates --- goal/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/goal/README.md b/goal/README.md index 7628ae02bc..10100348c2 100644 --- a/goal/README.md +++ b/goal/README.md @@ -315,12 +315,13 @@ See [NumPy boolean indexing](https://numpy.org/doc/stable/user/basics.indexing.h | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| `(a > 0.5)` | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | +| `tmath.Greater(a, 0.5)` | same: | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | | `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | | `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | | `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | -| | |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | +| `tensor.Mask(a,` `tmath.Less(a, 0.5)` | same: |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | +| `tensor.Flatten(tensor.Mask(a,` `tmath.Less(a, 0.5))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 | | | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | ## Advanced index-based indexing @@ -341,6 +342,8 @@ See [NumPy integer indexing](https://numpy.org/doc/stable/user/basics.indexing.h ## Basic math operations (add, multiply, etc) +In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ operations because those are well-defined for all dimensionalities and are consistent across the different operators, whereas matrix multiplication is specifically used in a 2D linear algebra context, and is not well defined for the other operators. + | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | | | |`a * b` | `a .* b` | element-wise multiply | From 6d33c635af9db1e9e0ed4e9fb28e5911ed86aa0b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 16:53:30 -0700 Subject: [PATCH 135/311] table updates --- goal/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/goal/README.md b/goal/README.md index 10100348c2..775cb70728 100644 --- a/goal/README.md +++ b/goal/README.md @@ -221,7 +221,9 @@ TODO: update above # Math mode -In general, Goal is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well. The Goal global functions are named the same as NumPy, without the `np.` prefix, so existing code can be converted by just removing that prefix. Corresponding field-like properties of tensors are converted into into appropriate method calls. +The math mode in Goal is designed to be generally compatible with Python NumPy / SciPy syntax, so that the widespread experience with that syntax transfers well to Goal. This syntax is also largely compatible with MATLAB and other languages as well. However, we did not fully replicate the NumPy syntax, instead choosing to clean up a few things and generally increase consistency with Go. + +In general the Goal global functions are named the same as NumPy, without the `np.` prefix, which improves readability. It should be very straightforward to write a conversion utility that converts existing NumPy code into Goal code, and that is a better process than trying to make Goal itself perfectly compatible. All elements of a Goal math expression are [tensors](../tensor) (i.e., `tensor.Tensor`), which can represent everything from a scalar to an n-dimenstional tensor, with different _views_ that support the arbitrary slicing and flexible forms of indexing documented in the table below. These are called an `ndarray` in NumPy terms. See [array vs. tensor](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html#array-or-matrix-which-should-i-use) NumPy docs for more information. Note that Goal does not have a distinct `matrix` type; everything is a tensor, and when these are 2D, they function appropriately via the [matrix](../tensor/matrix) package. @@ -321,7 +323,7 @@ See [NumPy boolean indexing](https://numpy.org/doc/stable/user/basics.indexing.h | `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | | `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | | `tensor.Mask(a,` `tmath.Less(a, 0.5)` | same: |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | -| `tensor.Flatten(tensor.Mask(a,` `tmath.Less(a, 0.5))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 | +| `tensor.Flatten(` `tensor.Mask(a,` `tmath.Less(a, 0.5))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 (as a copy, not a view) | | | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | ## Advanced index-based indexing From 85626d9e21a976a6a142e37ec600ff96f90ee09c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 17:27:44 -0700 Subject: [PATCH 136/311] better Mask return as plain tensor if no bools --- goal/README.md | 15 ++++++++++++++- tensor/masked.go | 4 ++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/goal/README.md b/goal/README.md index 775cb70728..cc31b56d0e 100644 --- a/goal/README.md +++ b/goal/README.md @@ -323,7 +323,7 @@ See [NumPy boolean indexing](https://numpy.org/doc/stable/user/basics.indexing.h | `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | | `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | | `tensor.Mask(a,` `tmath.Less(a, 0.5)` | same: |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | -| `tensor.Flatten(` `tensor.Mask(a,` `tmath.Less(a, 0.5))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 (as a copy, not a view) | +| `tensor.Flatten(` `tensor.Mask(a,` `tmath.Less(a, 0.5)))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 (as a copy, not a view) | | | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | ## Advanced index-based indexing @@ -397,3 +397,16 @@ todo: huge amount of work needed to support complex numbers throughout! | | |`np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of `a` | | | |`signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | +## TODO + +* bool &&, || ! +* flatten etc selector calls work with complex exprs. +* basic go expressions in math mode -- see how that works +* # # surrounding in go, shell modes +* more creation routines +* update readme table +* record to datafs +* make a simple tutorial example showing basic ops + + + diff --git a/tensor/masked.go b/tensor/masked.go index 86ac1159e1..8de35cf84b 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -48,12 +48,12 @@ func NewMasked(tsr Tensor, mask ...*Bool) *Masked { // Mask is the general purpose masking function, which checks // if the mask arg is a Bool and uses if so. // Otherwise, it logs an error. -func Mask(tsr, mask Tensor) *Masked { +func Mask(tsr, mask Tensor) Tensor { if mb, ok := mask.(*Bool); ok { return NewMasked(tsr, mb) } errors.Log(errors.New("tensor.Mask: provided tensor is not a Bool tensor")) - return NewMasked(tsr) + return tsr } // AsMasked returns the tensor as a [Masked] view. From 7aa21abe4347f534ed6637af8474d0bfa8d0710c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 26 Sep 2024 23:27:06 -0700 Subject: [PATCH 137/311] unary operators properly applied to tensor objects --- goal/transpile/math.go | 21 +++++++- goal/transpile/transpile_test.go | 3 ++ tensor/align.go | 51 ++++++++++++------- tensor/tmath/bool.go | 42 +++++++++++++++ tensor/tmath/ops.go | 36 +++++++++++++ .../cogentcore_org-core-tensor-tmath.go | 11 ++++ .../symbols/cogentcore_org-core-tensor.go | 2 + 7 files changed, 147 insertions(+), 19 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 48bfd02047..f5e4164894 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -397,6 +397,10 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn = "LessEqual" case token.GEQ: fn = "GreaterEqual" + case token.LOR: + fn = "Or" + case token.LAND: + fn = "And" default: fmt.Println("binary token:", ex.Op) } @@ -411,8 +415,23 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { } func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { - mp.addToken(ex.Op) + if _, isbl := ex.X.(*ast.BasicLit); isbl { + mp.addToken(ex.Op) + mp.expr(ex.X) + return + } + fn := "" + switch ex.Op { + case token.NOT: + fn = "Not" + case token.SUB: + fn = "Negate" + } + mp.startFunc("tmath." + fn) + mp.addToken(token.LPAREN) mp.expr(ex.X) + mp.out.Add(token.RPAREN) + mp.endFunc() } func (mp *mathParse) defineStmt(as *ast.AssignStmt) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 96a7dbb139..7376ca5cb3 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -207,6 +207,7 @@ func TestMath(t *testing.T) { {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, {"# a = x + y", `a = tmath.Add(x, y)`}, + {"# a = -x", `a = tmath.Negate(x)`}, {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, @@ -235,7 +236,9 @@ func TestMath(t *testing.T) { {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, {"# b := a > 5", `b := tmath.Greater(a, tensor.NewIntScalar(5))`}, + {"# b := !a", `b := tmath.Not(a)`}, {"# b := a[a > 5]", `b := tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5)))`}, + {"# a > 5 || a < 1", `tmath.Or(tmath.Greater(a, tensor.NewIntScalar(5)), tmath.Less(a, tensor.NewIntScalar(1)))`}, } st := NewState() diff --git a/tensor/align.go b/tensor/align.go index 04d9752962..3318f27328 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -127,9 +127,7 @@ func FloatAssignFunc(fun func(a, b float64) float64, a, b Tensor) error { return err } alen := as.Len() - VectorizeThreaded(1, func(tsr ...Tensor) int { - return alen - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return alen }, func(idx int, tsr ...Tensor) { ai := as.IndexFrom1D(idx) bi := WrapIndex1D(bs, ai...) @@ -145,9 +143,7 @@ func StringAssignFunc(fun func(a, b string) string, a, b Tensor) error { return err } alen := as.Len() - VectorizeThreaded(1, func(tsr ...Tensor) int { - return alen - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return alen }, func(idx int, tsr ...Tensor) { ai := as.IndexFrom1D(idx) bi := WrapIndex1D(bs, ai...) @@ -169,9 +165,7 @@ func FloatBinaryFuncOut(fun func(a, b float64) float64, a, b Tensor, out Values) } out.SetShapeSizes(os.Sizes...) olen := os.Len() - VectorizeThreaded(1, func(tsr ...Tensor) int { - return olen - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return olen }, func(idx int, tsr ...Tensor) { oi := os.IndexFrom1D(idx) ai := WrapIndex1D(as, oi...) @@ -194,9 +188,7 @@ func StringBinaryFuncOut(fun func(a, b string) string, a, b Tensor, out Values) } out.SetShapeSizes(os.Sizes...) olen := os.Len() - VectorizeThreaded(1, func(tsr ...Tensor) int { - return olen - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return olen }, func(idx int, tsr ...Tensor) { oi := os.IndexFrom1D(idx) ai := WrapIndex1D(as, oi...) @@ -225,9 +217,7 @@ func BoolStringsFuncOut(fun func(a, b string) bool, a, b Tensor, out *Bool) erro } out.SetShapeSizes(os.Sizes...) olen := os.Len() - VectorizeThreaded(5, func(tsr ...Tensor) int { - return olen - }, + VectorizeThreaded(5, func(tsr ...Tensor) int { return olen }, func(idx int, tsr ...Tensor) { oi := os.IndexFrom1D(idx) ai := WrapIndex1D(as, oi...) @@ -254,9 +244,7 @@ func BoolFloatsFuncOut(fun func(a, b float64) bool, a, b Tensor, out *Bool) erro } out.SetShapeSizes(os.Sizes...) olen := os.Len() - VectorizeThreaded(5, func(tsr ...Tensor) int { - return olen - }, + VectorizeThreaded(5, func(tsr ...Tensor) int { return olen }, func(idx int, tsr ...Tensor) { oi := os.IndexFrom1D(idx) ai := WrapIndex1D(as, oi...) @@ -265,3 +253,30 @@ func BoolFloatsFuncOut(fun func(a, b float64) bool, a, b Tensor, out *Bool) erro }, a, b, out) return nil } + +// BoolIntsFunc sets boolean output value based on a function involving +// int values from the two tensors. +func BoolIntsFunc(fun func(a, b int) bool, a, b Tensor) *Bool { + out := NewBool() + errors.Log(BoolIntsFuncOut(fun, a, b, out)) + return out +} + +// BoolIntsFuncOut sets boolean output value based on a function involving +// int values from the two tensors. +func BoolIntsFuncOut(fun func(a, b int) bool, a, b Tensor, out *Bool) error { + as, bs, os, err := AlignShapes(a, b) + if err != nil { + return err + } + out.SetShapeSizes(os.Sizes...) + olen := os.Len() + VectorizeThreaded(5, func(tsr ...Tensor) int { return olen }, + func(idx int, tsr ...Tensor) { + oi := os.IndexFrom1D(idx) + ai := WrapIndex1D(as, oi...) + bi := WrapIndex1D(bs, oi...) + out.SetBool1D(fun(tsr[0].Int1D(ai), tsr[1].Int1D(bi)), idx) + }, a, b, out) + return nil +} diff --git a/tensor/tmath/bool.go b/tensor/tmath/bool.go index facd1d3be9..f45e104896 100644 --- a/tensor/tmath/bool.go +++ b/tensor/tmath/bool.go @@ -5,6 +5,7 @@ package tmath import ( + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" ) @@ -15,6 +16,9 @@ func init() { tensor.AddFunc("tmath.NotEqual", NotEqual) tensor.AddFunc("tmath.LessEqual", LessEqual) tensor.AddFunc("tmath.GreaterEqual", GreaterEqual) + tensor.AddFunc("tmath.Or", Or) + tensor.AddFunc("tmath.And", And) + tensor.AddFunc("tmath.Not", Not) } // Equal stores in the output the bool value a == b. @@ -94,3 +98,41 @@ func GreaterEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { } return tensor.BoolFloatsFuncOut(func(a, b float64) bool { return a >= b }, a, b, out) } + +// Or stores in the output the bool value a || b. +func Or(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(OrOut, a, b) +} + +// OrOut stores in the output the bool value a || b. +func OrOut(a, b tensor.Tensor, out *tensor.Bool) error { + return tensor.BoolIntsFuncOut(func(a, b int) bool { return a > 0 || b > 0 }, a, b, out) +} + +// And stores in the output the bool value a || b. +func And(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2Bool(AndOut, a, b) +} + +// AndOut stores in the output the bool value a || b. +func AndOut(a, b tensor.Tensor, out *tensor.Bool) error { + return tensor.BoolIntsFuncOut(func(a, b int) bool { return a > 0 && b > 0 }, a, b, out) +} + +// Not stores in the output the bool value !a. +func Not(a tensor.Tensor) tensor.Tensor { + out := tensor.NewBool() + errors.Log(NotOut(a, out)) + return out +} + +// NotOut stores in the output the bool value !a. +func NotOut(a tensor.Tensor, out *tensor.Bool) error { + out.SetShapeSizes(a.Shape().Sizes...) + alen := a.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return alen }, + func(idx int, tsr ...tensor.Tensor) { + out.SetBool1D(tsr[0].Int1D(idx) == 0, idx) + }, a, out) + return nil +} diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 8d688fd059..3c48292f43 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -5,6 +5,8 @@ package tmath import ( + "math" + "cogentcore.org/core/tensor" ) @@ -14,6 +16,7 @@ func init() { tensor.AddFunc("tmath.SubAssign", SubAssign) tensor.AddFunc("tmath.MulAssign", MulAssign) tensor.AddFunc("tmath.DivAssign", DivAssign) + tensor.AddFunc("tmath.ModAssign", ModAssign) tensor.AddFunc("tmath.Inc", Inc) tensor.AddFunc("tmath.Dec", Dec) @@ -22,7 +25,9 @@ func init() { tensor.AddFunc("tmath.Sub", Sub) tensor.AddFunc("tmath.Mul", Mul) tensor.AddFunc("tmath.Div", Div) + tensor.AddFunc("tmath.Mod", Mod) + tensor.AddFunc("tmath.Negate", Negate) } // Assign assigns values from b into a. @@ -53,6 +58,11 @@ func DivAssign(a, b tensor.Tensor) error { return tensor.FloatAssignFunc(func(a, b float64) float64 { return a / b }, a, b) } +// ModAssign does %= modulus assign values from b into a. +func ModAssign(a, b tensor.Tensor) error { + return tensor.FloatAssignFunc(func(a, b float64) float64 { return math.Mod(a, b) }, a, b) +} + // Inc increments values in given tensor by 1. func Inc(a tensor.Tensor) error { alen := a.Len() @@ -115,3 +125,29 @@ func Div(a, b tensor.Tensor) tensor.Tensor { func DivOut(a, b tensor.Tensor, out tensor.Values) error { return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a / b }, a, b, out) } + +// Mod performs modulus a%b on tensors into output. +func Mod(a, b tensor.Tensor) tensor.Tensor { + return tensor.CallOut2(ModOut, a, b) +} + +// ModOut performs modulus a%b on tensors into output. +func ModOut(a, b tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return math.Mod(a, b) }, a, b, out) +} + +// Negate stores in the output the bool value -a. +func Negate(a tensor.Tensor) tensor.Tensor { + return tensor.CallOut1(NegateOut, a) +} + +// NegateOut stores in the output the bool value -a. +func NegateOut(a tensor.Tensor, out tensor.Values) error { + out.SetShapeSizes(a.Shape().Sizes...) + alen := a.Len() + tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return alen }, + func(idx int, tsr ...tensor.Tensor) { + out.SetFloat1D(-tsr[0].Float1D(idx), idx) + }, a, out) + return nil +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go index d97f23a1e9..858b857a0c 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -19,6 +19,8 @@ func init() { "Add": reflect.ValueOf(tmath.Add), "AddAssign": reflect.ValueOf(tmath.AddAssign), "AddOut": reflect.ValueOf(tmath.AddOut), + "And": reflect.ValueOf(tmath.And), + "AndOut": reflect.ValueOf(tmath.AndOut), "Asin": reflect.ValueOf(tmath.Asin), "AsinOut": reflect.ValueOf(tmath.AsinOut), "Asinh": reflect.ValueOf(tmath.Asinh), @@ -83,11 +85,20 @@ func init() { "LogOut": reflect.ValueOf(tmath.LogOut), "Logb": reflect.ValueOf(tmath.Logb), "LogbOut": reflect.ValueOf(tmath.LogbOut), + "Mod": reflect.ValueOf(tmath.Mod), + "ModAssign": reflect.ValueOf(tmath.ModAssign), + "ModOut": reflect.ValueOf(tmath.ModOut), "Mul": reflect.ValueOf(tmath.Mul), "MulAssign": reflect.ValueOf(tmath.MulAssign), "MulOut": reflect.ValueOf(tmath.MulOut), + "Negate": reflect.ValueOf(tmath.Negate), + "NegateOut": reflect.ValueOf(tmath.NegateOut), + "Not": reflect.ValueOf(tmath.Not), "NotEqual": reflect.ValueOf(tmath.NotEqual), "NotEqualOut": reflect.ValueOf(tmath.NotEqualOut), + "NotOut": reflect.ValueOf(tmath.NotOut), + "Or": reflect.ValueOf(tmath.Or), + "OrOut": reflect.ValueOf(tmath.OrOut), "Round": reflect.ValueOf(tmath.Round), "RoundOut": reflect.ValueOf(tmath.RoundOut), "RoundToEven": reflect.ValueOf(tmath.RoundToEven), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index f7e724d784..24f87c307d 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -34,6 +34,8 @@ func init() { "Ascending": reflect.ValueOf(tensor.Ascending), "BoolFloatsFunc": reflect.ValueOf(tensor.BoolFloatsFunc), "BoolFloatsFuncOut": reflect.ValueOf(tensor.BoolFloatsFuncOut), + "BoolIntsFunc": reflect.ValueOf(tensor.BoolIntsFunc), + "BoolIntsFuncOut": reflect.ValueOf(tensor.BoolIntsFuncOut), "BoolStringsFunc": reflect.ValueOf(tensor.BoolStringsFunc), "BoolStringsFuncOut": reflect.ValueOf(tensor.BoolStringsFuncOut), "BoolToFloat64": reflect.ValueOf(tensor.BoolToFloat64), From 7ce6e72769e028629a694409d4828794ee0678ac Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 02:04:17 -0700 Subject: [PATCH 138/311] if statement working in math mode --- goal/README.md | 3 --- goal/cmd/goal/testdata/test.goal | 18 +++++++++---- goal/testdata/test.goal | 5 ++++ goal/transpile/math.go | 43 ++++++++++++++++++++++---------- goal/transpile/parser.go | 11 ++++++-- goal/transpile/transpile_test.go | 6 +++++ tensor/tmath/bool.go | 18 ++++++------- 7 files changed, 72 insertions(+), 32 deletions(-) diff --git a/goal/README.md b/goal/README.md index cc31b56d0e..52fb74d45c 100644 --- a/goal/README.md +++ b/goal/README.md @@ -399,8 +399,6 @@ todo: huge amount of work needed to support complex numbers throughout! ## TODO -* bool &&, || ! -* flatten etc selector calls work with complex exprs. * basic go expressions in math mode -- see how that works * # # surrounding in go, shell modes * more creation routines @@ -409,4 +407,3 @@ todo: huge amount of work needed to support complex numbers throughout! * make a simple tutorial example showing basic ops - diff --git a/goal/cmd/goal/testdata/test.goal b/goal/cmd/goal/testdata/test.goal index 79956ffe66..0b7858312e 100644 --- a/goal/cmd/goal/testdata/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -7,11 +7,19 @@ for i, fn := range goalib.SplitLines(`/bin/ls -1`) { fmt.Println(i, fn) } -# x := 1 -# y := 4 -# a := x * 2 -# b := x + y -# c := x * y + a * b +## + +x := 1 +y := 4 +a := x * 2 +b := x + y +c := x * y + a * b fmt.Println(c) +if 1 == 1 { + fmt.Println(true) +} + +## + diff --git a/goal/testdata/test.goal b/goal/testdata/test.goal index fdd2a23b3e..4eeb0850ee 100644 --- a/goal/testdata/test.goal +++ b/goal/testdata/test.goal @@ -9,3 +9,8 @@ fmt.Println(x) fmt.Println(nd) fmt.Println(sz) fmt.Println(sh) + +if 1 == 1 { + fmt.Println(true) +} + diff --git a/goal/transpile/math.go b/goal/transpile/math.go index f5e4164894..dc8ef3ae48 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -50,6 +50,9 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { fmt.Println("line code:", str) fmt.Println("parse err:", err) } + if len(stmts) == 0 { + return toks + } mp.stmtList(stmts) } else { ex, err := ParseExpr(str, mods) @@ -244,10 +247,13 @@ func (mp *mathParse) stmt(st ast.Stmt) { mp.addToken(token.SEMICOLON) } mp.expr(x.Cond) - if x.Body != nil { + mp.out.Add(token.IDENT, ".Bool1D(0)") // turn bool expr into actual bool + if x.Body != nil && len(x.Body.List) > 0 { mp.addToken(token.LBRACE) mp.stmtList(x.Body.List) mp.addToken(token.RBRACE) + } else { + mp.addToken(token.LBRACE) } if x.Else != nil { mp.addToken(token.ELSE) @@ -684,16 +690,22 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { } mp.callName(ex, x.Name, "") case *ast.SelectorExpr: + fun := x.Sel.Name if pkg, ok := x.X.(*ast.Ident); ok { - fun := x.Sel.Name if fw, ok := numpyFuncs[fun]; ok { - mp.callPropSelFun(ex, pkg.Name, fw) + mp.callPropSelFun(ex, x.X, fw) return } else { + // fmt.Println("call name:", fun, pkg.Name) mp.callName(ex, fun, pkg.Name) } } else { - fmt.Printf("call, weird sel: %#v\n", x.X) + if fw, ok := numpyFuncs[fun]; ok { + mp.callPropSelFun(ex, x.X, fw) + return + } + // todo: dot fun? + mp.expr(ex) } default: mp.expr(ex.Fun) @@ -705,10 +717,10 @@ func (mp *mathParse) callExpr(ex *ast.CallExpr) { } // this calls a "prop" function like ndim(a) on the object. -func (mp *mathParse) callPropFun(ex *ast.CallExpr, fw funWrap) { +func (mp *mathParse) callPropFun(cf *ast.CallExpr, fw funWrap) { ellip := fw.wrapFunc(mp) mp.idx += 2 - mp.exprList(ex.Args) // this is the tensor + mp.exprList(cf.Args) // this is the tensor mp.addToken(token.PERIOD) mp.out.Add(token.IDENT, fw.fun) if ellip { @@ -719,19 +731,23 @@ func (mp *mathParse) callPropFun(ex *ast.CallExpr, fw funWrap) { } // this calls global function through selector like: a.reshape() -func (mp *mathParse) callPropSelFun(ex *ast.CallExpr, obj string, fw funWrap) { +func (mp *mathParse) callPropSelFun(cf *ast.CallExpr, ex ast.Expr, fw funWrap) { mp.startFunc(fw.fun) - mp.addToken(token.LPAREN) // use the ( - mp.out.Add(token.IDENT, obj) + mp.out.Add(token.LPAREN) // use the ( + mp.expr(ex) mp.nextArg() // did first mp.idx += 2 - mp.addToken(token.COMMA) - mp.argsList(ex.Args) + if len(cf.Args) > 0 { + mp.addToken(token.COMMA) + mp.argsList(cf.Args) + } else { + mp.idx++ + } mp.addToken(token.RPAREN) mp.endFunc() } -func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { +func (mp *mathParse) callName(cf *ast.CallExpr, funName, pkgName string) { if fw, ok := numpyFuncs[funName]; ok { mp.startFunc(fw.fun) mp.addToken(token.LPAREN) // use the ( @@ -753,9 +769,10 @@ func (mp *mathParse) callName(ex *ast.CallExpr, funName, pkgName string) { } } if err != nil { // not a registered tensor function + // fmt.Println("regular fun", funName) mp.startFunc(funName) mp.addToken(token.LPAREN) // use the ( - mp.idx++ + mp.idx += 3 return } mp.startFunc(funName) diff --git a/goal/transpile/parser.go b/goal/transpile/parser.go index 2de3f924ed..d1ea86fd86 100644 --- a/goal/transpile/parser.go +++ b/goal/transpile/parser.go @@ -44,6 +44,9 @@ func ParseLine(code string, mode Mode) (stmts []ast.Stmt, err error) { if p.tok == token.SEMICOLON && p.lit == "\n" { p.next() } + if p.tok == token.RBRACE { + return + } p.expect(token.EOF) return @@ -419,8 +422,9 @@ func (p *parser) expectSemi() (comment *ast.CommentGroup) { } return comment default: - p.errorExpected(p.pos, "';'") - p.advance(stmtStart) + // math: allow unexpected endings.. + // p.errorExpected(p.pos, "';'") + // p.advance(stmtStart) } } return nil @@ -1504,6 +1508,9 @@ func (p *parser) parseBlockStmt() *ast.BlockStmt { } lbrace := p.expect(token.LBRACE) + if p.tok == token.EOF { // math: allow start only + return &ast.BlockStmt{Lbrace: lbrace} + } list := p.parseStmtList() rbrace := p.expect2(token.RBRACE) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 7376ca5cb3..a62717fca6 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -238,7 +238,13 @@ func TestMath(t *testing.T) { {"# b := a > 5", `b := tmath.Greater(a, tensor.NewIntScalar(5))`}, {"# b := !a", `b := tmath.Not(a)`}, {"# b := a[a > 5]", `b := tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5)))`}, + {"# b := a[a > 5].flatten()", `b := tensor.Flatten(tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5))))`}, + {"# a[:3, 2].copy()", `tensor.Clone(tensor.Reslice(a, tensor.Slice { Stop:3 } , 2))`}, + {"# a[:3, 2].reshape(4,2)", `tensor.Reshape(tensor.Reslice(a, tensor.Slice { Stop:3 } , 2), 4, 2)`}, {"# a > 5 || a < 1", `tmath.Or(tmath.Greater(a, tensor.NewIntScalar(5)), tmath.Less(a, tensor.NewIntScalar(1)))`}, + {"# if a[1,2] == 2 {", `if tmath.Equal(tensor.Reslice(a, 1, 2), tensor.NewIntScalar(2)) {`}, + {"# fmt.Println(a)", `fmt.Println(a)`}, + {"# }", `}`}, } st := NewState() diff --git a/tensor/tmath/bool.go b/tensor/tmath/bool.go index f45e104896..549450ef52 100644 --- a/tensor/tmath/bool.go +++ b/tensor/tmath/bool.go @@ -22,7 +22,7 @@ func init() { } // Equal stores in the output the bool value a == b. -func Equal(a, b tensor.Tensor) tensor.Tensor { +func Equal(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(EqualOut, a, b) } @@ -35,7 +35,7 @@ func EqualOut(a, b tensor.Tensor, out *tensor.Bool) error { } // Less stores in the output the bool value a < b. -func Less(a, b tensor.Tensor) tensor.Tensor { +func Less(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(LessOut, a, b) } @@ -48,7 +48,7 @@ func LessOut(a, b tensor.Tensor, out *tensor.Bool) error { } // Greater stores in the output the bool value a > b. -func Greater(a, b tensor.Tensor) tensor.Tensor { +func Greater(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(GreaterOut, a, b) } @@ -61,7 +61,7 @@ func GreaterOut(a, b tensor.Tensor, out *tensor.Bool) error { } // NotEqual stores in the output the bool value a != b. -func NotEqual(a, b tensor.Tensor) tensor.Tensor { +func NotEqual(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(NotEqualOut, a, b) } @@ -74,7 +74,7 @@ func NotEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { } // LessEqual stores in the output the bool value a <= b. -func LessEqual(a, b tensor.Tensor) tensor.Tensor { +func LessEqual(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(LessEqualOut, a, b) } @@ -87,7 +87,7 @@ func LessEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { } // GreaterEqual stores in the output the bool value a >= b. -func GreaterEqual(a, b tensor.Tensor) tensor.Tensor { +func GreaterEqual(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(GreaterEqualOut, a, b) } @@ -100,7 +100,7 @@ func GreaterEqualOut(a, b tensor.Tensor, out *tensor.Bool) error { } // Or stores in the output the bool value a || b. -func Or(a, b tensor.Tensor) tensor.Tensor { +func Or(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(OrOut, a, b) } @@ -110,7 +110,7 @@ func OrOut(a, b tensor.Tensor, out *tensor.Bool) error { } // And stores in the output the bool value a || b. -func And(a, b tensor.Tensor) tensor.Tensor { +func And(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(AndOut, a, b) } @@ -120,7 +120,7 @@ func AndOut(a, b tensor.Tensor, out *tensor.Bool) error { } // Not stores in the output the bool value !a. -func Not(a tensor.Tensor) tensor.Tensor { +func Not(a tensor.Tensor) *tensor.Bool { out := tensor.NewBool() errors.Log(NotOut(a, out)) return out From 865a4efe2022ef6c3153d0ad5a2430c863103a15 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 03:29:58 -0700 Subject: [PATCH 139/311] for and for range statements working in math mode --- goal/cmd/goal/testdata/test.goal | 10 +++++ goal/transpile/math.go | 73 +++++++++++++++++++++++++++++++- goal/transpile/transpile_test.go | 4 +- 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/goal/cmd/goal/testdata/test.goal b/goal/cmd/goal/testdata/test.goal index 0b7858312e..c7d0090a50 100644 --- a/goal/cmd/goal/testdata/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -21,5 +21,15 @@ if 1 == 1 { fmt.Println(true) } +for i := 0; i < 3; i++ { + fmt.Println(i) +} + +m := reshape(arange(36),6,6) + +for i, v := range m { + fmt.Println(i, v) +} + ## diff --git a/goal/transpile/math.go b/goal/transpile/math.go index dc8ef3ae48..7390c46631 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -208,8 +208,14 @@ func (mp *mathParse) stmt(st ast.Stmt) { mp.expr(x.Value) case *ast.IncDecStmt: + fn := "Inc" + if x.Tok == token.DEC { + fn = "Dec" + } + mp.startFunc("tmath." + fn) + mp.out.Add(token.LPAREN) mp.expr(x.X) - mp.addToken(x.Tok) + mp.addToken(token.RPAREN) case *ast.AssignStmt: switch x.Tok { @@ -260,8 +266,71 @@ func (mp *mathParse) stmt(st ast.Stmt) { mp.stmt(x.Else) } + case *ast.ForStmt: + mp.addToken(token.FOR) + mp.stmt(x.Init) + if x.Init != nil { + mp.addToken(token.SEMICOLON) + } + mp.expr(x.Cond) + if x.Cond != nil { + mp.out.Add(token.IDENT, ".Bool1D(0)") // turn bool expr into actual bool + mp.addToken(token.SEMICOLON) + } + mp.stmt(x.Post) + if x.Body != nil && len(x.Body.List) > 0 { + mp.addToken(token.LBRACE) + mp.stmtList(x.Body.List) + mp.addToken(token.RBRACE) + } else { + mp.addToken(token.LBRACE) + } + + case *ast.RangeStmt: + if x.Key == nil || x.Value == nil { + fmt.Println("for range statement requires both index and value variables") + return + } + knm := x.Key.(*ast.Ident).Name + vnm := x.Value.(*ast.Ident).Name + enm := x.X.(*ast.Ident).Name + + mp.addToken(token.FOR) + mp.expr(x.Key) + mp.idx += 2 + mp.addToken(token.DEFINE) + mp.out.Add(token.IDENT, "0") + mp.out.Add(token.SEMICOLON) + mp.out.Add(token.IDENT, knm) + mp.out.Add(token.IDENT, "<") + mp.out.Add(token.IDENT, enm) + mp.out.Add(token.PERIOD) + mp.out.Add(token.IDENT, "Len") + mp.idx++ + mp.out.Add(token.LPAREN) + mp.out.Add(token.RPAREN) + mp.idx++ + mp.out.Add(token.SEMICOLON) + mp.idx++ + mp.out.Add(token.IDENT, knm) + mp.out.Add(token.INC) + mp.out.Add(token.LBRACE) + + mp.out.Add(token.IDENT, vnm) + mp.out.Add(token.DEFINE) + mp.out.Add(token.IDENT, enm) + mp.out.Add(token.IDENT, ".Float1D") + mp.out.Add(token.LPAREN) + mp.out.Add(token.IDENT, knm) + mp.out.Add(token.RPAREN) + + if x.Body != nil && len(x.Body.List) > 0 { + mp.stmtList(x.Body.List) + mp.addToken(token.RBRACE) + } + // TODO - // CaseClause: SwitchStmt:, TypeSwitchStmt:, CommClause:, SelectStmt:, ForStmt:, RangeStmt: + // CaseClause: SwitchStmt:, TypeSwitchStmt:, CommClause:, SelectStmt: } } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index a62717fca6..d8663bc9ed 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -242,9 +242,11 @@ func TestMath(t *testing.T) { {"# a[:3, 2].copy()", `tensor.Clone(tensor.Reslice(a, tensor.Slice { Stop:3 } , 2))`}, {"# a[:3, 2].reshape(4,2)", `tensor.Reshape(tensor.Reslice(a, tensor.Slice { Stop:3 } , 2), 4, 2)`}, {"# a > 5 || a < 1", `tmath.Or(tmath.Greater(a, tensor.NewIntScalar(5)), tmath.Less(a, tensor.NewIntScalar(1)))`}, - {"# if a[1,2] == 2 {", `if tmath.Equal(tensor.Reslice(a, 1, 2), tensor.NewIntScalar(2)) {`}, {"# fmt.Println(a)", `fmt.Println(a)`}, {"# }", `}`}, + {"# if a[1,2] == 2 {", `if tmath.Equal(tensor.Reslice(a, 1, 2), tensor.NewIntScalar(2)).Bool1D(0) {`}, + {"# for i := 0; i < 3; i++ {", `for i := tensor.NewIntScalar(0); tmath.Less(i, tensor.NewIntScalar(3)).Bool1D(0); tmath.Inc(i) {`}, + {"# for i, v := range a {", `for i := 0; i < a.Len(); i++ { v := a .Float1D(i)`}, } st := NewState() From 437cd0ceb646eb3c9cfc72a42754a6b9581d81b2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 03:35:18 -0700 Subject: [PATCH 140/311] update test.goal --- goal/cmd/goal/testdata/test.goal | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/goal/cmd/goal/testdata/test.goal b/goal/cmd/goal/testdata/test.goal index c7d0090a50..f4c9b31248 100644 --- a/goal/cmd/goal/testdata/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -17,7 +17,9 @@ c := x * y + a * b fmt.Println(c) -if 1 == 1 { +m := reshape(arange(36),6,6) + +if m[1,1] == 7 { fmt.Println(true) } @@ -25,8 +27,6 @@ for i := 0; i < 3; i++ { fmt.Println(i) } -m := reshape(arange(36),6,6) - for i, v := range m { fmt.Println(i, v) } From 9efcc50374d39093a1d3108b029e1ca633fef758 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 10:26:04 -0700 Subject: [PATCH 141/311] table updates --- goal/README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/goal/README.md b/goal/README.md index 52fb74d45c..6be697f477 100644 --- a/goal/README.md +++ b/goal/README.md @@ -313,15 +313,16 @@ See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.htm ## Boolean tensors and indexing -See [NumPy boolean indexing](https://numpy.org/doc/stable/user/basics.indexing.html#boolean-array-indexing) +See [NumPy boolean indexing](https://numpy.org/doc/stable/user/basics.indexing.html#boolean-array-indexing). + +Note that Goal only supports boolean logical operators (`&&` and `||`) on boolean tensors, not the single bitwise operators `&` and `|`. | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | | `tmath.Greater(a, 0.5)` | same: | `(a > 0.5)` | `(a > 0.5)` | `bool` tensor of shape `a` with elements `(v > 0.5)` | -| `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | -| `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | -| `a & b` | `a & b` | `bitand(a,b)` | element bitwise AND operator on `bool` or `int` tensors | -| `a \| b` | `a \| b` | `bitor(a,b)` | element bitwise OR operator on `bool` or `int` tensors | +| `tmath.And(a, b)` | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | +| `tmath.Or(a, b)` | `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | +| `tmath.Negate(a)` | `!a` | ? | ? | element-wise negation on `bool` tensors | | `tensor.Mask(a,` `tmath.Less(a, 0.5)` | same: |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | | `tensor.Flatten(` `tensor.Mask(a,` `tmath.Less(a, 0.5)))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 (as a copy, not a view) | | | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | @@ -348,10 +349,10 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| | |`a * b` | `a .* b` | element-wise multiply | -| | |`a/b` | `a./b` | element-wise divide | -| `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | -| `cos(a)` | `cos(a)` | `cos(a)` | element-wise function application | +| `tmath.Mul(a,b)` | same: |`a * b` | `a .* b` | element-wise multiply | +| `tmath.Div(a,b)` | same: |`a/b` | `a./b` | element-wise divide | +| `tmath.Pow(a,3)` | same: | `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | +| `tmath.Cos(a)` | same: | `cos(a)` | `cos(a)` | element-wise function application | ## 2D Matrix Linear Algebra @@ -399,7 +400,6 @@ todo: huge amount of work needed to support complex numbers throughout! ## TODO -* basic go expressions in math mode -- see how that works * # # surrounding in go, shell modes * more creation routines * update readme table From e22364edf4af19ed607bbfea70f93da3ae2085c4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 11:54:58 -0700 Subject: [PATCH 142/311] major improvement: goal uses yaegicore symbols to register functions instead of requiring manual AddFunc calls! --- goal/README.md | 10 +++- goal/transpile/addfuncs.go | 38 ++++++++++++ goal/transpile/math.go | 19 +----- tensor/README.md | 6 +- tensor/align.go | 9 --- tensor/convert.go | 105 -------------------------------- tensor/create.go | 110 ++++++++++++++++++++++++++++++++++ tensor/funcs.go | 2 +- tensor/stats/metric/matrix.go | 9 --- tensor/stats/metric/misc.go | 4 -- tensor/stats/stats/group.go | 7 --- tensor/stats/stats/norm.go | 7 --- tensor/tmath/bool.go | 12 ---- tensor/tmath/math.go | 40 ------------- tensor/tmath/ops.go | 20 ------- 15 files changed, 163 insertions(+), 235 deletions(-) create mode 100644 goal/transpile/addfuncs.go create mode 100644 tensor/create.go diff --git a/goal/README.md b/goal/README.md index 6be697f477..85acd894e2 100644 --- a/goal/README.md +++ b/goal/README.md @@ -241,6 +241,8 @@ The _view_ versions of `Tensor` include `Sliced`, `Reshaped`, `Masked`, `Indexe The following sections provide a full list of equivalents between the `tensor` Go code, Goal, NumPy, and MATLAB, based on the table in [numpy-for-matlab-users](https://numpy.org/doc/stable/user/numpy-for-matlab-users.html). * The _same:_ in Goal means that the same NumPy syntax works in Goal, minus the `np.` prefix, and likewise for _or:_ (where Goal also has additional syntax). +* In the `tensor.Go` code, we sometimes just write a scalar number for simplicity, but these are actually `tensor.NewFloat64Scalar` etc. +* Goal also has support for `string` tensors, e.g., for labels, and operators such as addition that make sense for strings are supported. Otherwise, strings are automatically converted to numbers using the `tensor.Float` interface. If you have any doubt about whether you've got a `tensor.Float64` when you expect one, use `tensor.AsFloat64Tensor` which makes sure. ## Tensor shape @@ -323,14 +325,16 @@ Note that Goal only supports boolean logical operators (`&&` and `||`) on boolea | `tmath.And(a, b)` | `a && b` | `logical_and(a,b)` | `a & b` | element-wise AND operator on `bool` tensors | | `tmath.Or(a, b)` | `a \|\| b` | `np.logical_or(a,b)` | `a \| b` | element-wise OR operator on `bool` tensors | | `tmath.Negate(a)` | `!a` | ? | ? | element-wise negation on `bool` tensors | -| `tensor.Mask(a,` `tmath.Less(a, 0.5)` | same: |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | +| `tmath.Assign(` `tensor.Mask(a,` `tmath.Less(a, 0.5),` `0)` | same: |`a[a < 0.5]=0` | `a(a<0.5)=0` | `a` with elements less than 0.5 zeroed out | | `tensor.Flatten(` `tensor.Mask(a,` `tmath.Less(a, 0.5)))` | same: |`a[a < 0.5].flatten()` | ? | a 1D list of the elements of `a` < 0.5 (as a copy, not a view) | -| | |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | +| `tensor.Mul(a,` `tmath.Greater(a, 0.5))` | same: |`a * (a > 0.5)` | `a .* (a>0.5)` | `a` with elements less than 0.5 zeroed out | ## Advanced index-based indexing See [NumPy integer indexing](https://numpy.org/doc/stable/user/basics.indexing.html#integer-array-indexing). Note that the current NumPy version of indexed is rather complex and difficult for many people to understand, as articulated in this [NEP 21 proposal](https://numpy.org/neps/nep-0021-advanced-indexing.html). +**TODO:** not yet implemented: + | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | | | |`a[np.ix_([1, 3, 4], [0, 2])]` | `a([2,4,5],[1,3])` | rows 2,4 and 5 and columns 1 and 3. | @@ -349,8 +353,10 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | +| `tmath.Add(a,b)` | same: |`a + b` | `a .+ b` | element-wise addition; Goal does this string-wise for string tensors | | `tmath.Mul(a,b)` | same: |`a * b` | `a .* b` | element-wise multiply | | `tmath.Div(a,b)` | same: |`a/b` | `a./b` | element-wise divide | +| `tmath.Mod(a,b)` | same: |`a%b` | `a./b` | element-wise modulous (works for float and int) | | `tmath.Pow(a,3)` | same: | `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | | `tmath.Cos(a)` | same: | `cos(a)` | `cos(a)` | element-wise function application | diff --git a/goal/transpile/addfuncs.go b/goal/transpile/addfuncs.go new file mode 100644 index 0000000000..ef64a0d49b --- /dev/null +++ b/goal/transpile/addfuncs.go @@ -0,0 +1,38 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package transpile + +import ( + "path" + "reflect" + "strings" + + "cogentcore.org/core/tensor" + "cogentcore.org/core/yaegicore/symbols" +) + +func init() { + AddYaegiTensorFuncs() +} + +// AddYaegiTensorFuncs grabs all tensor* package functions registered +// in yaegicore and adds them to the `tensor.Funcs` map so we can +// properly convert symbols to either tensors or basic literals, +// depending on the arg types for the current function. +func AddYaegiTensorFuncs() { + for pth, symap := range symbols.Symbols { + if !strings.Contains(pth, "/core/tensor/") { + continue + } + _, pkg := path.Split(pth) + for name, val := range symap { + if val.Kind() != reflect.Func { + continue + } + pnm := pkg + "." + name + tensor.AddFunc(pnm, val.Interface()) + } + } +} diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 7390c46631..4b62d9079d 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -14,19 +14,6 @@ import ( "cogentcore.org/core/tensor" ) -func init() { - tensor.AddFunc("tensor.Reshape", tensor.Reshape) - tensor.AddFunc("tensor.Reslice", tensor.Reslice) - tensor.AddFunc("tensor.Mask", tensor.Mask) - tensor.AddFunc("tensor.NewFloat64", tensor.NewFloat64) - tensor.AddFunc("tensor.NewIntScalar", tensor.NewIntScalar) - tensor.AddFunc("tensor.NewFloat64Scalar", tensor.NewFloat64Scalar) - tensor.AddFunc("tensor.NewStringScalar", tensor.NewStringScalar) - tensor.AddFunc("tensor.NewIntFromValues", tensor.NewIntFromValues) - tensor.AddFunc("tensor.NewFloat64FromValues", tensor.NewFloat64FromValues) - tensor.AddFunc("tensor.NewStringFromValues", tensor.NewStringFromValues) -} - func MathParse(toks Tokens, code string, fullLine bool) Tokens { nt := len(toks) if nt == 0 { @@ -804,9 +791,9 @@ func (mp *mathParse) callPropSelFun(cf *ast.CallExpr, ex ast.Expr, fw funWrap) { mp.startFunc(fw.fun) mp.out.Add(token.LPAREN) // use the ( mp.expr(ex) - mp.nextArg() // did first mp.idx += 2 if len(cf.Args) > 0 { + mp.nextArg() // did first mp.addToken(token.COMMA) mp.argsList(cf.Args) } else { @@ -828,10 +815,10 @@ func (mp *mathParse) callName(cf *ast.CallExpr, funName, pkgName string) { funName = pkgName + "." + funName _, err = tensor.FuncByName(funName) } else { // non-package qualified names are _only_ in tmath! can be lowercase - _, err = tensor.FuncByName(funName) + _, err = tensor.FuncByName("tmath." + funName) if err != nil { funName = strings.ToUpper(funName[:1]) + funName[1:] // first letter uppercased - _, err = tensor.FuncByName(funName) + _, err = tensor.FuncByName("tmath." + funName) } if err == nil { // registered, must be in tmath funName = "tmath." + funName diff --git a/tensor/README.md b/tensor/README.md index 7a0a8a7eb7..f61593a8cc 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -26,12 +26,12 @@ The `float64` ("Float"), `int` ("Int"), and `string` ("String") types are used a There is also a `RowMajor` sub-interface for tensors (implemented by the `Values` and `Rows` types), which supports `[Set]FloatRow[Cell]` methods that provide optimized access to row major data. See [Standard shapes](#standard-shapes) for more info. -The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets an index and a list of tensors, and a varargs list of tensors. In general it is completely up to the compute function how to interpret the index, although we also support the "broadcasting" principles from NumPy for binary functions operating on two tensors, as discussed below. There is a Threaded version of this for parallelizable functions, and a GPU version in the [gosl](../gpu/gosl) Go-as-a-shading-language package. - -All tensor package functions are registered using a global name-to-function map (`Funcs`), which can be used to retrieve the function by name, along with relevant metadata used by [goal](../goal). This allows functions to be called via enums, in the `stats` and `metrics` packages, for example. These functions are the equivalent of the [ufunc](https://numpy.org/doc/stable/user/basics.ufuncs.html) universal functions in NumPy. +The `Vectorize` function and its variants provide a universal "apply function to tensor data" mechanism (often called a "map" function, but that name is already taken in Go). It takes an `N` function that determines how many indexes to iterate over (and this function can also do any initialization prior to iterating), a compute function that gets the current index value, and a varargs list of tensors. In general it is completely up to the compute function how to interpret the index, although we also support the "broadcasting" principles from NumPy for binary functions operating on two tensors, as discussed below. There is a Threaded version of this for parallelizable functions, and a GPU version in the [gosl](../gpu/gosl) Go-as-a-shading-language package. To support the best possible performance in compute-intensive code, we have written all the core tensor functions in an `Out` suffixed version that takes the output tensor as an additional input argument (it must be a `Values` type), which allows an appropriately sized tensor to be used to hold the outputs on repeated function calls, instead of requiring new memory allocations every time. These versions are used in other calls where appropriate. The function without the `Out` suffix just wraps the `Out` version, and is what is called directly by Goal, where the output return value is essential for proper chaining of operations. +To support proper argument handling for tensor functions, the [goal](../goal) transpiler registers all tensor package functions into the global name-to-function map (`tensor.Funcs`), which is used to retrieve the function by name, along with relevant arg metadata. This registry is also key for enum sets of functions, in the `stats` and `metrics` packages, for example, to be able to call the corresponding function. Goal uses symbols collected in the [yaegicore](../yaegicore) package to populate the Funcs, but enums should directly add themselves to ensure they are always available even outside of Goal. + * [table](table) organizes multiple Tensors as columns in a data `Table`, aligned by a common outer row dimension. Because the columns are tensors, each cell (value associated with a given row) can also be n-dimensional, allowing efficient representation of patterns and other high-dimensional data. Furthermore, the entire column is organized as a single contiguous slice of data, so it can be efficiently processed. A `Table` automatically supplies a shared list of row Indexes for its `Indexed` columns, efficiently allowing all the heterogeneous data columns to be sorted and filtered together. Data that is encoded as a slice of `struct`s can be bidirectionally converted to / from a Table, which then provides more powerful sorting, filtering and other functionality, including [plot/plotcore](../plot/plotcore). diff --git a/tensor/align.go b/tensor/align.go index 3318f27328..22cddaaa29 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -11,15 +11,6 @@ import ( "cogentcore.org/core/base/errors" ) -func init() { - AddFunc("tensor.FloatAssignFunc", FloatAssignFunc) - AddFunc("tensor.StringAssignFunc", StringAssignFunc) - AddFunc("tensor.FloatBinaryFunc", FloatBinaryFunc) - AddFunc("tensor.StringBinaryFunc", StringBinaryFunc) - AddFunc("tensor.BoolStringsFunc", BoolStringsFunc) - AddFunc("tensor.BoolFloatsFunc", BoolFloatsFunc) -} - // AlignShapes aligns the shapes of two tensors, a and b for a binary // computation producing an output, returning the effective aligned shapes // for a, b, and the output, all with the same number of dimensions. diff --git a/tensor/convert.go b/tensor/convert.go index c63e7da7e8..2439f0a74a 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -105,111 +105,6 @@ func SetShapeFrom(vals Values, from Tensor) { vals.SetShapeSizes(from.ShapeSizes()...) } -// NewFloat64Scalar is a convenience method for a Tensor -// representation of a single float64 scalar value. -func NewFloat64Scalar(val float64) *Float64 { - return NewNumberFromValues(val) -} - -// NewIntScalar is a convenience method for a Tensor -// representation of a single int scalar value. -func NewIntScalar(val int) *Int { - return NewNumberFromValues(val) -} - -// NewStringScalar is a convenience method for a Tensor -// representation of a single string scalar value. -func NewStringScalar(val string) *String { - return NewStringFromValues(val) -} - -// NewFloat64FromValues returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewFloat64FromValues(vals ...float64) *Float64 { - return NewNumberFromValues(vals...) -} - -// NewIntFromValues returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewIntFromValues(vals ...int) *Int { - return NewNumberFromValues(vals...) -} - -// NewStringFromValues returns a new 1-dimensional tensor of given value type -// initialized directly from the given slice values, which are not copied. -// The resulting Tensor thus "wraps" the given values. -func NewStringFromValues(vals ...string) *String { - n := len(vals) - tsr := &String{} - tsr.Values = vals - tsr.SetShapeSizes(n) - return tsr -} - -// SetAllFloat64 sets all values of given tensor to given value. -func SetAllFloat64(tsr Tensor, val float64) { - VectorizeThreaded(1, func(tsr ...Tensor) int { - return tsr[0].Len() - }, - func(idx int, tsr ...Tensor) { - tsr[0].SetFloat1D(val, idx) - }, tsr) -} - -// SetAllInt sets all values of given tensor to given value. -func SetAllInt(tsr Tensor, val int) { - VectorizeThreaded(1, func(tsr ...Tensor) int { - return tsr[0].Len() - }, - func(idx int, tsr ...Tensor) { - tsr[0].SetInt1D(val, idx) - }, tsr) -} - -// SetAllString sets all values of given tensor to given value. -func SetAllString(tsr Tensor, val string) { - VectorizeThreaded(1, func(tsr ...Tensor) int { - return tsr[0].Len() - }, - func(idx int, tsr ...Tensor) { - tsr[0].SetString1D(val, idx) - }, tsr) -} - -// NewFloat64Full returns a new tensor full of given scalar value, -// of given shape sizes. -func NewFloat64Full(val float64, sizes ...int) *Float64 { - tsr := NewFloat64(sizes...) - SetAllFloat64(tsr, val) - return tsr -} - -// NewFloat64Ones returns a new tensor full of 1s, -// of given shape sizes. -func NewFloat64Ones(sizes ...int) *Float64 { - tsr := NewFloat64(sizes...) - SetAllFloat64(tsr, 1.0) - return tsr -} - -// NewIntFull returns a new tensor full of given scalar value, -// of given shape sizes. -func NewIntFull(val int, sizes ...int) *Int { - tsr := NewInt(sizes...) - SetAllInt(tsr, val) - return tsr -} - -// NewStringFull returns a new tensor full of given scalar value, -// of given shape sizes. -func NewStringFull(val string, sizes ...int) *String { - tsr := NewString(sizes...) - SetAllString(tsr, val) - return tsr -} - // AsFloat64Scalar returns the first value of tensor as a float64 scalar. // Returns 0 if no values. func AsFloat64Scalar(tsr Tensor) float64 { diff --git a/tensor/create.go b/tensor/create.go new file mode 100644 index 0000000000..68367b4d31 --- /dev/null +++ b/tensor/create.go @@ -0,0 +1,110 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +// NewFloat64Scalar is a convenience method for a Tensor +// representation of a single float64 scalar value. +func NewFloat64Scalar(val float64) *Float64 { + return NewNumberFromValues(val) +} + +// NewIntScalar is a convenience method for a Tensor +// representation of a single int scalar value. +func NewIntScalar(val int) *Int { + return NewNumberFromValues(val) +} + +// NewStringScalar is a convenience method for a Tensor +// representation of a single string scalar value. +func NewStringScalar(val string) *String { + return NewStringFromValues(val) +} + +// NewFloat64FromValues returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewFloat64FromValues(vals ...float64) *Float64 { + return NewNumberFromValues(vals...) +} + +// NewIntFromValues returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewIntFromValues(vals ...int) *Int { + return NewNumberFromValues(vals...) +} + +// NewStringFromValues returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewStringFromValues(vals ...string) *String { + n := len(vals) + tsr := &String{} + tsr.Values = vals + tsr.SetShapeSizes(n) + return tsr +} + +// SetAllFloat64 sets all values of given tensor to given value. +func SetAllFloat64(tsr Tensor, val float64) { + VectorizeThreaded(1, func(tsr ...Tensor) int { + return tsr[0].Len() + }, + func(idx int, tsr ...Tensor) { + tsr[0].SetFloat1D(val, idx) + }, tsr) +} + +// SetAllInt sets all values of given tensor to given value. +func SetAllInt(tsr Tensor, val int) { + VectorizeThreaded(1, func(tsr ...Tensor) int { + return tsr[0].Len() + }, + func(idx int, tsr ...Tensor) { + tsr[0].SetInt1D(val, idx) + }, tsr) +} + +// SetAllString sets all values of given tensor to given value. +func SetAllString(tsr Tensor, val string) { + VectorizeThreaded(1, func(tsr ...Tensor) int { + return tsr[0].Len() + }, + func(idx int, tsr ...Tensor) { + tsr[0].SetString1D(val, idx) + }, tsr) +} + +// NewFloat64Full returns a new tensor full of given scalar value, +// of given shape sizes. +func NewFloat64Full(val float64, sizes ...int) *Float64 { + tsr := NewFloat64(sizes...) + SetAllFloat64(tsr, val) + return tsr +} + +// NewFloat64Ones returns a new tensor full of 1s, +// of given shape sizes. +func NewFloat64Ones(sizes ...int) *Float64 { + tsr := NewFloat64(sizes...) + SetAllFloat64(tsr, 1.0) + return tsr +} + +// NewIntFull returns a new tensor full of given scalar value, +// of given shape sizes. +func NewIntFull(val int, sizes ...int) *Int { + tsr := NewInt(sizes...) + SetAllInt(tsr, val) + return tsr +} + +// NewStringFull returns a new tensor full of given scalar value, +// of given shape sizes. +func NewStringFull(val string, sizes ...int) *String { + tsr := NewString(sizes...) + SetAllString(tsr, val) + return tsr +} diff --git a/tensor/funcs.go b/tensor/funcs.go index 94de1f54ca..3452243d95 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -126,7 +126,7 @@ func AddFunc(name string, fun any) error { } _, ok := Funcs[name] if ok { - return errors.Log(fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name)) + return fmt.Errorf("tensor.AddFunc: function of name %q already exists, not added", name) } fn, err := NewFunc(name, fun) if errors.Log(err) != nil { diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 3cec07ab02..fc60ccf461 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -14,15 +14,6 @@ import ( "gonum.org/v1/gonum/mat" ) -func init() { - tensor.AddFunc("metric.Matrix", Matrix) - tensor.AddFunc("metric.CrossMatrix", CrossMatrix) - tensor.AddFunc("metric.CovarianceMatrix", CovarianceMatrix) - tensor.AddFunc("metric.PCA", PCA) - tensor.AddFunc("metric.SVD", SVD) - tensor.AddFunc("metric.ProjectOnMatrixColumn", ProjectOnMatrixColumn) -} - // MatrixOut computes the rows x rows square distance / similarity matrix // between the patterns for each row of the given higher dimensional input tensor, // which must have at least 2 dimensions: the outermost rows, diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 3a559dc345..212373eeb3 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -10,10 +10,6 @@ import ( "cogentcore.org/core/tensor" ) -func init() { - tensor.AddFunc("metric.ClosestRow", ClosestRow) -} - // ClosestRow returns the closest fit between probe pattern and patterns in // a "vocabulary" tensor with outermost row dimension, using given metric // function, which must fit the MetricFunc signature. diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index d1ad8e49d5..1ad3c08349 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -12,13 +12,6 @@ import ( "cogentcore.org/core/tensor/table" ) -// note: we cannot register these functions because they take vararg tensors!! -// -// func init() { -// tensor.AddFunc("stats.Groups", Groups, 0, tensor.AnyFirstArg) -// tensor.AddFunc("stats.GroupAll", GroupAll, 0, tensor.AnyFirstArg) -// } - // Groups generates indexes for each unique value in each of the given tensors. // One can then use the resulting indexes for the [tensor.Rows] indexes to // perform computations restricted to grouped subsets of data, as in the diff --git a/tensor/stats/stats/norm.go b/tensor/stats/stats/norm.go index 1146ebef8d..a112e3acae 100644 --- a/tensor/stats/stats/norm.go +++ b/tensor/stats/stats/norm.go @@ -10,13 +10,6 @@ import ( "cogentcore.org/core/tensor/tmath" ) -func init() { - tensor.AddFunc("stats.ZScore", ZScore) - tensor.AddFunc("stats.UnitNorm", UnitNorm) - tensor.AddFunc("stats.Clamp", Clamp) - tensor.AddFunc("stats.Binarize", Binarize) -} - // ZScore computes Z-normalized values into given output tensor, // subtracting the Mean and dividing by the standard deviation. func ZScore(a tensor.Tensor) tensor.Values { diff --git a/tensor/tmath/bool.go b/tensor/tmath/bool.go index 549450ef52..1e80f49013 100644 --- a/tensor/tmath/bool.go +++ b/tensor/tmath/bool.go @@ -9,18 +9,6 @@ import ( "cogentcore.org/core/tensor" ) -func init() { - tensor.AddFunc("tmath.Equal", Equal) - tensor.AddFunc("tmath.Less", Less) - tensor.AddFunc("tmath.Greater", Greater) - tensor.AddFunc("tmath.NotEqual", NotEqual) - tensor.AddFunc("tmath.LessEqual", LessEqual) - tensor.AddFunc("tmath.GreaterEqual", GreaterEqual) - tensor.AddFunc("tmath.Or", Or) - tensor.AddFunc("tmath.And", And) - tensor.AddFunc("tmath.Not", Not) -} - // Equal stores in the output the bool value a == b. func Equal(a, b tensor.Tensor) *tensor.Bool { return tensor.CallOut2Bool(EqualOut, a, b) diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index c918b1243e..bb86430ccc 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -10,46 +10,6 @@ import ( "cogentcore.org/core/tensor" ) -func init() { - tensor.AddFunc("Abs", Abs) - tensor.AddFunc("Acos", Acos) - tensor.AddFunc("Acosh", Acosh) - tensor.AddFunc("Asin", Asin) - tensor.AddFunc("Asinh", Asinh) - tensor.AddFunc("Atan", Atan) - tensor.AddFunc("Atanh", Atanh) - tensor.AddFunc("Cbrt", Cbrt) - tensor.AddFunc("Ceil", Ceil) - tensor.AddFunc("Cos", Cos) - tensor.AddFunc("Cosh", Cosh) - tensor.AddFunc("Erf", Erf) - tensor.AddFunc("Erfc", Erfc) - tensor.AddFunc("Erfcinv", Erfcinv) - tensor.AddFunc("Erfinv", Erfinv) - tensor.AddFunc("Exp", Exp) - tensor.AddFunc("Exp2", Exp2) - tensor.AddFunc("Expm1", Expm1) - tensor.AddFunc("Floor", Floor) - tensor.AddFunc("Gamma", Gamma) - tensor.AddFunc("J0", J0) - tensor.AddFunc("J1", J1) - tensor.AddFunc("Log", Log) - tensor.AddFunc("Log10", Log10) - tensor.AddFunc("Log1p", Log1p) - tensor.AddFunc("Log2", Log2) - tensor.AddFunc("Logb", Logb) - tensor.AddFunc("Round", Round) - tensor.AddFunc("RoundToEven", RoundToEven) - tensor.AddFunc("Sin", Sin) - tensor.AddFunc("Sinh", Sinh) - tensor.AddFunc("Sqrt", Sqrt) - tensor.AddFunc("Tan", Tan) - tensor.AddFunc("Tanh", Tanh) - tensor.AddFunc("Trunc", Trunc) - tensor.AddFunc("Y0", Y0) - tensor.AddFunc("Y1", Y1) -} - func Abs(in tensor.Tensor) tensor.Values { return tensor.CallOut1(AbsOut, in) } diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 3c48292f43..92411e258c 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -10,26 +10,6 @@ import ( "cogentcore.org/core/tensor" ) -func init() { - tensor.AddFunc("tmath.Assign", Assign) - tensor.AddFunc("tmath.AddAssign", AddAssign) - tensor.AddFunc("tmath.SubAssign", SubAssign) - tensor.AddFunc("tmath.MulAssign", MulAssign) - tensor.AddFunc("tmath.DivAssign", DivAssign) - tensor.AddFunc("tmath.ModAssign", ModAssign) - - tensor.AddFunc("tmath.Inc", Inc) - tensor.AddFunc("tmath.Dec", Dec) - - tensor.AddFunc("tmath.Add", Add) - tensor.AddFunc("tmath.Sub", Sub) - tensor.AddFunc("tmath.Mul", Mul) - tensor.AddFunc("tmath.Div", Div) - tensor.AddFunc("tmath.Mod", Mod) - - tensor.AddFunc("tmath.Negate", Negate) -} - // Assign assigns values from b into a. func Assign(a, b tensor.Tensor) error { return tensor.FloatAssignFunc(func(a, b float64) float64 { return b }, a, b) From feeaf91f959e5648f4bce976822c05be906238b0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 12:52:06 -0700 Subject: [PATCH 143/311] tensor: use FloatFunc for math funcs, add binary calls --- tensor/align.go | 28 +++- tensor/tmath/math.go | 306 ++++++++++++++------------------------ tensor/tmath/math_test.go | 46 ++++++ tensor/tmath/ops.go | 10 +- tensor/vectorize.go | 8 +- 5 files changed, 192 insertions(+), 206 deletions(-) diff --git a/tensor/align.go b/tensor/align.go index 22cddaaa29..f04ad96bc6 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -144,12 +144,15 @@ func StringAssignFunc(fun func(a, b string) string, a, b Tensor) error { } // FloatBinaryFunc sets output to a binary function of a, b float64 values. -func FloatBinaryFunc(fun func(a, b float64) float64, a, b Tensor) Tensor { - return CallOut2Gen1(FloatBinaryFuncOut, fun, a, b) +// The flops (floating point operations) estimate is used to control parallel +// threading using goroutines, and should reflect number of flops in the function. +// See [VectorizeThreaded] for more information. +func FloatBinaryFunc(flops int, fun func(a, b float64) float64, a, b Tensor) Tensor { + return CallOut2Gen2(FloatBinaryFuncOut, flops, fun, a, b) } // FloatBinaryFuncOut sets output to a binary function of a, b float64 values. -func FloatBinaryFuncOut(fun func(a, b float64) float64, a, b Tensor, out Values) error { +func FloatBinaryFuncOut(flops int, fun func(a, b float64) float64, a, b Tensor, out Values) error { as, bs, os, err := AlignShapes(a, b) if err != nil { return err @@ -189,6 +192,25 @@ func StringBinaryFuncOut(fun func(a, b string) string, a, b Tensor, out Values) return nil } +// FloatFunc sets output to a function of tensor float64 values. +// The flops (floating point operations) estimate is used to control parallel +// threading using goroutines, and should reflect number of flops in the function. +// See [VectorizeThreaded] for more information. +func FloatFunc(flops int, fun func(in float64) float64, in Tensor) Values { + return CallOut1Gen2(FloatFuncOut, flops, fun, in) +} + +// FloatFuncOut sets output to a function of tensor float64 values. +func FloatFuncOut(flops int, fun func(in float64) float64, in Tensor, out Values) error { + SetShapeFrom(out, in) + n := in.Len() + VectorizeThreaded(flops, func(tsr ...Tensor) int { return n }, + func(idx int, tsr ...Tensor) { + tsr[1].SetFloat1D(fun(tsr[0].Float1D(idx)), idx) + }, in, out) + return nil +} + ////////////////////////// Bool // BoolStringsFunc sets boolean output value based on a function involving diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index bb86430ccc..2b1ef9dbea 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -15,11 +15,7 @@ func Abs(in tensor.Tensor) tensor.Values { } func AbsOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Abs(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Abs(a) }, in, out) } func Acos(in tensor.Tensor) tensor.Values { @@ -27,11 +23,7 @@ func Acos(in tensor.Tensor) tensor.Values { } func AcosOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Acos(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Acos(a) }, in, out) } func Acosh(in tensor.Tensor) tensor.Values { @@ -39,11 +31,7 @@ func Acosh(in tensor.Tensor) tensor.Values { } func AcoshOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Acosh(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Acosh(a) }, in, out) } func Asin(in tensor.Tensor) tensor.Values { @@ -51,11 +39,7 @@ func Asin(in tensor.Tensor) tensor.Values { } func AsinOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Asin(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Asin(a) }, in, out) } func Asinh(in tensor.Tensor) tensor.Values { @@ -63,11 +47,7 @@ func Asinh(in tensor.Tensor) tensor.Values { } func AsinhOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Asinh(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Asinh(a) }, in, out) } func Atan(in tensor.Tensor) tensor.Values { @@ -75,11 +55,7 @@ func Atan(in tensor.Tensor) tensor.Values { } func AtanOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Atan(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Atan(a) }, in, out) } func Atanh(in tensor.Tensor) tensor.Values { @@ -87,11 +63,7 @@ func Atanh(in tensor.Tensor) tensor.Values { } func AtanhOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Atanh(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Atanh(a) }, in, out) } func Cbrt(in tensor.Tensor) tensor.Values { @@ -99,11 +71,7 @@ func Cbrt(in tensor.Tensor) tensor.Values { } func CbrtOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Cbrt(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Cbrt(a) }, in, out) } func Ceil(in tensor.Tensor) tensor.Values { @@ -111,11 +79,7 @@ func Ceil(in tensor.Tensor) tensor.Values { } func CeilOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Ceil(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Ceil(a) }, in, out) } func Cos(in tensor.Tensor) tensor.Values { @@ -123,11 +87,7 @@ func Cos(in tensor.Tensor) tensor.Values { } func CosOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Cos(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Cos(a) }, in, out) } func Cosh(in tensor.Tensor) tensor.Values { @@ -135,11 +95,7 @@ func Cosh(in tensor.Tensor) tensor.Values { } func CoshOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Cosh(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Cosh(a) }, in, out) } func Erf(in tensor.Tensor) tensor.Values { @@ -147,11 +103,7 @@ func Erf(in tensor.Tensor) tensor.Values { } func ErfOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Erf(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Erf(a) }, in, out) } func Erfc(in tensor.Tensor) tensor.Values { @@ -159,11 +111,7 @@ func Erfc(in tensor.Tensor) tensor.Values { } func ErfcOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Erfc(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Erfc(a) }, in, out) } func Erfcinv(in tensor.Tensor) tensor.Values { @@ -171,11 +119,7 @@ func Erfcinv(in tensor.Tensor) tensor.Values { } func ErfcinvOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Erfcinv(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Erfcinv(a) }, in, out) } func Erfinv(in tensor.Tensor) tensor.Values { @@ -183,11 +127,7 @@ func Erfinv(in tensor.Tensor) tensor.Values { } func ErfinvOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Erfinv(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Erfinv(a) }, in, out) } func Exp(in tensor.Tensor) tensor.Values { @@ -195,11 +135,7 @@ func Exp(in tensor.Tensor) tensor.Values { } func ExpOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Exp(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Exp(a) }, in, out) } func Exp2(in tensor.Tensor) tensor.Values { @@ -207,11 +143,7 @@ func Exp2(in tensor.Tensor) tensor.Values { } func Exp2Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Exp2(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Exp2(a) }, in, out) } func Expm1(in tensor.Tensor) tensor.Values { @@ -219,11 +151,7 @@ func Expm1(in tensor.Tensor) tensor.Values { } func Expm1Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Expm1(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Expm1(a) }, in, out) } func Floor(in tensor.Tensor) tensor.Values { @@ -231,11 +159,7 @@ func Floor(in tensor.Tensor) tensor.Values { } func FloorOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Floor(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Floor(a) }, in, out) } func Gamma(in tensor.Tensor) tensor.Values { @@ -243,11 +167,7 @@ func Gamma(in tensor.Tensor) tensor.Values { } func GammaOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Gamma(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Gamma(a) }, in, out) } func J0(in tensor.Tensor) tensor.Values { @@ -255,11 +175,7 @@ func J0(in tensor.Tensor) tensor.Values { } func J0Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.J0(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.J0(a) }, in, out) } func J1(in tensor.Tensor) tensor.Values { @@ -267,11 +183,7 @@ func J1(in tensor.Tensor) tensor.Values { } func J1Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.J1(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.J1(a) }, in, out) } func Log(in tensor.Tensor) tensor.Values { @@ -279,11 +191,7 @@ func Log(in tensor.Tensor) tensor.Values { } func LogOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Log(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Log(a) }, in, out) } func Log10(in tensor.Tensor) tensor.Values { @@ -291,11 +199,7 @@ func Log10(in tensor.Tensor) tensor.Values { } func Log10Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Log10(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Log10(a) }, in, out) } func Log1p(in tensor.Tensor) tensor.Values { @@ -303,11 +207,7 @@ func Log1p(in tensor.Tensor) tensor.Values { } func Log1pOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Log1p(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Log1p(a) }, in, out) } func Log2(in tensor.Tensor) tensor.Values { @@ -315,11 +215,7 @@ func Log2(in tensor.Tensor) tensor.Values { } func Log2Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Log2(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Log2(a) }, in, out) } func Logb(in tensor.Tensor) tensor.Values { @@ -327,11 +223,7 @@ func Logb(in tensor.Tensor) tensor.Values { } func LogbOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Logb(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Logb(a) }, in, out) } func Round(in tensor.Tensor) tensor.Values { @@ -339,11 +231,7 @@ func Round(in tensor.Tensor) tensor.Values { } func RoundOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Round(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Round(a) }, in, out) } func RoundToEven(in tensor.Tensor) tensor.Values { @@ -351,11 +239,7 @@ func RoundToEven(in tensor.Tensor) tensor.Values { } func RoundToEvenOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.RoundToEven(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.RoundToEven(a) }, in, out) } func Sin(in tensor.Tensor) tensor.Values { @@ -363,11 +247,7 @@ func Sin(in tensor.Tensor) tensor.Values { } func SinOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Sin(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Sin(a) }, in, out) } func Sinh(in tensor.Tensor) tensor.Values { @@ -375,11 +255,7 @@ func Sinh(in tensor.Tensor) tensor.Values { } func SinhOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Sinh(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Sinh(a) }, in, out) } func Sqrt(in tensor.Tensor) tensor.Values { @@ -387,11 +263,7 @@ func Sqrt(in tensor.Tensor) tensor.Values { } func SqrtOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Sqrt(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Sqrt(a) }, in, out) } func Tan(in tensor.Tensor) tensor.Values { @@ -399,11 +271,7 @@ func Tan(in tensor.Tensor) tensor.Values { } func TanOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Tan(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Tan(a) }, in, out) } func Tanh(in tensor.Tensor) tensor.Values { @@ -411,11 +279,7 @@ func Tanh(in tensor.Tensor) tensor.Values { } func TanhOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Tanh(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Tanh(a) }, in, out) } func Trunc(in tensor.Tensor) tensor.Values { @@ -423,11 +287,7 @@ func Trunc(in tensor.Tensor) tensor.Values { } func TruncOut(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Trunc(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Trunc(a) }, in, out) } func Y0(in tensor.Tensor) tensor.Values { @@ -435,11 +295,7 @@ func Y0(in tensor.Tensor) tensor.Values { } func Y0Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Y0(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Y0(a) }, in, out) } func Y1(in tensor.Tensor) tensor.Values { @@ -447,25 +303,85 @@ func Y1(in tensor.Tensor) tensor.Values { } func Y1Out(in tensor.Tensor, out tensor.Values) error { - tensor.SetShapeFrom(out, in) - tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { - tsr[1].SetFloat1D(math.Y1(tsr[0].Float1D(idx)), idx) - }, in, out) - return nil + return tensor.FloatFuncOut(1, func(a float64) float64 { return math.Y1(a) }, in, out) +} + +//////////////////////// Binary + +func Atan2(y, x tensor.Tensor) tensor.Values { + return tensor.CallOut2(Atan2Out, y, x) +} + +func Atan2Out(y, x tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Atan2(a, b) }, y, x, out) +} + +func Copysign(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(CopysignOut, x, y) +} + +func CopysignOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Copysign(a, b) }, x, y, out) +} + +func Dim(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(DimOut, x, y) +} + +func DimOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Dim(a, b) }, x, y, out) +} + +func Hypot(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(HypotOut, x, y) +} + +func HypotOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Hypot(a, b) }, x, y, out) +} + +func Max(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(MaxOut, x, y) +} + +func MaxOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Max(a, b) }, x, y, out) +} + +func Min(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(MinOut, x, y) +} + +func MinOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Min(a, b) }, x, y, out) +} + +func Nextafter(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(NextafterOut, x, y) +} + +func NextafterOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Nextafter(a, b) }, x, y, out) +} + +func Pow(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(PowOut, x, y) +} + +func PowOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Pow(a, b) }, x, y, out) +} + +func Remainder(x, y tensor.Tensor) tensor.Values { + return tensor.CallOut2(RemainderOut, x, y) +} + +func RemainderOut(x, y tensor.Tensor, out tensor.Values) error { + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Remainder(a, b) }, x, y, out) } /* -func Atan2(y, in tensor.Tensor, out tensor.Values) -func Copysign(f, sign float64) float64 -func Dim(x, y float64) float64 -func Hypot(p, q float64) float64 -func Max(x, y float64) float64 -func Min(x, y float64) float64 -func Mod(x, y float64) float64 -func Nextafter(x, y float64) (r float64) func Nextafter32(x, y float32) (r float32) -func Pow(x, y float64) float64 -func Remainder(x, y float64) float64 func Inf(sign int) float64 func IsInf(f float64, sign int) bool diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index 3aee561808..e36535fbd9 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -68,3 +68,49 @@ func TestMath(t *testing.T) { } } } + +type twof func(x, y float64) float64 +type ttwof func(x, y tensor.Tensor, out tensor.Values) error + +func TestMathBinary(t *testing.T) { + scalar := tensor.NewFloat64Scalar(-5.5) + scout := scalar.Clone() + + vals := []float64{-1.507556722888818, -1.2060453783110545, -0.9045340337332908, -0.6030226891555273, -0.3015113445777635, 0, 0.3015113445777635, 0.603022689155527, 0.904534033733291, 1.2060453783110545, 1.507556722888818, .3} + + oned := tensor.NewNumberFromValues(vals...) + oneout := oned.Clone() + + cell2d := tensor.NewFloat32(5, 2, 6) + _, cells := cell2d.Shape().RowCellSize() + assert.Equal(t, cells, 12) + tensor.VectorizeThreaded(1, tensor.NFirstLen, func(idx int, tsr ...tensor.Tensor) { + ci := idx % cells + cell2d.SetFloat1D(oned.Float1D(ci), idx) + }, cell2d) + cellout := cell2d.Clone() + + mfuncs := []twof{math.Atan2, math.Copysign, math.Dim, math.Hypot, math.Max, math.Min, math.Nextafter, math.Pow, math.Remainder} + tfuncs := []ttwof{Atan2Out, CopysignOut, DimOut, HypotOut, MaxOut, MinOut, NextafterOut, PowOut, RemainderOut} + + for i, fun := range mfuncs { + tf := tfuncs[i] + tf(scalar, scalar, scout) + tf(oned, oned, oneout) + tf(cell2d, cell2d, cellout) + + testEqual(t, fun(scalar.Float1D(0), scalar.Float1D(0)), scout.Float1D(0)) + for i, v := range vals { + testEqual(t, fun(v, v), oneout.Float1D(i)) + } + lv := len(vals) + for r := range 5 { + // fmt.Println(r) + si := lv * r + for c, v := range vals { + ov := tensor.AsFloat32Tensor(cellout).Values[si+c] + testEqual(t, fun(v, v), float64(ov)) + } + } + } +} diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 92411e258c..3c71ae6d67 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -73,7 +73,7 @@ func AddOut(a, b tensor.Tensor, out tensor.Values) error { if a.IsString() { return tensor.StringBinaryFuncOut(func(a, b string) string { return a + b }, a, b, out) } - return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a + b }, a, b, out) + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return a + b }, a, b, out) } // Sub subtracts tensors into output. @@ -83,7 +83,7 @@ func Sub(a, b tensor.Tensor) tensor.Tensor { // SubOut subtracts two tensors into output. func SubOut(a, b tensor.Tensor, out tensor.Values) error { - return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a - b }, a, b, out) + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return a - b }, a, b, out) } // Mul multiplies tensors into output. @@ -93,7 +93,7 @@ func Mul(a, b tensor.Tensor) tensor.Tensor { // MulOut multiplies two tensors into output. func MulOut(a, b tensor.Tensor, out tensor.Values) error { - return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a * b }, a, b, out) + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return a * b }, a, b, out) } // Div divides tensors into output. @@ -103,7 +103,7 @@ func Div(a, b tensor.Tensor) tensor.Tensor { // DivOut divides two tensors into output. func DivOut(a, b tensor.Tensor, out tensor.Values) error { - return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return a / b }, a, b, out) + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return a / b }, a, b, out) } // Mod performs modulus a%b on tensors into output. @@ -113,7 +113,7 @@ func Mod(a, b tensor.Tensor) tensor.Tensor { // ModOut performs modulus a%b on tensors into output. func ModOut(a, b tensor.Tensor, out tensor.Values) error { - return tensor.FloatBinaryFuncOut(func(a, b float64) float64 { return math.Mod(a, b) }, a, b, out) + return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return math.Mod(a, b) }, a, b, out) } // Negate stores in the output the bool value -a. diff --git a/tensor/vectorize.go b/tensor/vectorize.go index 1e5379fe1a..dcfffd1bd0 100644 --- a/tensor/vectorize.go +++ b/tensor/vectorize.go @@ -11,11 +11,11 @@ import ( ) var ( - // ThreadingThreshod is the threshold in number of tensor elements - // to engage actual parallel processing. + // ThreadingThreshod is the threshold in number of flops (floating point ops), + // computed as tensor N * flops per element, to engage actual parallel processing. // Heuristically, numbers below this threshold do not result in // an overall speedup, due to overhead costs. - ThreadingThreshold = 100_000 + ThreadingThreshold = 10_000 // NumThreads is the number of threads to use for parallel threading. // The default of 0 causes the [runtime.GOMAXPROCS] to be used. @@ -51,6 +51,8 @@ func Vectorize(nfun func(tsr ...Tensor) int, fun func(idx int, tsr ...Tensor), t // (floating point operations) for the function exceeds the [ThreadingThreshold]. // Heuristically, numbers below this threshold do not result // in an overall speedup, due to overhead costs. +// Each elemental math operation in the function adds a flop. +// See estimates in [tmath] for basic math functions. func VectorizeThreaded(flops int, nfun func(tsr ...Tensor) int, fun func(idx int, tsr ...Tensor), tsr ...Tensor) { n := nfun(tsr...) if n <= 0 { From 46d5974db51b4b29365c5a8d166590806d9d1a5c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 13:43:00 -0700 Subject: [PATCH 144/311] add power operator as ** --- goal/transpile/math.go | 15 +++++++++++++++ goal/transpile/transpile_test.go | 14 ++++++++++++++ .../cogentcore_org-core-tensor-tmath.go | 18 ++++++++++++++++++ .../symbols/cogentcore_org-core-tensor.go | 2 ++ 4 files changed, 49 insertions(+) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 4b62d9079d..de0aef2507 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -335,6 +335,10 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.UnaryExpr: mp.unaryExpr(x) + case *ast.StarExpr: + mp.addToken(token.MUL) + mp.expr(x.X) + case *ast.BinaryExpr: mp.binaryExpr(x) @@ -445,6 +449,10 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { fn = "Sub" case token.MUL: fn = "Mul" + if un, ok := ex.Y.(*ast.StarExpr); ok { // ** power operator + ex.Y = un.X + fn = "Pow" + } case token.QUO: fn = "Div" case token.EQL: @@ -471,6 +479,9 @@ func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { mp.expr(ex.X) mp.out.Add(token.COMMA) mp.idx++ + if fn == "Pow" { + mp.idx++ + } mp.expr(ex.Y) mp.out.Add(token.RPAREN) mp.endFunc() @@ -488,6 +499,10 @@ func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { fn = "Not" case token.SUB: fn = "Negate" + default: // * goes to StarExpr -- not sure what else could happen here? + mp.addToken(ex.Op) + mp.expr(ex.X) + return } mp.startFunc("tmath." + fn) mp.addToken(token.LPAREN) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index d8663bc9ed..801a88c120 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -200,6 +200,19 @@ goal.Run("ls", "-la", "args...") } } +// Use this for testing the current thing working on. +func TestCur(t *testing.T) { + // logx.UserLevel = slog.LevelDebug + tests := []exIn{ + {"# a := x ** 2", `a := tmath.Pow(x, tensor.NewIntScalar(2))`}, + } + st := NewState() + for _, test := range tests { + o := st.TranspileLine(test.i) + assert.Equal(t, test.e, o) + } +} + func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ @@ -207,6 +220,7 @@ func TestMath(t *testing.T) { {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, {"# a = x + y", `a = tmath.Add(x, y)`}, + {"# a := x ** 2", `a := tmath.Pow(x, tensor.NewIntScalar(2))`}, {"# a = -x", `a = tmath.Negate(x)`}, {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go index 858b857a0c..d97bba9505 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go @@ -27,6 +27,8 @@ func init() { "AsinhOut": reflect.ValueOf(tmath.AsinhOut), "Assign": reflect.ValueOf(tmath.Assign), "Atan": reflect.ValueOf(tmath.Atan), + "Atan2": reflect.ValueOf(tmath.Atan2), + "Atan2Out": reflect.ValueOf(tmath.Atan2Out), "AtanOut": reflect.ValueOf(tmath.AtanOut), "Atanh": reflect.ValueOf(tmath.Atanh), "AtanhOut": reflect.ValueOf(tmath.AtanhOut), @@ -34,11 +36,15 @@ func init() { "CbrtOut": reflect.ValueOf(tmath.CbrtOut), "Ceil": reflect.ValueOf(tmath.Ceil), "CeilOut": reflect.ValueOf(tmath.CeilOut), + "Copysign": reflect.ValueOf(tmath.Copysign), + "CopysignOut": reflect.ValueOf(tmath.CopysignOut), "Cos": reflect.ValueOf(tmath.Cos), "CosOut": reflect.ValueOf(tmath.CosOut), "Cosh": reflect.ValueOf(tmath.Cosh), "CoshOut": reflect.ValueOf(tmath.CoshOut), "Dec": reflect.ValueOf(tmath.Dec), + "Dim": reflect.ValueOf(tmath.Dim), + "DimOut": reflect.ValueOf(tmath.DimOut), "Div": reflect.ValueOf(tmath.Div), "DivAssign": reflect.ValueOf(tmath.DivAssign), "DivOut": reflect.ValueOf(tmath.DivOut), @@ -66,6 +72,8 @@ func init() { "GreaterEqual": reflect.ValueOf(tmath.GreaterEqual), "GreaterEqualOut": reflect.ValueOf(tmath.GreaterEqualOut), "GreaterOut": reflect.ValueOf(tmath.GreaterOut), + "Hypot": reflect.ValueOf(tmath.Hypot), + "HypotOut": reflect.ValueOf(tmath.HypotOut), "Inc": reflect.ValueOf(tmath.Inc), "J0": reflect.ValueOf(tmath.J0), "J0Out": reflect.ValueOf(tmath.J0Out), @@ -85,6 +93,10 @@ func init() { "LogOut": reflect.ValueOf(tmath.LogOut), "Logb": reflect.ValueOf(tmath.Logb), "LogbOut": reflect.ValueOf(tmath.LogbOut), + "Max": reflect.ValueOf(tmath.Max), + "MaxOut": reflect.ValueOf(tmath.MaxOut), + "Min": reflect.ValueOf(tmath.Min), + "MinOut": reflect.ValueOf(tmath.MinOut), "Mod": reflect.ValueOf(tmath.Mod), "ModAssign": reflect.ValueOf(tmath.ModAssign), "ModOut": reflect.ValueOf(tmath.ModOut), @@ -93,12 +105,18 @@ func init() { "MulOut": reflect.ValueOf(tmath.MulOut), "Negate": reflect.ValueOf(tmath.Negate), "NegateOut": reflect.ValueOf(tmath.NegateOut), + "Nextafter": reflect.ValueOf(tmath.Nextafter), + "NextafterOut": reflect.ValueOf(tmath.NextafterOut), "Not": reflect.ValueOf(tmath.Not), "NotEqual": reflect.ValueOf(tmath.NotEqual), "NotEqualOut": reflect.ValueOf(tmath.NotEqualOut), "NotOut": reflect.ValueOf(tmath.NotOut), "Or": reflect.ValueOf(tmath.Or), "OrOut": reflect.ValueOf(tmath.OrOut), + "Pow": reflect.ValueOf(tmath.Pow), + "PowOut": reflect.ValueOf(tmath.PowOut), + "Remainder": reflect.ValueOf(tmath.Remainder), + "RemainderOut": reflect.ValueOf(tmath.RemainderOut), "Round": reflect.ValueOf(tmath.Round), "RoundOut": reflect.ValueOf(tmath.RoundOut), "RoundToEven": reflect.ValueOf(tmath.RoundToEven), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 24f87c307d..eebc2dc72b 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -62,6 +62,8 @@ func init() { "FloatAssignFunc": reflect.ValueOf(tensor.FloatAssignFunc), "FloatBinaryFunc": reflect.ValueOf(tensor.FloatBinaryFunc), "FloatBinaryFuncOut": reflect.ValueOf(tensor.FloatBinaryFuncOut), + "FloatFunc": reflect.ValueOf(tensor.FloatFunc), + "FloatFuncOut": reflect.ValueOf(tensor.FloatFuncOut), "FullAxis": reflect.ValueOf(tensor.FullAxis), "FuncByName": reflect.ValueOf(tensor.FuncByName), "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), From 4f323bfce197ed4d060d058c7dad929a66937bb2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 13:52:12 -0700 Subject: [PATCH 145/311] As*Tensor -> As* plain -- these are quite handy; fixed datafs --- tensor/convert.go | 20 ++++++++----------- tensor/examples/datafs-sim/sim.go | 3 +-- tensor/stats/metric/matrix.go | 8 ++++---- .../symbols/cogentcore_org-core-tensor.go | 8 ++++---- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/tensor/convert.go b/tensor/convert.go index 2439f0a74a..d94a471130 100644 --- a/tensor/convert.go +++ b/tensor/convert.go @@ -177,12 +177,12 @@ func AsStringSlice(tsr Tensor) []string { return slc } -// AsFloat64Tensor returns the tensor as a [Float64] tensor. +// AsFloat64 returns the tensor as a [Float64] tensor. // If already is a Float64, it is returned as such. // Otherwise, a new Float64 tensor is created and values are copied. // Use this function for interfacing with gonum or other apis that // only operate on float64 types. -func AsFloat64Tensor(tsr Tensor) *Float64 { +func AsFloat64(tsr Tensor) *Float64 { if f, ok := tsr.(*Float64); ok { return f } @@ -191,10 +191,10 @@ func AsFloat64Tensor(tsr Tensor) *Float64 { return f } -// AsFloat32Tensor returns the tensor as a [Float32] tensor. +// AsFloat32 returns the tensor as a [Float32] tensor. // If already is a Float32, it is returned as such. // Otherwise, a new Float32 tensor is created and values are copied. -func AsFloat32Tensor(tsr Tensor) *Float32 { +func AsFloat32(tsr Tensor) *Float32 { if f, ok := tsr.(*Float32); ok { return f } @@ -203,12 +203,10 @@ func AsFloat32Tensor(tsr Tensor) *Float32 { return f } -// AsStringTensor returns the tensor as a [String] tensor. +// AsString returns the tensor as a [String] tensor. // If already is a String, it is returned as such. // Otherwise, a new String tensor is created and values are copied. -// Use this function for interfacing with gonum or other apis that -// only operate on float64 types. -func AsStringTensor(tsr Tensor) *String { +func AsString(tsr Tensor) *String { if f, ok := tsr.(*String); ok { return f } @@ -217,12 +215,10 @@ func AsStringTensor(tsr Tensor) *String { return f } -// AsIntTensor returns the tensor as a [Int] tensor. +// AsInt returns the tensor as a [Int] tensor. // If already is a Int, it is returned as such. // Otherwise, a new Int tensor is created and values are copied. -// Use this function for interfacing with gonum or other apis that -// only operate on float64 types. -func AsIntTensor(tsr Tensor) *Int { +func AsInt(tsr Tensor) *Int { if f, ok := tsr.(*Int); ok { return f } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index c926d16644..97162c97ba 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -104,7 +104,6 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a logd, _ := dir.Mkdir(level) sitems := ss.Stats.ValuesFunc(nil) nctr := ss.Config.Item("N" + level).AsInt() - stout := tensor.NewFloat64Scalar(0) // tmp stat output for _, st := range sitems { if st.IsString() { continue @@ -118,7 +117,7 @@ func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, a lt := dd.NewOfType(ag.String(), st.DataType(), nctr) lt.Metadata().Copy(*st.Metadata()) tensor.SetCalcFunc(lt, func() error { - ag.Call(src, stout) + stout := ag.Call(src) ctr := ss.Stats.Item(level).AsInt() lt.SetFloatRow(stout.FloatRow(0), ctr) return nil diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index fc60ccf461..37e94c32e3 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -199,7 +199,7 @@ func CovarianceMatrix(fun any, in tensor.Tensor) tensor.Values { // Note that PCA produces results in the *opposite* order of [SVD]. func PCA(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { n := covar.DimSize(0) - cv, err := matrix.NewSymmetric(tensor.AsFloat64Tensor(covar)) + cv, err := matrix.NewSymmetric(tensor.AsFloat64(covar)) if err != nil { return err } @@ -213,7 +213,7 @@ func PCA(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { var ev mat.Dense eig.VectorsTo(&ev) matrix.CopyDense(eigenvecs, &ev) - fv := tensor.AsFloat64Tensor(vals) + fv := tensor.AsFloat64(vals) eig.Values(fv.Values) if fv != vals { vals.(tensor.Values).CopyFrom(fv) @@ -230,7 +230,7 @@ func PCA(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { // Note that SVD produces results in the *opposite* order of [PCA]. func SVD(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { n := covar.DimSize(0) - cv, err := matrix.NewSymmetric(tensor.AsFloat64Tensor(covar)) + cv, err := matrix.NewSymmetric(tensor.AsFloat64(covar)) if err != nil { return err } @@ -244,7 +244,7 @@ func SVD(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { var ev mat.Dense eig.UTo(&ev) matrix.CopyDense(eigenvecs, &ev) - fv := tensor.AsFloat64Tensor(vals) + fv := tensor.AsFloat64(vals) eig.Values(fv.Values) if fv != vals { vals.(tensor.Values).CopyFrom(fv) diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index eebc2dc72b..3d3ba8e6f7 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -16,21 +16,21 @@ func init() { "AlignForAssign": reflect.ValueOf(tensor.AlignForAssign), "AlignShapes": reflect.ValueOf(tensor.AlignShapes), "As1D": reflect.ValueOf(tensor.As1D), - "AsFloat32Tensor": reflect.ValueOf(tensor.AsFloat32Tensor), + "AsFloat32": reflect.ValueOf(tensor.AsFloat32), + "AsFloat64": reflect.ValueOf(tensor.AsFloat64), "AsFloat64Scalar": reflect.ValueOf(tensor.AsFloat64Scalar), "AsFloat64Slice": reflect.ValueOf(tensor.AsFloat64Slice), - "AsFloat64Tensor": reflect.ValueOf(tensor.AsFloat64Tensor), "AsIndexed": reflect.ValueOf(tensor.AsIndexed), + "AsInt": reflect.ValueOf(tensor.AsInt), "AsIntScalar": reflect.ValueOf(tensor.AsIntScalar), "AsIntSlice": reflect.ValueOf(tensor.AsIntSlice), - "AsIntTensor": reflect.ValueOf(tensor.AsIntTensor), "AsMasked": reflect.ValueOf(tensor.AsMasked), "AsReshaped": reflect.ValueOf(tensor.AsReshaped), "AsRows": reflect.ValueOf(tensor.AsRows), "AsSliced": reflect.ValueOf(tensor.AsSliced), + "AsString": reflect.ValueOf(tensor.AsString), "AsStringScalar": reflect.ValueOf(tensor.AsStringScalar), "AsStringSlice": reflect.ValueOf(tensor.AsStringSlice), - "AsStringTensor": reflect.ValueOf(tensor.AsStringTensor), "Ascending": reflect.ValueOf(tensor.Ascending), "BoolFloatsFunc": reflect.ValueOf(tensor.BoolFloatsFunc), "BoolFloatsFuncOut": reflect.ValueOf(tensor.BoolFloatsFuncOut), From bca653aa88eb118e80473d5bcf6451110c0fc073 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 15:57:03 -0700 Subject: [PATCH 146/311] much better Sprintf function that takes optional format and ensures result is fully aligned, even if it takes 2 tabs per value instead of 1. --- goal/README.md | 2 +- goal/transpile/math.go | 18 +-- goal/transpile/transpile_test.go | 9 +- tensor/align.go | 14 +++ tensor/base.go | 59 ---------- tensor/bool.go | 2 +- tensor/create.go | 81 +++++++++++-- tensor/funcs_test.go | 21 ++++ tensor/indexed.go | 2 +- tensor/io.go | 110 ++++++++++++++++++ tensor/masked.go | 2 +- tensor/number.go | 2 +- tensor/reshaped.go | 2 +- tensor/rows.go | 2 +- tensor/sliced.go | 2 +- tensor/slices.go | 16 --- tensor/string.go | 2 +- tensor/tmath/ops.go | 8 +- .../symbols/cogentcore_org-core-tensor.go | 7 +- 19 files changed, 247 insertions(+), 114 deletions(-) diff --git a/goal/README.md b/goal/README.md index 85acd894e2..5aa8719087 100644 --- a/goal/README.md +++ b/goal/README.md @@ -271,7 +271,7 @@ The following sections provide a full list of equivalents between the `tensor` G | `tensor.NewFloat64(3,4,5)` | `zeros(3, 4, 5)` | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional tensor of float64 zeros | | `tensor.NewFloat64Ones(3,4)` | `ones(3, 4)` | `np.ones((3, 4))` | `ones(3,4)` | 3x4 2D tensor of 64-bit floating point ones | | `tensor.NewFloat64Full(5.5, 3,4)` | `full(5.5, 3, 4)` | `np.full((3, 4), 5.5)` | ? | 3x4 2D tensor of 5.5; Goal variadic arg structure requires value to come first | -| TODO: | | `rand([3, 4])` or `slrand([3,4], c, fi)` | `rng.random(3, 4)` | `rand(3,4)` | 3x4 2D float64 tensor with uniform random 0..1 elements; `rand` uses current Go `rand` source, while `slrand` uses [gosl](../gpu/gosl/slrand) GPU-safe call with counter `c` and function index `fi` and key = index of element | +| `tensor.NewFloat64Rand(3,4)` | `rand(3, 4)` or `slrand(c, fi, 3, 4)` | `rng.random(3, 4)` | `rand(3,4)` | 3x4 2D float64 tensor with uniform random 0..1 elements; `rand` uses current Go `rand` source, while `slrand` uses [gosl](../gpu/gosl/slrand) GPU-safe call with counter `c` and function index `fi` and key = index of element | | TODO: | |`np.concatenate((a,b),1)` or `np.hstack((a,b))` or `np.column_stack((a,b))` or `np.c_[a,b]` | `[a b]` | concatenate columns of a and b | | TODO: | |`np.concatenate((a,b))` or `np.vstack((a,b))` or `np.r_[a,b]` | `[a; b]` | concatenate rows of a and b | | TODO: | |`np.tile(a, (m, n))` | `repmat(a, m, n)` | create m by n copies of a | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index de0aef2507..e4fb8ed63d 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -742,14 +742,16 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { // nofun = do not accept a function version, just a method var numpyFuncs = map[string]funWrap{ // "array": {"tensor.NewFloatFromValues", ""}, // todo: probably not right, maybe don't have? - "zeros": {"tensor.NewFloat64", ""}, - "full": {"tensor.NewFloat64Full", ""}, - "ones": {"tensor.NewFloat64Ones", ""}, - "arange": {"tensor.NewSliceInts", ""}, - "reshape": {"tensor.Reshape", ""}, - "copy": {"tensor.Clone", ""}, - "flatten": {"tensor.Flatten", "nofun"}, - "squeeze": {"tensor.Squeeze", "nofun"}, + "zeros": {"tensor.NewFloat64", ""}, + "full": {"tensor.NewFloat64Full", ""}, + "ones": {"tensor.NewFloat64Ones", ""}, + "rand": {"tensor.NewFloat64Rand", ""}, + "arange": {"tensor.NewIntRange", ""}, + "linspace": {"tensor.NewFloat64SpacedLinear", ""}, + "reshape": {"tensor.Reshape", ""}, + "copy": {"tensor.Clone", ""}, + "flatten": {"tensor.Flatten", "nofun"}, + "squeeze": {"tensor.Squeeze", "nofun"}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 801a88c120..de3a3ea642 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# a := x ** 2", `a := tmath.Pow(x, tensor.NewIntScalar(2))`}, + {"# a := linspace(0, 5, 6, true)", `a := tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, } st := NewState() for _, test := range tests { @@ -228,12 +228,13 @@ func TestMath(t *testing.T) { {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, - {"# a := arange(36)", `a := tensor.NewSliceInts(36)`}, - {"# a := arange(36, 0,0,-1)", `a := tensor.NewSliceInts(36, 0, 0, - 1)`}, + {"# a := arange(36)", `a := tensor.NewIntRange(36)`}, + {"# a := arange(36, 0, -1)", `a := tensor.NewIntRange(36, 0, - 1)`}, + {"# a := linspace(0, 5, 6, true)", `a := tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, - {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewSliceInts(36), 6, 6)`}, + {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewIntRange(36), 6, 6)`}, {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, diff --git a/tensor/align.go b/tensor/align.go index f04ad96bc6..79b046c6fd 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -211,6 +211,20 @@ func FloatFuncOut(flops int, fun func(in float64) float64, in Tensor, out Values return nil } +// FloatSetFunc sets tensor float64 values from a function, +// which gets the index. Must be parallel threadsafe. +// The flops (floating point operations) estimate is used to control parallel +// threading using goroutines, and should reflect number of flops in the function. +// See [VectorizeThreaded] for more information. +func FloatSetFunc(flops int, fun func(idx int) float64, a Tensor) error { + n := a.Len() + VectorizeThreaded(flops, func(tsr ...Tensor) int { return n }, + func(idx int, tsr ...Tensor) { + tsr[0].SetFloat1D(fun(idx), idx) + }, a) + return nil +} + ////////////////////////// Bool // BoolStringsFunc sets boolean output value based on a function involving diff --git a/tensor/base.go b/tensor/base.go index 15f1fda04a..9e0d7bf590 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -5,10 +5,8 @@ package tensor import ( - "fmt" "reflect" "slices" - "strings" "unsafe" "cogentcore.org/core/base/metadata" @@ -134,60 +132,3 @@ func (tsr *Base[T]) StringRowCell(row, cell int) string { func (tsr *Base[T]) Label() string { return label(tsr.Meta.Name(), &tsr.shape) } - -// Sprint returns a string representation of the given tensor, -// with a maximum length of as given: output is terminated -// when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. -func Sprint(tsr Tensor, maxLen int) string { - return sprint(tsr, maxLen) -} - -func label(nm string, sh *Shape) string { - if nm != "" { - nm += " " + sh.String() - } else { - nm = sh.String() - } - return nm -} - -// sprint is the underlying impl of String -func sprint(tsr Tensor, maxLen int) string { - if maxLen == 0 { - maxLen = MaxSprintLength - } - var b strings.Builder - sh := tsr.Shape() - b.WriteString(tsr.Label()) - noidx := false - if tsr.NumDims() == 1 && tsr.Len() < 8 { - b.WriteString(" ") - noidx = true - } else { - b.WriteString("\n") - } - oddRow := false - rows, cols, _, _ := Projection2DShape(sh, oddRow) - ctr := 0 - for r := range rows { - rc, _ := Projection2DCoords(sh, oddRow, r, 0) - if !noidx { - b.WriteString(fmt.Sprintf("%v: ", rc)) - } - ri := r - for c := 0; c < cols; c++ { - if tsr.IsString() { - b.WriteString(fmt.Sprintf("%s ", Projection2DString(tsr, oddRow, ri, c))) - } else { - b.WriteString(fmt.Sprintf("%7g ", Projection2DValue(tsr, oddRow, ri, c))) - } - } - b.WriteString("\n") - ctr += cols - if ctr > maxLen { - b.WriteString("...\n") - break - } - } - return b.String() -} diff --git a/tensor/bool.go b/tensor/bool.go index 9a88805876..7233563d9c 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -63,7 +63,7 @@ func BoolToInt(bv bool) int { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Bool) String() string { return sprint(tsr, 0) } +func (tsr *Bool) String() string { return Sprintf(tsr, 0, "") } // Label satisfies the core.Labeler interface for a summary description of the tensor func (tsr *Bool) Label() string { diff --git a/tensor/create.go b/tensor/create.go index 68367b4d31..a2e1992cb0 100644 --- a/tensor/create.go +++ b/tensor/create.go @@ -4,6 +4,11 @@ package tensor +import ( + "math/rand" + "slices" +) + // NewFloat64Scalar is a convenience method for a Tensor // representation of a single float64 scalar value. func NewFloat64Scalar(val float64) *Float64 { @@ -49,9 +54,7 @@ func NewStringFromValues(vals ...string) *String { // SetAllFloat64 sets all values of given tensor to given value. func SetAllFloat64(tsr Tensor, val float64) { - VectorizeThreaded(1, func(tsr ...Tensor) int { - return tsr[0].Len() - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return tsr[0].Len() }, func(idx int, tsr ...Tensor) { tsr[0].SetFloat1D(val, idx) }, tsr) @@ -59,9 +62,7 @@ func SetAllFloat64(tsr Tensor, val float64) { // SetAllInt sets all values of given tensor to given value. func SetAllInt(tsr Tensor, val int) { - VectorizeThreaded(1, func(tsr ...Tensor) int { - return tsr[0].Len() - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return tsr[0].Len() }, func(idx int, tsr ...Tensor) { tsr[0].SetInt1D(val, idx) }, tsr) @@ -69,9 +70,7 @@ func SetAllInt(tsr Tensor, val int) { // SetAllString sets all values of given tensor to given value. func SetAllString(tsr Tensor, val string) { - VectorizeThreaded(1, func(tsr ...Tensor) int { - return tsr[0].Len() - }, + VectorizeThreaded(1, func(tsr ...Tensor) int { return tsr[0].Len() }, func(idx int, tsr ...Tensor) { tsr[0].SetString1D(val, idx) }, tsr) @@ -108,3 +107,67 @@ func NewStringFull(val string, sizes ...int) *String { SetAllString(tsr, val) return tsr } + +// NewFloat64Rand returns a new tensor full of random numbers from +// global random source, of given shape sizes. +func NewFloat64Rand(sizes ...int) *Float64 { + tsr := NewFloat64(sizes...) + FloatSetFunc(1, func(idx int) float64 { return rand.Float64() }, tsr) + return tsr +} + +// NewIntRange returns a new [Int] [Tensor] with given [Slice] +// range parameters, with the same semantics as NumPy arange based on +// the number of arguments passed: +// - 1 = stop +// - 2 = start, stop +// - 3 = start, stop, step +func NewIntRange(svals ...int) *Int { + if len(svals) == 0 { + return NewInt() + } + sl := Slice{} + switch len(svals) { + case 1: + sl.Stop = svals[0] + case 2: + sl.Start = svals[0] + sl.Stop = svals[1] + case 3: + sl.Start = svals[0] + sl.Stop = svals[1] + sl.Step = svals[2] + } + return sl.IntTensor(sl.Stop) +} + +// NewFloat64SpacedLinear returns a new [Float64] tensor with num linearly +// spaced numbers between start and stop values, as tensors, which +// must be the same length and determine the cell shape of the output. +// If num is 0, then a default of 50 is used. +// If endpoint = true, then the stop value is _inclusive_, i.e., it will +// be the final value, otherwise it is exclusive. +// This corresponds to the NumPy linspace function. +func NewFloat64SpacedLinear(start, stop Tensor, num int, endpoint bool) *Float64 { + if num <= 0 { + num = 50 + } + fnum := float64(num) + if endpoint { + fnum -= 1 + } + step := Clone(start) + n := step.Len() + for i := range n { + step.SetFloat1D((stop.Float1D(i)-start.Float1D(i))/fnum, i) + } + tsz := slices.Clone(start.Shape().Sizes) + tsz = append([]int{num}, tsz...) + tsr := NewFloat64(tsz...) + for r := range num { + for i := range n { + tsr.SetFloatRowCell(start.Float1D(i)+float64(r)*step.Float1D(i), r, i) + } + } + return tsr +} diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index e2465ab132..358c7e4dd0 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -69,3 +69,24 @@ func TestAlign(t *testing.T) { assert.Equal(t, []int{3, 1, 4}, as.Sizes) assert.Equal(t, []int{1, 3, 4}, bs.Sizes) } + +func TestCreate(t *testing.T) { + ar := NewIntRange(5) + assert.Equal(t, []int{0, 1, 2, 3, 4}, AsIntSlice(ar)) + + ar = NewIntRange(2, 5) + assert.Equal(t, []int{2, 3, 4}, AsIntSlice(ar)) + + ar = NewIntRange(0, 5, 2) + assert.Equal(t, []int{0, 2, 4}, AsIntSlice(ar)) + + lr := NewFloat64SpacedLinear(NewFloat64Scalar(0), NewFloat64Scalar(5), 6, true) + assert.Equal(t, []float64{0, 1, 2, 3, 4, 5}, AsFloat64Slice(lr)) + + lr = NewFloat64SpacedLinear(NewFloat64Scalar(0), NewFloat64Scalar(5), 5, false) + assert.Equal(t, []float64{0, 1, 2, 3, 4}, AsFloat64Slice(lr)) + + lr2 := NewFloat64SpacedLinear(NewFloat64FromValues(0, 2), NewFloat64FromValues(5, 7), 5, false) + // fmt.Println(lr2) + assert.Equal(t, []float64{0, 2, 1, 3, 2, 4, 3, 5, 4, 6}, AsFloat64Slice(lr2)) +} diff --git a/tensor/indexed.go b/tensor/indexed.go index 96ee963216..4beb62b8b6 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -80,7 +80,7 @@ func (ix *Indexed) SourceIndexesFrom1D(oned int) []int { } func (ix *Indexed) Label() string { return label(ix.Metadata().Name(), ix.Shape()) } -func (ix *Indexed) String() string { return sprint(ix, 0) } +func (ix *Indexed) String() string { return Sprintf(ix, 0, "") } func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } func (ix *Indexed) IsString() bool { return ix.Tensor.IsString() } func (ix *Indexed) DataType() reflect.Kind { return ix.Tensor.DataType() } diff --git a/tensor/io.go b/tensor/io.go index 44adaa4b70..e5201ea4a1 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -6,13 +6,17 @@ package tensor import ( "encoding/csv" + "fmt" "io" "log" "os" "strconv" + "strings" "cogentcore.org/core/base/metadata" + "cogentcore.org/core/base/reflectx" "cogentcore.org/core/core" + "cogentcore.org/core/math32" ) // Delim are standard CSV delimiter options (Tab, Comma, Space) @@ -153,3 +157,109 @@ func ReadCSV(tsr Tensor, r io.Reader, delim Delims) error { done: return nil } + +func label(nm string, sh *Shape) string { + if nm != "" { + nm += " " + sh.String() + } else { + nm = sh.String() + } + return nm +} + +// Sprintf returns a string representation of the given tensor, +// with a maximum length of as given: output is terminated +// when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. +// The format is the per-element format string, which should include +// any delimiter or spacing between elements (which will apply to last +// element too). If empty it uses compact defaults for the data type. +func Sprintf(tsr Tensor, maxLen int, format string) string { + if maxLen == 0 { + maxLen = MaxSprintLength + } + colWd := 1 // column width in tabs + defFmt := format == "" + isint := false + if defFmt { + switch { + case tsr.IsString(): + format = "%15s\t" + case reflectx.KindIsInt(tsr.DataType()): + isint = true + format = "%7g\t" + default: + format = "%7.3g\t" + } + } + mxlen := 0 + for i := range maxLen { + s := "" + if tsr.IsString() { + s = fmt.Sprintf(format, tsr.String1D(i)) + } else { + s = fmt.Sprintf(format, tsr.Float1D(i)) + } + if len(s) > mxlen { + mxlen = len(s) + } + } + colWd = int(math32.IntMultipleGE(float32(mxlen), 8)) / 8 + if colWd > 1 && !tsr.IsString() && defFmt { // should be 2 + if isint { + format = "%15g\t" + } else { + format = "%15.7g\t" + } + } + sh := tsr.Shape() + oddRow := false + rows, cols, _, _ := Projection2DShape(sh, oddRow) + var b strings.Builder + b.WriteString(tsr.Label()) + noidx := false + if tsr.NumDims() == 1 && tsr.Len() < 8 { + b.WriteString(" ") + noidx = true + } else { + b.WriteString("\n") + } + if !noidx && tsr.NumDims() > 1 && cols > 1 { + b.WriteString("\t") + for c := 0; c < cols; c++ { + _, cc := Projection2DCoords(sh, oddRow, 0, c) + b.WriteString(fmt.Sprintf("%v:\t", cc)) + if colWd > 1 { + b.WriteString(strings.Repeat("\t", colWd-1)) + } + } + b.WriteString("\n") + } + ctr := 0 + for r := range rows { + rc, _ := Projection2DCoords(sh, oddRow, r, 0) + if !noidx { + b.WriteString(fmt.Sprintf("%v:\t", rc)) + } + ri := r + for c := 0; c < cols; c++ { + s := "" + if tsr.IsString() { + s = fmt.Sprintf(format, Projection2DString(tsr, oddRow, ri, c)) + } else { + s = fmt.Sprintf(format, Projection2DValue(tsr, oddRow, ri, c)) + } + b.WriteString(s) + nt := int(math32.IntMultipleGE(float32(len(s)), 8)) / 8 + if nt < colWd { + b.WriteString(strings.Repeat("\t", nt-colWd)) + } + } + b.WriteString("\n") + ctr += cols + if ctr > maxLen { + b.WriteString("...\n") + break + } + } + return b.String() +} diff --git a/tensor/masked.go b/tensor/masked.go index 8de35cf84b..dee9053d4a 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -89,7 +89,7 @@ func (ms *Masked) SyncShape() { } func (ms *Masked) Label() string { return label(ms.Metadata().Name(), ms.Shape()) } -func (ms *Masked) String() string { return sprint(ms, 0) } +func (ms *Masked) String() string { return Sprintf(ms, 0, "") } func (ms *Masked) Metadata() *metadata.Data { return ms.Tensor.Metadata() } func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } func (ms *Masked) DataType() reflect.Kind { return ms.Tensor.DataType() } diff --git a/tensor/number.go b/tensor/number.go index abb62354f4..b6fc9940e4 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -95,7 +95,7 @@ func NewNumberFromValues[T num.Number](vals ...T) *Number[T] { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Number[T]) String() string { return sprint(tsr, 0) } +func (tsr *Number[T]) String() string { return Sprintf(tsr, 0, "") } func (tsr *Number[T]) IsString() bool { return false } diff --git a/tensor/reshaped.go b/tensor/reshaped.go index 5b86ca6a6c..df4f126ffb 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -143,7 +143,7 @@ func (rs *Reshaped) SetShapeSizes(sizes ...int) error { } func (rs *Reshaped) Label() string { return label(rs.Metadata().Name(), rs.Shape()) } -func (rs *Reshaped) String() string { return sprint(rs, 0) } +func (rs *Reshaped) String() string { return Sprintf(rs, 0, "") } func (rs *Reshaped) Metadata() *metadata.Data { return rs.Tensor.Metadata() } func (rs *Reshaped) IsString() bool { return rs.Tensor.IsString() } func (rs *Reshaped) DataType() reflect.Kind { return rs.Tensor.DataType() } diff --git a/tensor/rows.go b/tensor/rows.go index 835ff5384b..ec80da635f 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -86,7 +86,7 @@ func (rw *Rows) NumRows() int { return len(rw.Indexes) } -func (rw *Rows) String() string { return sprint(rw.Tensor, 0) } +func (rw *Rows) String() string { return Sprintf(rw.Tensor, 0, "") } func (rw *Rows) Label() string { return rw.Tensor.Label() } func (rw *Rows) Metadata() *metadata.Data { return rw.Tensor.Metadata() } func (rw *Rows) NumDims() int { return rw.Tensor.NumDims() } diff --git a/tensor/sliced.go b/tensor/sliced.go index c02df03a59..00429f2247 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -223,7 +223,7 @@ func (sl *Sliced) IndexesNeeded(d int) { } func (sl *Sliced) Label() string { return label(sl.Metadata().Name(), sl.Shape()) } -func (sl *Sliced) String() string { return sprint(sl, 0) } +func (sl *Sliced) String() string { return Sprintf(sl, 0, "") } func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } func (sl *Sliced) IsString() bool { return sl.Tensor.IsString() } func (sl *Sliced) DataType() reflect.Kind { return sl.Tensor.DataType() } diff --git a/tensor/slices.go b/tensor/slices.go index f396f9041d..0707f04eb7 100644 --- a/tensor/slices.go +++ b/tensor/slices.go @@ -158,19 +158,3 @@ func (sl Slice) IntTensor(size int) *Int { sl.ToIntSlice(size, tsr.Values) return tsr } - -// NewSliceInts returns a new [Int] [Tensor] with given [Slice] -// start, stop, step values (optional), for given dimension size. -func NewSliceInts(size int, svals ...int) *Int { - sl := Slice{} - if len(svals) > 0 { - sl.Start = svals[0] - } - if len(svals) > 1 { - sl.Stop = svals[1] - } - if len(svals) > 2 { - sl.Step = svals[2] - } - return sl.IntTensor(size) -} diff --git a/tensor/string.go b/tensor/string.go index 8c49b84e54..8aab94accc 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -50,7 +50,7 @@ func Float64ToString(val float64) string { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *String) String() string { - return sprint(tsr, 0) + return Sprintf(tsr, 0, "") } func (tsr *String) IsString() bool { diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 3c71ae6d67..a6930290c4 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -123,11 +123,5 @@ func Negate(a tensor.Tensor) tensor.Tensor { // NegateOut stores in the output the bool value -a. func NegateOut(a tensor.Tensor, out tensor.Values) error { - out.SetShapeSizes(a.Shape().Sizes...) - alen := a.Len() - tensor.VectorizeThreaded(1, func(tsr ...tensor.Tensor) int { return alen }, - func(idx int, tsr ...tensor.Tensor) { - out.SetFloat1D(-tsr[0].Float1D(idx), idx) - }, a, out) - return nil + return tensor.FloatFuncOut(1, func(in float64) float64 { return -in }, a, out) } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 3d3ba8e6f7..33a0028169 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -64,6 +64,7 @@ func init() { "FloatBinaryFuncOut": reflect.ValueOf(tensor.FloatBinaryFuncOut), "FloatFunc": reflect.ValueOf(tensor.FloatFunc), "FloatFuncOut": reflect.ValueOf(tensor.FloatFuncOut), + "FloatSetFunc": reflect.ValueOf(tensor.FloatSetFunc), "FullAxis": reflect.ValueOf(tensor.FullAxis), "FuncByName": reflect.ValueOf(tensor.FuncByName), "Funcs": reflect.ValueOf(&tensor.Funcs).Elem(), @@ -84,13 +85,16 @@ func init() { "NewFloat64FromValues": reflect.ValueOf(tensor.NewFloat64FromValues), "NewFloat64Full": reflect.ValueOf(tensor.NewFloat64Full), "NewFloat64Ones": reflect.ValueOf(tensor.NewFloat64Ones), + "NewFloat64Rand": reflect.ValueOf(tensor.NewFloat64Rand), "NewFloat64Scalar": reflect.ValueOf(tensor.NewFloat64Scalar), + "NewFloat64SpacedLinear": reflect.ValueOf(tensor.NewFloat64SpacedLinear), "NewFunc": reflect.ValueOf(tensor.NewFunc), "NewIndexed": reflect.ValueOf(tensor.NewIndexed), "NewInt": reflect.ValueOf(tensor.NewInt), "NewInt32": reflect.ValueOf(tensor.NewInt32), "NewIntFromValues": reflect.ValueOf(tensor.NewIntFromValues), "NewIntFull": reflect.ValueOf(tensor.NewIntFull), + "NewIntRange": reflect.ValueOf(tensor.NewIntRange), "NewIntScalar": reflect.ValueOf(tensor.NewIntScalar), "NewMasked": reflect.ValueOf(tensor.NewMasked), "NewOfType": reflect.ValueOf(tensor.NewOfType), @@ -99,7 +103,6 @@ func init() { "NewRows": reflect.ValueOf(tensor.NewRows), "NewShape": reflect.ValueOf(tensor.NewShape), "NewSlice": reflect.ValueOf(tensor.NewSlice), - "NewSliceInts": reflect.ValueOf(tensor.NewSliceInts), "NewSliced": reflect.ValueOf(tensor.NewSliced), "NewString": reflect.ValueOf(tensor.NewString), "NewStringFromValues": reflect.ValueOf(tensor.NewStringFromValues), @@ -137,7 +140,7 @@ func init() { "SlicesMagicN": reflect.ValueOf(tensor.SlicesMagicN), "SlicesMagicValues": reflect.ValueOf(tensor.SlicesMagicValues), "Space": reflect.ValueOf(tensor.Space), - "Sprint": reflect.ValueOf(tensor.Sprint), + "Sprintf": reflect.ValueOf(tensor.Sprintf), "Squeeze": reflect.ValueOf(tensor.Squeeze), "StableSort": reflect.ValueOf(tensor.StableSort), "StringAssignFunc": reflect.ValueOf(tensor.StringAssignFunc), From f23ce4755175d7840027788737449725bed5e085 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 16:23:02 -0700 Subject: [PATCH 147/311] linspace makes 1d for scalar inputs; fix key bug in sprintf --- goal/cmd/goal/testdata/test.goal | 3 +++ tensor/create.go | 11 ++++++++--- tensor/io.go | 12 +++++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/goal/cmd/goal/testdata/test.goal b/goal/cmd/goal/testdata/test.goal index f4c9b31248..ab8686e543 100644 --- a/goal/cmd/goal/testdata/test.goal +++ b/goal/cmd/goal/testdata/test.goal @@ -17,6 +17,9 @@ c := x * y + a * b fmt.Println(c) +l := linspace(0.1, 0.2, 5, true) +fmt.Println(l) + m := reshape(arange(36),6,6) if m[1,1] == 7 { diff --git a/tensor/create.go b/tensor/create.go index a2e1992cb0..0fe4b1c22e 100644 --- a/tensor/create.go +++ b/tensor/create.go @@ -161,9 +161,14 @@ func NewFloat64SpacedLinear(start, stop Tensor, num int, endpoint bool) *Float64 for i := range n { step.SetFloat1D((stop.Float1D(i)-start.Float1D(i))/fnum, i) } - tsz := slices.Clone(start.Shape().Sizes) - tsz = append([]int{num}, tsz...) - tsr := NewFloat64(tsz...) + var tsr *Float64 + if start.Len() == 1 { + tsr = NewFloat64(num) + } else { + tsz := slices.Clone(start.Shape().Sizes) + tsz = append([]int{num}, tsz...) + tsr = NewFloat64(tsz...) + } for r := range num { for i := range n { tsr.SetFloatRowCell(start.Float1D(i)+float64(r)*step.Float1D(i), r, i) diff --git a/tensor/io.go b/tensor/io.go index e5201ea4a1..0f735b9438 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -191,19 +191,20 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { format = "%7.3g\t" } } - mxlen := 0 - for i := range maxLen { + mxwd := 0 + n := min(tsr.Len(), maxLen) + for i := range n { s := "" if tsr.IsString() { s = fmt.Sprintf(format, tsr.String1D(i)) } else { s = fmt.Sprintf(format, tsr.Float1D(i)) } - if len(s) > mxlen { - mxlen = len(s) + if len(s) > mxwd { + mxwd = len(s) } } - colWd = int(math32.IntMultipleGE(float32(mxlen), 8)) / 8 + colWd = int(math32.IntMultipleGE(float32(mxwd), 8)) / 8 if colWd > 1 && !tsr.IsString() && defFmt { // should be 2 if isint { format = "%15g\t" @@ -234,6 +235,7 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { } b.WriteString("\n") } + // todo: could do something better for large numbers of columns.. ctr := 0 for r := range rows { rc, _ := Projection2DCoords(sh, oddRow, r, 0) From 732410e3c9d449bde8481bfc3aef596371580602 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 16:33:17 -0700 Subject: [PATCH 148/311] table updates --- goal/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/goal/README.md b/goal/README.md index 5aa8719087..33e76569b7 100644 --- a/goal/README.md +++ b/goal/README.md @@ -281,14 +281,12 @@ The following sections provide a full list of equivalents between the `tensor` G See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for details. -TODO: - | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| | |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector | -| | |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector | +| `tensor.NewIntRange(1, 11)` | same: |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector; `arange` in goal is always ints; use `linspace` or `tensor.AsFloat64` for floats | +| | same: |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector; 1 arg is the stop value in a slice | | | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | -| | |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | +| `tensor.NewFloat64SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | | | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | | | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | | | |`np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | @@ -357,7 +355,7 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | `tmath.Mul(a,b)` | same: |`a * b` | `a .* b` | element-wise multiply | | `tmath.Div(a,b)` | same: |`a/b` | `a./b` | element-wise divide | | `tmath.Mod(a,b)` | same: |`a%b` | `a./b` | element-wise modulous (works for float and int) | -| `tmath.Pow(a,3)` | same: | `a^3` or `a**3` | `a**3` | `a.^3` | element-wise exponentiation | +| `tmath.Pow(a,3)` | same: | `a**3` | `a.^3` | element-wise exponentiation | | `tmath.Cos(a)` | same: | `cos(a)` | `cos(a)` | element-wise function application | ## 2D Matrix Linear Algebra @@ -406,9 +404,11 @@ todo: huge amount of work needed to support complex numbers throughout! ## TODO -* # # surrounding in go, shell modes +* newaxis -> tensor.NewAxis constant; ... -> tensor.Ellipsis in Reslice * more creation routines * update readme table + +* # # surrounding in go, shell modes * record to datafs * make a simple tutorial example showing basic ops From 86dbc6e22c92cc9a5f088818698dfeba3de17e80 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 17:31:45 -0700 Subject: [PATCH 149/311] more runtime safety in range statement, re vars as idents --- goal/README.md | 2 +- goal/transpile/math.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/goal/README.md b/goal/README.md index 33e76569b7..639902a02b 100644 --- a/goal/README.md +++ b/goal/README.md @@ -286,7 +286,7 @@ See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for de | `tensor.NewIntRange(1, 11)` | same: |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector; `arange` in goal is always ints; use `linspace` or `tensor.AsFloat64` for floats | | | same: |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector; 1 arg is the stop value in a slice | | | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | -| `tensor.NewFloat64SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive | +| `t.NewFloat64``SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive of end (use `false` at end for exclusive) | | | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | | | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | | | |`np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index e4fb8ed63d..a96ef9ee02 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -278,9 +278,16 @@ func (mp *mathParse) stmt(st ast.Stmt) { fmt.Println("for range statement requires both index and value variables") return } - knm := x.Key.(*ast.Ident).Name - vnm := x.Value.(*ast.Ident).Name - enm := x.X.(*ast.Ident).Name + ki, _ := x.Key.(*ast.Ident) + vi, _ := x.Value.(*ast.Ident) + ei, _ := x.X.(*ast.Ident) + if ki == nil || vi == nil || ei == nil { + fmt.Println("for range statement requires all variables (index, value, range) to be variable names, not other expressions") + return + } + knm := ki.Name + vnm := vi.Name + enm := ei.Name mp.addToken(token.FOR) mp.expr(x.Key) From c5a5c0fb15aca24d669850aa39139a227aa72df9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 17:33:56 -0700 Subject: [PATCH 150/311] table --- goal/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/goal/README.md b/goal/README.md index 639902a02b..68d2af4585 100644 --- a/goal/README.md +++ b/goal/README.md @@ -286,7 +286,7 @@ See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for de | `tensor.NewIntRange(1, 11)` | same: |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector; `arange` in goal is always ints; use `linspace` or `tensor.AsFloat64` for floats | | | same: |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector; 1 arg is the stop value in a slice | | | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | -| `t.NewFloat64``SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive of end (use `false` at end for exclusive) | +| `t.NewFloat64` `SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive of end (use `false` at end for exclusive) | | | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | | | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | | | |`np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | From 7feda229a63033c8c3db82b799fd8d3ca3402d8e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 17:35:10 -0700 Subject: [PATCH 151/311] table --- goal/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/goal/README.md b/goal/README.md index 68d2af4585..031191ac47 100644 --- a/goal/README.md +++ b/goal/README.md @@ -287,10 +287,10 @@ See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for de | | same: |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector; 1 arg is the stop value in a slice | | | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | | `t.NewFloat64` `SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive of end (use `false` at end for exclusive) | -| | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | -| | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],np.r_[0:6.]` | | the best way to eval functions on a grid | -| | |`np.meshgrid([1,2,4],[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | -| | |`np.ix_([1,2,4],[2,4,5])` | | the best way to eval functions on a grid | +| | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],` `r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | +| | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],` `np.r_[0:6.]` | | the best way to eval functions on a grid | +| | |`np.meshgrid([1,2,4],` `[2,4,5])` | `[x,y]=meshgrid([1,2,4],[2,4,5])` | | +| | |`np.ix_([1,2,4],` `[2,4,5])` | | the best way to eval functions on a grid | ## Basic indexing From 36fd707e72a4c6f3af13f4a4f284390ddb5d6215 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 27 Sep 2024 21:03:48 -0700 Subject: [PATCH 152/311] start on math datafs interface -- special limited command set for math mode --- goal/README.md | 2 +- goal/goal.go | 1 + goal/transpile/datafs.go | 45 +++++++++++++++++++ goal/transpile/math.go | 38 +++++++++++++--- goal/transpile/state.go | 6 ++- goal/transpile/transpile.go | 6 --- goal/transpile/transpile_test.go | 4 +- tensor/datafs/data.go | 11 +++++ tensor/datafs/dir.go | 37 +++++++++++++++ .../cogentcore_org-core-tensor-datafs.go | 4 ++ 10 files changed, 138 insertions(+), 16 deletions(-) create mode 100644 goal/transpile/datafs.go diff --git a/goal/README.md b/goal/README.md index 031191ac47..dc14c5de7c 100644 --- a/goal/README.md +++ b/goal/README.md @@ -285,7 +285,7 @@ See [NumPy](https://numpy.org/doc/stable/user/how-to-partition.html) docs for de | ------------ | ----------- | ------ | ------ | ---------------- | | `tensor.NewIntRange(1, 11)` | same: |`np.arange(1., 11.)` or `np.r_[1.:11.]` or `np.r_[1:10:10j]` | `1:10` | create an increasing vector; `arange` in goal is always ints; use `linspace` or `tensor.AsFloat64` for floats | | | same: |`np.arange(10.)` or `np.r_[:10.]` or `np.r_[:9:10j]` | `0:9` | create an increasing vector; 1 arg is the stop value in a slice | -| | |`np.arange(1.,11.)[:, np.newaxis]` | `[1:10]'` | create a column vector | +| | |`np.arange(1.,11.)` `[:, np.newaxis]` | `[1:10]'` | create a column vector | | `t.NewFloat64` `SpacedLinear(` `1, 3, 4, true)` | `linspace(1,3,4,true)` |`np.linspace(1,3,4)` | `linspace(1,3,4)` | 4 equally spaced samples between 1 and 3, inclusive of end (use `false` at end for exclusive) | | | |`np.mgrid[0:9.,0:6.]` or `np.meshgrid(r_[0:9.],` `r_[0:6.])` | `[x,y]=meshgrid(0:8,0:5)` | two 2D tensors: one of x values, the other of y values | | | |`ogrid[0:9.,0:6.]` or `np.ix_(np.r_[0:9.],` `np.r_[0:6.]` | | the best way to eval functions on a grid | diff --git a/goal/goal.go b/goal/goal.go index 7e57e493ad..edf7f517f8 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -102,6 +102,7 @@ func NewGoal() *Goal { }, } gl.TrState.FuncToVar = true + gl.TrState.MathRecord = true gl.Config.StdIO.SetFromOS() gl.SSH = sshclient.NewConfig(&gl.Config) gl.SSHClients = make(map[string]*sshclient.Client) diff --git a/goal/transpile/datafs.go b/goal/transpile/datafs.go new file mode 100644 index 0000000000..b751edbef2 --- /dev/null +++ b/goal/transpile/datafs.go @@ -0,0 +1,45 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package transpile + +import ( + "errors" + "fmt" + + "cogentcore.org/core/tensor/datafs" +) + +var datafsCommands = map[string]func(mp *mathParse) error{ + "cd": cd, + "mkdir": mkdir, + "ls": ls, +} + +func cd(mp *mathParse) error { + var dir string + if len(mp.ewords) > 1 { + dir = mp.ewords[1] + } + return datafs.Chdir(dir) +} + +func mkdir(mp *mathParse) error { + if len(mp.ewords) == 1 { + return errors.New("datafs mkdir requires a directory name") + } + dir := mp.ewords[1] + _, err := datafs.CurDir.Mkdir(dir) + return err +} + +func ls(mp *mathParse) error { + var dir string + if len(mp.ewords) > 1 { + dir = mp.ewords[1] + } + _ = dir + fmt.Println(datafs.CurDir.ListShort(false, 0)) + return nil +} diff --git a/goal/transpile/math.go b/goal/transpile/math.go index a96ef9ee02..73103aad05 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -14,7 +14,9 @@ import ( "cogentcore.org/core/tensor" ) -func MathParse(toks Tokens, code string, fullLine bool) Tokens { +// TranspileMath does math mode transpiling. fullLine indicates code should be +// full statement(s). +func (st *State) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { nt := len(toks) if nt == 0 { return nil @@ -26,12 +28,24 @@ func MathParse(toks Tokens, code string, fullLine bool) Tokens { str += toks[nt-1].Str[1:] } // fmt.Println(str) - mp := mathParse{toks: toks, code: code} + mp := mathParse{state: st, toks: toks, code: code} // mp.trace = true mods := AllErrors // | Trace if fullLine { + ewords, err := ExecWords(str) + if len(ewords) > 0 { + if cmd, ok := datafsCommands[ewords[0]]; ok { + mp.ewords = ewords + err := cmd(&mp) + if err != nil { + fmt.Println(ewords[0]+":", err.Error()) + } + return nil + } + } + stmts, err := ParseLine(str, mods) if err != nil { fmt.Println("line code:", str) @@ -69,11 +83,13 @@ type funcInfo struct { // mathParse has the parsing state type mathParse struct { - code string // code string - toks Tokens // source tokens we are parsing - idx int // current index in source tokens -- critical to sync as we "use" source - out Tokens // output tokens we generate - trace bool // trace of parsing -- turn on to see alignment + state *State + code string // code string + toks Tokens // source tokens we are parsing + ewords []string // exec words + idx int // current index in source tokens -- critical to sync as we "use" source + out Tokens // output tokens we generate + trace bool // trace of parsing -- turn on to see alignment // stack of function info -- top of stack reflects the current function funcs stack.Stack[*funcInfo] @@ -519,11 +535,19 @@ func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { } func (mp *mathParse) defineStmt(as *ast.AssignStmt) { + firstStmt := mp.idx == 0 mp.exprList(as.Lhs) mp.addToken(as.Tok) mp.startFunc("") // dummy single arg tensor function mp.exprList(as.Rhs) mp.endFunc() + if firstStmt && mp.state.MathRecord { + nvar, ok := as.Lhs[0].(*ast.Ident) + if ok { + mp.out.Add(token.SEMICOLON) + mp.out.Add(token.IDENT, "datafs.Record("+nvar.Name+",`"+nvar.Name+"`)") + } + } } func (mp *mathParse) assignStmt(as *ast.AssignStmt) { diff --git a/goal/transpile/state.go b/goal/transpile/state.go index ad687ab3ad..4dd8a77f69 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -30,6 +30,10 @@ type State struct { // MathMode is on when math mode is turned on. MathMode bool + // MathRecord is state of auto-recording of data into current directory + // in math mode. + MathRecord bool + // depth of delim at the end of the current line. if 0, was complete. ParenDepth, BraceDepth, BrackDepth, TypeDepth, DeclDepth int @@ -52,7 +56,7 @@ type State struct { // NewState returns a new transpiling state; mostly for testing func NewState() *State { - st := &State{FuncToVar: true} + st := &State{FuncToVar: true, MathRecord: true} return st } diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 79a58d9869..1bb6f12d19 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -298,9 +298,3 @@ func (st *State) TranspileExec(ewords []string, output bool) Tokens { endExec() return etoks } - -// TranspileMath does math mode transpiling. fullLine indicates code should be -// full statement(s). -func (st *State) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { - return MathParse(toks, code, fullLine) -} diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index de3a3ea642..ff48b98f2c 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -207,6 +207,7 @@ func TestCur(t *testing.T) { {"# a := linspace(0, 5, 6, true)", `a := tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, } st := NewState() + st.MathRecord = false for _, test := range tests { o := st.TranspileLine(test.i) assert.Equal(t, test.e, o) @@ -229,7 +230,7 @@ func TestMath(t *testing.T) { {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, {"# a := arange(36)", `a := tensor.NewIntRange(36)`}, - {"# a := arange(36, 0, -1)", `a := tensor.NewIntRange(36, 0, - 1)`}, + {"# a := arange(36, 0, -1)", `a := tensor.NewIntRange(36, 0, - 1)`}, {"# a := linspace(0, 5, 6, true)", `a := tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, @@ -265,6 +266,7 @@ func TestMath(t *testing.T) { } st := NewState() + st.MathRecord = false for _, test := range tests { o := st.TranspileLine(test.i) assert.Equal(t, test.e, o) diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 5eb18c67eb..0a213b8de7 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -107,6 +107,17 @@ func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Val return tsr } +// NewData creates a new Data node for given tensor with given name. +// returns an error if the data name already exists +func (d *Data) NewData(tsr tensor.Tensor, name string) (*Data, error) { + nd, err := newData(d, name) + if err != nil { + return nil, err + } + nd.Data = tsr + return nd, nil +} + func (d *Data) KnownFileInfo() fileinfo.Known { if d.Data == nil { return fileinfo.Unknown diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 9e0b70dc7e..cb302dc439 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -18,6 +18,43 @@ import ( "cogentcore.org/core/tensor/table" ) +var ( + // CurDir is the current working directory. + CurDir *Data + + // CurRoot is the current root datafs system. + // A default root datafs is created at startup. + CurRoot *Data +) + +func init() { + CurRoot, _ = NewDir("root") + CurDir = CurRoot +} + +// Record saves given tensor to current directory with given name. +func Record(tsr tensor.Tensor, name string) error { + _, err := CurDir.NewData(tsr, name) + return err // todo: could prompt about conficts, or always overwrite existing? +} + +// Chdir changes the current working datafs directory to the named directory. +func Chdir(dir string) error { + if CurDir == nil { + CurDir = CurRoot + } + if dir == "" { + CurDir = CurRoot + return nil + } + ndir, err := CurDir.DirAtPath(dir) + if err != nil { + return err + } + CurDir = ndir + return nil +} + // Dir is a map of directory entry names to Data nodes. // It retains the order that items were added in, which is // the natural order items are processed in. diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go b/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go index bec8384506..44b232809a 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go @@ -10,11 +10,15 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/datafs/datafs"] = map[string]reflect.Value{ // function, constant and variable definitions + "Chdir": reflect.ValueOf(datafs.Chdir), + "CurDir": reflect.ValueOf(&datafs.CurDir).Elem(), + "CurRoot": reflect.ValueOf(&datafs.CurRoot).Elem(), "DirOnly": reflect.ValueOf(datafs.DirOnly), "Long": reflect.ValueOf(datafs.Long), "NewDir": reflect.ValueOf(datafs.NewDir), "Overwrite": reflect.ValueOf(datafs.Overwrite), "Preserve": reflect.ValueOf(datafs.Preserve), + "Record": reflect.ValueOf(datafs.Record), "Recursive": reflect.ValueOf(datafs.Recursive), "Short": reflect.ValueOf(datafs.Short), From e2739170aaa350bda5e30a24100114cfa33155ab Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 28 Sep 2024 00:52:55 -0700 Subject: [PATCH 153/311] datafs integration with goal first pass working, planets example runs interactive goal. --- goal/cmd/goal/goal.go | 111 +------------ goal/cmd/goal/typegen.go | 15 -- goal/interpreter/config.go | 119 +++++++++++++ goal/interpreter/imports.go | 2 - goal/interpreter/interpreter.go | 5 +- goal/interpreter/typegen.go | 21 +++ goal/transpile/datafs.go | 27 +-- goal/transpile/math.go | 16 +- goal/transpile/token.go | 7 + tensor/datafs/commands.go | 157 ++++++++++++++++++ tensor/datafs/dir.go | 37 ----- tensor/datafs/list.go | 20 ++- tensor/examples/planets/planets.go | 22 ++- .../cogentcore_org-core-tensor-datafs.go | 4 + 14 files changed, 373 insertions(+), 190 deletions(-) delete mode 100644 goal/cmd/goal/typegen.go create mode 100644 goal/interpreter/config.go create mode 100644 goal/interpreter/typegen.go create mode 100644 tensor/datafs/commands.go diff --git a/goal/cmd/goal/goal.go b/goal/cmd/goal/goal.go index a692a21990..7aeced5f8e 100644 --- a/goal/cmd/goal/goal.go +++ b/goal/cmd/goal/goal.go @@ -6,118 +6,13 @@ package main import ( - "fmt" - "os" - "path/filepath" - "strings" - - "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/fsx" "cogentcore.org/core/cli" - "cogentcore.org/core/goal" "cogentcore.org/core/goal/interpreter" - "github.com/cogentcore/yaegi/interp" ) -//go:generate core generate -add-types -add-funcs - -// Config is the configuration information for the goal cli. -type Config struct { - - // Input is the input file to run/compile. - // If this is provided as the first argument, - // then the program will exit after running, - // unless the Interactive mode is flagged. - Input string `posarg:"0" required:"-"` - - // Expr is an optional expression to evaluate, which can be used - // in addition to the Input file to run, to execute commands - // defined within that file for example, or as a command to run - // prior to starting interactive mode if no Input is specified. - Expr string `flag:"e,expr"` - - // Args is an optional list of arguments to pass in the run command. - // These arguments will be turned into an "args" local variable in the goal. - // These are automatically processed from any leftover arguments passed, so - // you should not need to specify this flag manually. - Args []string `cmd:"run" posarg:"leftover" required:"-"` - - // Interactive runs the interactive command line after processing any input file. - // Interactive mode is the default mode for the run command unless an input file - // is specified. - Interactive bool `cmd:"run" flag:"i,interactive"` -} - func main() { //types:skip opts := cli.DefaultOptions("goal", "An interactive tool for running and compiling Goal (Go augmented language).") - cli.Run(opts, &Config{}, Run, Build) -} - -// Run runs the specified goal file. If no file is specified, -// it runs an interactive shell that allows the user to input goal. -func Run(c *Config) error { //cli:cmd -root - in := interpreter.NewInterpreter(interp.Options{}) - in.Config() - if len(c.Args) > 0 { - in.Eval("args := goalib.StringsToAnys(" + fmt.Sprintf("%#v)", c.Args)) - } - - if c.Input == "" { - return Interactive(c, in) - } - code := "" - if errors.Log1(fsx.FileExists(c.Input)) { - b, err := os.ReadFile(c.Input) - if err != nil && c.Expr == "" { - return err - } - code = string(b) - } - if c.Expr != "" { - if code != "" { - code += "\n" - } - code += c.Expr + "\n" - } - - _, _, err := in.Eval(code) - if err == nil { - err = in.Goal.TrState.DepthError() - } - if c.Interactive { - return Interactive(c, in) - } - return err -} - -// Interactive runs an interactive shell that allows the user to input goal. -func Interactive(c *Config, in *interpreter.Interpreter) error { - if c.Expr != "" { - in.Eval(c.Expr) - } - in.Interactive() - return nil -} - -// Build builds the specified input goal file, or all .goal files in the current -// directory if no input is specified, to corresponding .go file name(s). -// If the file does not already contain a "package" specification, then -// "package main; func main()..." wrappers are added, which allows the same -// code to be used in interactive and Go compiled modes. -func Build(c *Config) error { - var fns []string - if c.Input != "" { - fns = []string{c.Input} - } else { - fns = fsx.Filenames(".", ".goal") - } - var errs []error - for _, fn := range fns { - ofn := strings.TrimSuffix(fn, filepath.Ext(fn)) + ".go" - err := goal.NewGoal().TranspileFile(fn, ofn) - if err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) + cfg := &interpreter.Config{} + cfg.InteractiveFunc = interpreter.Interactive + cli.Run(opts, cfg, interpreter.Run, interpreter.Build) } diff --git a/goal/cmd/goal/typegen.go b/goal/cmd/goal/typegen.go deleted file mode 100644 index 966f3334df..0000000000 --- a/goal/cmd/goal/typegen.go +++ /dev/null @@ -1,15 +0,0 @@ -// Code generated by "core generate -add-types -add-funcs"; DO NOT EDIT. - -package main - -import ( - "cogentcore.org/core/types" -) - -var _ = types.AddType(&types.Type{Name: "main.Config", IDName: "config", Doc: "Config is the configuration information for the goal cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "Input", Doc: "Input is the input file to run/compile.\nIf this is provided as the first argument,\nthen the program will exit after running,\nunless the Interactive mode is flagged."}, {Name: "Expr", Doc: "Expr is an optional expression to evaluate, which can be used\nin addition to the Input file to run, to execute commands\ndefined within that file for example, or as a command to run\nprior to starting interactive mode if no Input is specified."}, {Name: "Args", Doc: "Args is an optional list of arguments to pass in the run command.\nThese arguments will be turned into an \"args\" local variable in the goal.\nThese are automatically processed from any leftover arguments passed, so\nyou should not need to specify this flag manually."}, {Name: "Interactive", Doc: "Interactive runs the interactive command line after processing any input file.\nInteractive mode is the default mode for the run command unless an input file\nis specified."}}}) - -var _ = types.AddFunc(&types.Func{Name: "main.Run", Doc: "Run runs the specified goal file. If no file is specified,\nit runs an interactive shell that allows the user to input goal.", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}}, Args: []string{"c"}, Returns: []string{"error"}}) - -var _ = types.AddFunc(&types.Func{Name: "main.Interactive", Doc: "Interactive runs an interactive shell that allows the user to input goal.", Args: []string{"c", "in"}, Returns: []string{"error"}}) - -var _ = types.AddFunc(&types.Func{Name: "main.Build", Doc: "Build builds the specified input goal file, or all .goal files in the current\ndirectory if no input is specified, to corresponding .go file name(s).\nIf the file does not already contain a \"package\" specification, then\n\"package main; func main()...\" wrappers are added, which allows the same\ncode to be used in interactive and Go compiled modes.", Args: []string{"c"}, Returns: []string{"error"}}) diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go new file mode 100644 index 0000000000..563b3bd6f8 --- /dev/null +++ b/goal/interpreter/config.go @@ -0,0 +1,119 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package interpreter + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/fsx" + "cogentcore.org/core/goal" + "github.com/cogentcore/yaegi/interp" +) + +//go:generate core generate -add-types -add-funcs + +// Config is the configuration information for the goal cli. +type Config struct { + + // Input is the input file to run/compile. + // If this is provided as the first argument, + // then the program will exit after running, + // unless the Interactive mode is flagged. + Input string `posarg:"0" required:"-"` + + // Expr is an optional expression to evaluate, which can be used + // in addition to the Input file to run, to execute commands + // defined within that file for example, or as a command to run + // prior to starting interactive mode if no Input is specified. + Expr string `flag:"e,expr"` + + // Args is an optional list of arguments to pass in the run command. + // These arguments will be turned into an "args" local variable in the goal. + // These are automatically processed from any leftover arguments passed, so + // you should not need to specify this flag manually. + Args []string `cmd:"run" posarg:"leftover" required:"-"` + + // Interactive runs the interactive command line after processing any input file. + // Interactive mode is the default mode for the run command unless an input file + // is specified. + Interactive bool `cmd:"run" flag:"i,interactive"` + + // InteractiveFunc is the function to run in interactive mode. + // set it to your own function as needed. + InteractiveFunc func(c *Config, in *Interpreter) error +} + +// Run runs the specified goal file. If no file is specified, +// it runs an interactive shell that allows the user to input goal. +func Run(c *Config) error { //cli:cmd -root + in := NewInterpreter(interp.Options{}) + in.Config() + if len(c.Args) > 0 { + in.Eval("args := goalib.StringsToAnys(" + fmt.Sprintf("%#v)", c.Args)) + } + + if c.Input == "" { + return c.InteractiveFunc(c, in) + } + code := "" + if errors.Log1(fsx.FileExists(c.Input)) { + b, err := os.ReadFile(c.Input) + if err != nil && c.Expr == "" { + return err + } + code = string(b) + } + if c.Expr != "" { + if code != "" { + code += "\n" + } + code += c.Expr + "\n" + } + + _, _, err := in.Eval(code) + if err == nil { + err = in.Goal.TrState.DepthError() + } + if c.Interactive { + return c.InteractiveFunc(c, in) + } + return err +} + +// Interactive runs an interactive shell that allows the user to input goal. +func Interactive(c *Config, in *Interpreter) error { + if c.Expr != "" { + in.Eval(c.Expr) + } + in.Interactive() + return nil +} + +// Build builds the specified input goal file, or all .goal files in the current +// directory if no input is specified, to corresponding .go file name(s). +// If the file does not already contain a "package" specification, then +// "package main; func main()..." wrappers are added, which allows the same +// code to be used in interactive and Go compiled modes. +func Build(c *Config) error { + var fns []string + if c.Input != "" { + fns = []string{c.Input} + } else { + fns = fsx.Filenames(".", ".goal") + } + var errs []error + for _, fn := range fns { + ofn := strings.TrimSuffix(fn, filepath.Ext(fn)) + ".go" + err := goal.NewGoal().TranspileFile(fn, ofn) + if err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} diff --git a/goal/interpreter/imports.go b/goal/interpreter/imports.go index 4ef7fd4635..b46f9f789e 100644 --- a/goal/interpreter/imports.go +++ b/goal/interpreter/imports.go @@ -4,8 +4,6 @@ package interpreter -//go:generate ./make - import ( "reflect" diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index 09efaab334..23f5101cde 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -18,6 +18,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/goal" + "cogentcore.org/core/tensor/datafs" _ "cogentcore.org/core/tensor/stats/metric" _ "cogentcore.org/core/tensor/stats/stats" _ "cogentcore.org/core/tensor/tmath" @@ -77,11 +78,13 @@ func NewInterpreter(options interp.Options) *Interpreter { func (in *Interpreter) Prompt() string { dp := in.Goal.TrState.TotalDepth() pc := ">" + dir := in.Goal.HostAndDir() if in.Goal.TrState.MathMode { pc = "#" + dir = datafs.CurDir.Path() } if dp == 0 { - return in.Goal.HostAndDir() + " " + pc + " " + return dir + " " + pc + " " } res := pc + " " for range dp { diff --git a/goal/interpreter/typegen.go b/goal/interpreter/typegen.go new file mode 100644 index 0000000000..c0a0961344 --- /dev/null +++ b/goal/interpreter/typegen.go @@ -0,0 +1,21 @@ +// Code generated by "core generate -add-types -add-funcs"; DO NOT EDIT. + +package interpreter + +import ( + "cogentcore.org/core/types" +) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/interpreter.Config", IDName: "config", Doc: "Config is the configuration information for the goal cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "Input", Doc: "Input is the input file to run/compile.\nIf this is provided as the first argument,\nthen the program will exit after running,\nunless the Interactive mode is flagged."}, {Name: "Expr", Doc: "Expr is an optional expression to evaluate, which can be used\nin addition to the Input file to run, to execute commands\ndefined within that file for example, or as a command to run\nprior to starting interactive mode if no Input is specified."}, {Name: "Args", Doc: "Args is an optional list of arguments to pass in the run command.\nThese arguments will be turned into an \"args\" local variable in the goal.\nThese are automatically processed from any leftover arguments passed, so\nyou should not need to specify this flag manually."}, {Name: "Interactive", Doc: "Interactive runs the interactive command line after processing any input file.\nInteractive mode is the default mode for the run command unless an input file\nis specified."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/interpreter.Interpreter", IDName: "interpreter", Doc: "Interpreter represents one running shell context", Fields: []types.Field{{Name: "Goal", Doc: "the goal shell"}, {Name: "HistFile", Doc: "HistFile is the name of the history file to open / save.\nDefaults to ~/.goal-history for the default goal shell.\nUpdate this prior to running Config() to take effect."}, {Name: "Interp", Doc: "the yaegi interpreter"}}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Run", Doc: "Run runs the specified goal file. If no file is specified,\nit runs an interactive shell that allows the user to input goal.", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}}, Args: []string{"c"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Interactive", Doc: "Interactive runs an interactive shell that allows the user to input goal.", Args: []string{"c", "in"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Build", Doc: "Build builds the specified input goal file, or all .goal files in the current\ndirectory if no input is specified, to corresponding .go file name(s).\nIf the file does not already contain a \"package\" specification, then\n\"package main; func main()...\" wrappers are added, which allows the same\ncode to be used in interactive and Go compiled modes.", Args: []string{"c"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.init"}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.NewInterpreter", Doc: "NewInterpreter returns a new [Interpreter] initialized with the given options.\nIt automatically imports the standard library and configures necessary shell\nfunctions. End user app must call [Interp.Config] after importing any additional\nsymbols, prior to running the interpreter.", Args: []string{"options"}, Returns: []string{"Interpreter"}}) diff --git a/goal/transpile/datafs.go b/goal/transpile/datafs.go index b751edbef2..7e188cf0bc 100644 --- a/goal/transpile/datafs.go +++ b/goal/transpile/datafs.go @@ -6,9 +6,7 @@ package transpile import ( "errors" - "fmt" - - "cogentcore.org/core/tensor/datafs" + "go/token" ) var datafsCommands = map[string]func(mp *mathParse) error{ @@ -22,7 +20,11 @@ func cd(mp *mathParse) error { if len(mp.ewords) > 1 { dir = mp.ewords[1] } - return datafs.Chdir(dir) + mp.out.Add(token.IDENT, "datafs.Chdir") + mp.out.Add(token.LPAREN) + mp.out.Add(token.STRING, `"`+dir+`"`) + mp.out.Add(token.RPAREN) + return nil } func mkdir(mp *mathParse) error { @@ -30,16 +32,19 @@ func mkdir(mp *mathParse) error { return errors.New("datafs mkdir requires a directory name") } dir := mp.ewords[1] - _, err := datafs.CurDir.Mkdir(dir) - return err + mp.out.Add(token.IDENT, "datafs.Mkdir") + mp.out.Add(token.LPAREN) + mp.out.Add(token.STRING, `"`+dir+`"`) + mp.out.Add(token.RPAREN) + return nil } func ls(mp *mathParse) error { - var dir string - if len(mp.ewords) > 1 { - dir = mp.ewords[1] + mp.out.Add(token.IDENT, "datafs.List") + mp.out.Add(token.LPAREN) + for i := 1; i < len(mp.ewords); i++ { + mp.out.Add(token.STRING, `"`+mp.ewords[i]+`"`) } - _ = dir - fmt.Println(datafs.CurDir.ListShort(false, 0)) + mp.out.Add(token.RPAREN) return nil } diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 73103aad05..bed2e6cebb 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -41,8 +41,10 @@ func (st *State) TranspileMath(toks Tokens, code string, fullLine bool) Tokens { err := cmd(&mp) if err != nil { fmt.Println(ewords[0]+":", err.Error()) + return nil + } else { + return mp.out } - return nil } } @@ -317,14 +319,12 @@ func (mp *mathParse) stmt(st ast.Stmt) { mp.out.Add(token.PERIOD) mp.out.Add(token.IDENT, "Len") mp.idx++ - mp.out.Add(token.LPAREN) - mp.out.Add(token.RPAREN) + mp.out.AddMulti(token.LPAREN, token.RPAREN) mp.idx++ mp.out.Add(token.SEMICOLON) mp.idx++ mp.out.Add(token.IDENT, knm) - mp.out.Add(token.INC) - mp.out.Add(token.LBRACE) + mp.out.AddMulti(token.INC, token.LBRACE) mp.out.Add(token.IDENT, vnm) mp.out.Add(token.DEFINE) @@ -765,8 +765,7 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { mp.addToken(token.LBRACE) mp.exprList(il.Indices) mp.addToken(token.RBRACE) - mp.out.Add(token.ELLIPSIS) - mp.out.Add(token.RPAREN) + mp.out.AddMulti(token.ELLIPSIS, token.RPAREN) mp.endFunc() } @@ -895,8 +894,7 @@ func (mp *mathParse) ident(id *ast.Ident) { mp.out.Add(token.IDENT, "tensor.AsIntSlice") mp.out.Add(token.LPAREN) mp.addCur() - mp.out.Add(token.RPAREN) - mp.out.Add(token.ELLIPSIS) + mp.out.AddMulti(token.RPAREN, token.ELLIPSIS) } else { mp.addCur() } diff --git a/goal/transpile/token.go b/goal/transpile/token.go index a409ac7487..126adbd144 100644 --- a/goal/transpile/token.go +++ b/goal/transpile/token.go @@ -71,6 +71,13 @@ func (tk *Tokens) Add(tok token.Token, str ...string) *Token { return nt } +// AddMulti adds new basic tokens (not IDENT). +func (tk *Tokens) AddMulti(tok ...token.Token) { + for _, t := range tok { + tk.Add(t) + } +} + // AddTokens adds given tokens to our list func (tk *Tokens) AddTokens(toks ...*Token) *Tokens { *tk = append(*tk, toks...) diff --git a/tensor/datafs/commands.go b/tensor/datafs/commands.go new file mode 100644 index 0000000000..203c4c39bb --- /dev/null +++ b/tensor/datafs/commands.go @@ -0,0 +1,157 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package datafs + +import ( + "fmt" + "io/fs" + "path" + "strings" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" +) + +var ( + // CurDir is the current working directory. + CurDir *Data + + // CurRoot is the current root datafs system. + // A default root datafs is created at startup. + CurRoot *Data +) + +func init() { + CurRoot, _ = NewDir("root") + CurDir = CurRoot +} + +// Record saves given tensor to current directory with given name. +func Record(tsr tensor.Tensor, name string) error { + _, err := CurDir.NewData(tsr, name) + return err // todo: could prompt about conficts, or always overwrite existing? +} + +// Chdir changes the current working datafs directory to the named directory. +func Chdir(dir string) error { + if CurDir == nil { + CurDir = CurRoot + } + if dir == "" { + CurDir = CurRoot + return nil + } + ndir, err := CurDir.DirAtPath(dir) + if err != nil { + return err + } + CurDir = ndir + return nil +} + +// Mkdir creates a new directory with the specified name +// in the current directory. +func Mkdir(dir string) *Data { + if CurDir == nil { + CurDir = CurRoot + } + if dir == "" { + err := &fs.PathError{Op: "Mkdir", Path: dir, Err: errors.New("path must not be empty")} + errors.Log(err) + return nil + } + return errors.Log1(CurDir.Mkdir(dir)) +} + +// List lists files using arguments (options and path) from the current directory. +func List(opts ...string) error { + if CurDir == nil { + CurDir = CurRoot + } + + long := false + recursive := false + if len(opts) > 0 && len(opts[0]) > 0 && opts[0][0] == '-' { + op := opts[0] + if strings.Contains(op, "l") { + long = true + } + if strings.Contains(op, "r") { + recursive = true + } + opts = opts[1:] + } + dir := CurDir + if len(opts) > 0 { + nd, err := CurDir.DirAtPath(opts[0]) + if err == nil { + dir = nd + } + } + ls := dir.List(long, recursive) + fmt.Println(ls) + return nil +} + +// Get returns the data item as a tensor at given path +// relative to the current working directory. +// This is the direct pointer to the data item, so changes +// to it will change the data item. Clone the data to make +// a new copy disconnected from the original. +func Get(name string) tensor.Tensor { + if CurDir == nil { + CurDir = CurRoot + } + if name == "" { + err := &fs.PathError{Op: "Get", Path: name, Err: errors.New("name must not be empty")} + errors.Log(err) + return nil + } + d, err := CurDir.ItemAtPath(name) + if errors.Log(err) != nil { + return nil + } + if d.IsDir() { + err := &fs.PathError{Op: "Get", Path: name, Err: errors.New("item is a directory, not a data item")} + errors.Log(err) + return nil + } + return d.Data +} + +// Set sets tensor data item to given data item name or path +// relative to the current working directory. +// If the item already exists, its previous data tensor is +// updated to the given one; if it doesn't, then a new data +// item is created. +func Set(name string, tsr tensor.Tensor) error { + if CurDir == nil { + CurDir = CurRoot + } + if name == "" { + err := &fs.PathError{Op: "Set", Path: name, Err: errors.New("name must not be empty")} + return errors.Log(err) + } + itm, err := CurDir.ItemAtPath(name) + if err == nil { + if itm.IsDir() { + err := &fs.PathError{Op: "Set", Path: name, Err: errors.New("existing item is a directory, not a data item")} + return errors.Log(err) + } + itm.Data = tsr + return nil + } + cd := CurDir + dir, name := path.Split(name) + if dir != "" { + d, err := CurDir.DirAtPath(dir) + if err != nil { + return errors.Log(err) + } + cd = d + } + _, err = cd.NewData(tsr, name) + return errors.Log(err) +} diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index cb302dc439..9e0b70dc7e 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -18,43 +18,6 @@ import ( "cogentcore.org/core/tensor/table" ) -var ( - // CurDir is the current working directory. - CurDir *Data - - // CurRoot is the current root datafs system. - // A default root datafs is created at startup. - CurRoot *Data -) - -func init() { - CurRoot, _ = NewDir("root") - CurDir = CurRoot -} - -// Record saves given tensor to current directory with given name. -func Record(tsr tensor.Tensor, name string) error { - _, err := CurDir.NewData(tsr, name) - return err // todo: could prompt about conficts, or always overwrite existing? -} - -// Chdir changes the current working datafs directory to the named directory. -func Chdir(dir string) error { - if CurDir == nil { - CurDir = CurRoot - } - if dir == "" { - CurDir = CurRoot - return nil - } - ndir, err := CurDir.DirAtPath(dir) - if err != nil { - return err - } - CurDir = ndir - return nil -} - // Dir is a map of directory entry names to Data nodes. // It retains the order that items were added in, which is // the natural order items are processed in. diff --git a/tensor/datafs/list.go b/tensor/datafs/list.go index 5fc375d5a6..2932bf99db 100644 --- a/tensor/datafs/list.go +++ b/tensor/datafs/list.go @@ -4,7 +4,11 @@ package datafs -import "strings" +import ( + "strings" + + "cogentcore.org/core/base/indent" +) const ( Short = false @@ -30,13 +34,14 @@ func (d *Data) List(long, recursive bool) string { return d.ListShort(recursive, 0) } -func (d *Data) ListShort(recursive bool, indent int) string { +func (d *Data) ListShort(recursive bool, ident int) string { var b strings.Builder items := d.ItemsFunc(nil) for _, it := range items { + b.WriteString(indent.Tabs(ident)) if it.IsDir() { if recursive { - b.WriteString("\n" + it.ListShort(recursive, indent+1)) + b.WriteString("\n" + it.ListShort(recursive, ident+1)) } else { b.WriteString(it.name + "/ ") } @@ -47,13 +52,18 @@ func (d *Data) ListShort(recursive bool, indent int) string { return b.String() } -func (d *Data) ListLong(recursive bool, indent int) string { +func (d *Data) ListLong(recursive bool, ident int) string { var b strings.Builder items := d.ItemsFunc(nil) for _, it := range items { + b.WriteString(indent.Tabs(ident)) if it.IsDir() { + b.WriteString(it.name + "/\n") + if recursive { + b.WriteString(it.ListLong(recursive, ident+1)) + } } else { - b.WriteString(it.String() + "\n") + b.WriteString(it.name + "\t" + it.String() + "\n") } } return b.String() diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go index 8a34822953..13c19a13e8 100644 --- a/tensor/examples/planets/planets.go +++ b/tensor/examples/planets/planets.go @@ -8,8 +8,10 @@ import ( "embed" "math" + "cogentcore.org/core/cli" "cogentcore.org/core/core" "cogentcore.org/core/events" + "cogentcore.org/core/goal/interpreter" "cogentcore.org/core/icons" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" @@ -80,10 +82,17 @@ func AnalyzePlanets(dir *datafs.Data) { } func main() { - dir, _ := datafs.NewDir("Planets") + dir := datafs.Mkdir("Planets") AnalyzePlanets(dir) - br := databrowser.NewBrowserWindow(dir, "Planets") + opts := cli.DefaultOptions("planets", "interactive data analysis.") + cfg := &interpreter.Config{} + cfg.InteractiveFunc = Interactive + cli.Run(opts, cfg, interpreter.Run, interpreter.Build) +} + +func Interactive(c *interpreter.Config, in *interpreter.Interpreter) error { + br := databrowser.NewBrowserWindow(datafs.CurRoot, "Planets") b := br.Parent.(*core.Body) b.AddTopBar(func(bar *core.Frame) { tb := core.NewToolbar(bar) @@ -97,5 +106,14 @@ func main() { }) }) }) + b.OnShow(func(e events.Event) { + go func() { + if c.Expr != "" { + in.Eval(c.Expr) + } + in.Interactive() + }() + }) core.Wait() + return nil } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go b/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go index 44b232809a..b5be53dfb9 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go @@ -14,12 +14,16 @@ func init() { "CurDir": reflect.ValueOf(&datafs.CurDir).Elem(), "CurRoot": reflect.ValueOf(&datafs.CurRoot).Elem(), "DirOnly": reflect.ValueOf(datafs.DirOnly), + "Get": reflect.ValueOf(datafs.Get), + "List": reflect.ValueOf(datafs.List), "Long": reflect.ValueOf(datafs.Long), + "Mkdir": reflect.ValueOf(datafs.Mkdir), "NewDir": reflect.ValueOf(datafs.NewDir), "Overwrite": reflect.ValueOf(datafs.Overwrite), "Preserve": reflect.ValueOf(datafs.Preserve), "Record": reflect.ValueOf(datafs.Record), "Recursive": reflect.ValueOf(datafs.Recursive), + "Set": reflect.ValueOf(datafs.Set), "Short": reflect.ValueOf(datafs.Short), // type definitions From 09c64b0c7ffcd13db830fb8f65f0f30581966a8b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 28 Sep 2024 01:51:20 -0700 Subject: [PATCH 154/311] properly parses newaxis and ellipsis in reslice syntax --- goal/README.md | 3 +-- goal/transpile/math.go | 14 ++++++++++++++ goal/transpile/parser.go | 6 ++++++ goal/transpile/transpile_test.go | 4 +++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/goal/README.md b/goal/README.md index dc14c5de7c..b73b66a439 100644 --- a/goal/README.md +++ b/goal/README.md @@ -303,6 +303,7 @@ See [NumPy basic indexing](https://numpy.org/doc/stable/user/basics.indexing.htm | `t.Reslice(a,` `1, t.FullAxis)` | same: |`a[1]` or `a[1, :]` | `a(2,:)` | entire second row of 2D tensor `a`; unspecified dimensions are equivalent to `:` (could omit second arg in Reslice too) | | `t.Reslice(a,` `Slice{Stop:5})` | same: |`a[0:5]` or `a[:5]` or `a[0:5, :]` | `a(1:5,:)` | 0..4 rows of `a`; uses same Go slice ranging here: (start:stop) where stop is _exclusive_ | | `t.Reslice(a,` `Slice{Start:-5})` | same: |`a[-5:]` | `a(end-4:end,:)` | last 5 rows of 2D tensor `a` | +| `t.Reslice(a,` `t.NewAxis,` `Slice{Start:-5})` | same: |`a[newaxis, -5:]` | ? | last 5 rows of 2D tensor `a`, as a column vector | | `t.Reslice(a,` `Slice{Stop:3},` `Slice{Start:4, Stop:9})` | same: |`a[0:3, 4:9]` | `a(1:3,5:9)` | The first through third rows and fifth through ninth columns of a 2D tensor, `a`. | | `t.Reslice(a,` `Slice{Start:2,` `Stop:25,` `Step:2}, t.FullAxis)` | same: |`a[2:21:2,:]` | `a(3:2:21,:)` | every other row of `a`, starting with the third and going to the twenty-first | | `t.Reslice(a,` `Slice{Step:2},` `t.FullAxis)` | same: |`a[::2, :]` | `a(1:2:end,:)` | every other row of `a`, starting with the first | @@ -404,12 +405,10 @@ todo: huge amount of work needed to support complex numbers throughout! ## TODO -* newaxis -> tensor.NewAxis constant; ... -> tensor.Ellipsis in Reslice * more creation routines * update readme table * # # surrounding in go, shell modes -* record to datafs * make a simple tutorial example showing basic ops diff --git a/goal/transpile/math.go b/goal/transpile/math.go index bed2e6cebb..86e7a41d76 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -358,6 +358,15 @@ func (mp *mathParse) expr(ex ast.Expr) { case *ast.UnaryExpr: mp.unaryExpr(x) + case *ast.Ellipsis: + cfun := mp.funcs.Peek() + if cfun != nil && cfun.Name == "tensor.Reslice" { + mp.out.Add(token.IDENT, "tensor.Ellipsis") + mp.idx++ + } else { + mp.addToken(token.ELLIPSIS) + } + case *ast.StarExpr: mp.addToken(token.MUL) mp.expr(x.X) @@ -890,6 +899,11 @@ func (mp *mathParse) ident(id *ast.Ident) { if id == nil { return } + if id.Name == "newaxis" { + mp.out.Add(token.IDENT, "tensor.NewAxis") + mp.idx++ + return + } if mp.curArgIsInts() { mp.out.Add(token.IDENT, "tensor.AsIntSlice") mp.out.Add(token.LPAREN) diff --git a/goal/transpile/parser.go b/goal/transpile/parser.go index d1ea86fd86..920a05f13f 100644 --- a/goal/transpile/parser.go +++ b/goal/transpile/parser.go @@ -1834,6 +1834,12 @@ func (p *parser) parsePrimaryExpr(x ast.Expr) ast.Expr { defer un(trace(p, "PrimaryExpr")) } + // math: ellipses can show up in index expression. + if p.tok == token.ELLIPSIS { + p.next() + return &ast.Ellipsis{Ellipsis: p.pos} + } + if x == nil { x = p.parseOperand() } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index ff48b98f2c..7e4e240d0e 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# a := linspace(0, 5, 6, true)", `a := tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, + {"# a[..., 2:]", `tensor.Reslice(a, tensor.Ellipsis, tensor.Slice { Start:2 } )`}, } st := NewState() st.MathRecord = false @@ -243,6 +243,8 @@ func TestMath(t *testing.T) { {"# a[::-1, 2]", `tensor.Reslice(a, tensor.Slice { Step: - 1 } , 2)`}, {"# a[:3, 2]", `tensor.Reslice(a, tensor.Slice { Stop:3 } , 2)`}, {"# a[2:, 2]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2)`}, + {"# a[2:, 2, newaxis]", `tensor.Reslice(a, tensor.Slice { Start:2 } , 2, tensor.NewAxis)`}, + {"# a[..., 2:]", `tensor.Reslice(a, tensor.Ellipsis, tensor.Slice { Start:2 } )`}, {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, {"# c := cos(a)", `c := tmath.Cos(a)`}, From 5f6c7f17c4d4bc053b5d30d192acc9c2950c7492 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 28 Sep 2024 02:14:21 -0700 Subject: [PATCH 155/311] fix tensor tests with new output and renames --- tensor/slices_test.go | 29 +++++++------ tensor/stats/stats/group_test.go | 4 +- tensor/table/table_test.go | 2 +- tensor/tensor_test.go | 72 ++++++++++++++++---------------- tensor/tmath/bool_test.go | 2 +- tensor/tmath/math_test.go | 4 +- 6 files changed, 59 insertions(+), 54 deletions(-) diff --git a/tensor/slices_test.go b/tensor/slices_test.go index 432aea67ea..77582d4242 100644 --- a/tensor/slices_test.go +++ b/tensor/slices_test.go @@ -72,50 +72,53 @@ func TestSlicedExpr(t *testing.T) { } res := `[3, 4] -[0]: 0 1 2 3 -[1]: 10 11 12 13 -[2]: 20 21 22 23 + [0]: [1]: [2]: [3]: +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 ` assert.Equal(t, res, ft.String()) // fmt.Println(ft) - res = `[1] 12 + res = `[1] 12 ` sl := Reslice(ft, 1, 2) - // fmt.Println(sl) assert.Equal(t, res, sl.String()) - res = `[4] 10 11 12 13 + res = `[4] 10 11 12 13 ` sl = Reslice(ft, 1) assert.Equal(t, res, sl.String()) - res = `[3] 2 12 22 + res = `[3] 2 12 22 ` sl = Reslice(ft, Ellipsis, 2) assert.Equal(t, res, sl.String()) res = `[3, 4] -[0]: 3 2 1 0 -[1]: 13 12 11 10 -[2]: 23 22 21 20 + [0]: [1]: [2]: [3]: +[0]: 3 2 1 0 +[1]: 13 12 11 10 +[2]: 23 22 21 20 ` sl = Reslice(ft, Ellipsis, Slice{Step: -1}) assert.Equal(t, res, sl.String()) res = `[1, 4] -[0]: 10 11 12 13 + [0]: [1]: [2]: [3]: +[0]: 10 11 12 13 ` sl = Reslice(ft, NewAxis, 1) assert.Equal(t, res, sl.String()) res = `[1, 3] -[0]: 1 11 21 + [0]: [1]: [2]: +[0]: 1 11 21 ` sl = Reslice(ft, NewAxis, FullAxis, 1) // keeps result as a column vector assert.Equal(t, res, sl.String()) - res = `[3] 1 11 21 + res = `[3] 1 11 21 ` sl = Reslice(ft, FullAxis, 1) assert.Equal(t, res, sl.String()) diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index 6ec5061b9d..e47bbc2666 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -30,8 +30,8 @@ func TestGroup(t *testing.T) { assert.NoError(t, err) ixs := dir.FlatValuesFunc(nil) - assert.Equal(t, []int{0, 1}, tensor.AsIntTensor(ixs[0]).Values) - assert.Equal(t, []int{2, 3}, tensor.AsIntTensor(ixs[1]).Values) + assert.Equal(t, []int{0, 1}, tensor.AsInt(ixs[0]).Values) + assert.Equal(t, []int{2, 3}, tensor.AsInt(ixs[1]).Values) err = TableGroupStats(dir, StatMean, dt, "Value") assert.NoError(t, err) diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index a666921bff..744f773ffe 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -127,7 +127,7 @@ func TestCells(t *testing.T) { ss := in.Tensor.SubSpace(i).(*tensor.Float32) // fmt.Println(ss.Values[:16]) - cl := tensor.AsFloat32Tensor(tensor.Cells1D(in, i)) + cl := tensor.AsFloat32(tensor.Cells1D(in, i)) // fmt.Println(cl.Values[:16]) assert.Equal(t, vals, ss.Values[:16]) assert.Equal(t, vals, cl.Values[:16]) diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 009fd775f7..8b6b05f4df 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -133,30 +133,30 @@ func TestSliced(t *testing.T) { } res := `[3, 4] -[0]: 0 1 2 3 -[1]: 10 11 12 13 -[2]: 20 21 22 23 + [0]: [1]: [2]: [3]: +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 ` assert.Equal(t, res, ft.String()) // fmt.Println(ft) res = `[2, 2] -[0]: 23 22 -[1]: 13 12 + [0]: [1]: +[0]: 23 22 +[1]: 13 12 ` sl := NewSliced(ft, []int{2, 1}, []int{3, 2}) - // fmt.Println(sl) assert.Equal(t, res, sl.String()) vl := sl.AsValues() assert.Equal(t, res, vl.String()) res = `[3, 1] -[0]: 2 -[1]: 12 -[2]: 22 +[0]: 2 +[1]: 12 +[2]: 22 ` sl2 := Reslice(ft, FullAxis, Slice{2, 3, 0}) - // fmt.Println(sl) assert.Equal(t, res, sl2.String()) vl = sl2.AsValues() @@ -174,9 +174,10 @@ func TestMasked(t *testing.T) { ms := NewMasked(ft) res := `[3, 4] -[0]: 0 1 2 3 -[1]: 10 11 12 13 -[2]: 20 21 22 23 + [0]: [1]: [2]: [3]: +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 ` assert.Equal(t, res, ms.String()) @@ -185,16 +186,15 @@ func TestMasked(t *testing.T) { return int(val)%10 == 2 }) res = `[3, 4] -[0]: NaN NaN 2 NaN -[1]: NaN NaN 12 NaN -[2]: NaN NaN 22 NaN + [0]: [1]: [2]: [3]: +[0]: NaN NaN 2 NaN +[1]: NaN NaN 12 NaN +[2]: NaN NaN 22 NaN ` - // fmt.Println(ms.String()) assert.Equal(t, res, ms.String()) - res = `[3] 2 12 22 + res = `[3] 2 12 22 ` - vl := ms.AsValues() assert.Equal(t, res, vl.String()) } @@ -222,10 +222,10 @@ func TestIndexed(t *testing.T) { ix := NewIndexed(ft, ixs) res := `[2, 2, 2] -[0]: 1 1 11 11 -[0]: 2 2 22 22 + [0 0]: [0 1]: [0 0]: [0 1]: +[0]: 1 1 11 11 +[0]: 2 2 22 22 ` - // fmt.Println(ix.String()) assert.Equal(t, res, ix.String()) vl := ix.AsValues() @@ -242,34 +242,36 @@ func TestReshaped(t *testing.T) { } res := `[4, 3] -[0]: 0 1 2 -[1]: 3 10 11 -[2]: 12 13 20 -[3]: 21 22 23 + [0]: [1]: [2]: +[0]: 0 1 2 +[1]: 3 10 11 +[2]: 12 13 20 +[3]: 21 22 23 ` rs := NewReshaped(ft, 4, 3) - // fmt.Println(rs) assert.Equal(t, res, rs.String()) res = `[1, 3, 4] -[0]: 0 1 2 3 -[0]: 10 11 12 13 -[0]: 20 21 22 23 + [0 0]: [0 1]: [0 2]: [0 3]: +[0]: 0 1 2 3 +[0]: 10 11 12 13 +[0]: 20 21 22 23 ` rs = NewReshaped(ft, int(NewAxis), 3, 4) assert.Equal(t, res, rs.String()) res = `[12] -[0]: 0 1 2 3 10 11 12 13 20 21 22 23 +[0]: 0 1 2 3 10 11 12 13 20 21 22 23 ` rs = NewReshaped(ft, -1) assert.Equal(t, res, rs.String()) res = `[4, 3] -[0]: 0 1 2 -[1]: 3 10 11 -[2]: 12 13 20 -[3]: 21 22 23 + [0]: [1]: [2]: +[0]: 0 1 2 +[1]: 3 10 11 +[2]: 12 13 20 +[3]: 21 22 23 ` rs = NewReshaped(ft, 4, -1) // fmt.Println(rs) diff --git a/tensor/tmath/bool_test.go b/tensor/tmath/bool_test.go index b0ec0a77cb..b48e44caca 100644 --- a/tensor/tmath/bool_test.go +++ b/tensor/tmath/bool_test.go @@ -12,7 +12,7 @@ import ( ) func TestBoolOps(t *testing.T) { - ar := tensor.NewSliceInts(12) + ar := tensor.NewIntRange(12) // fmt.Println(v) bo := tensor.NewBool() sc := tensor.NewIntScalar(6) diff --git a/tensor/tmath/math_test.go b/tensor/tmath/math_test.go index e36535fbd9..0c8a2ba468 100644 --- a/tensor/tmath/math_test.go +++ b/tensor/tmath/math_test.go @@ -62,7 +62,7 @@ func TestMath(t *testing.T) { // fmt.Println(r) si := lv * r for c, v := range vals { - ov := tensor.AsFloat32Tensor(cellout).Values[si+c] + ov := tensor.AsFloat32(cellout).Values[si+c] testEqual(t, fun(v), float64(ov)) } } @@ -108,7 +108,7 @@ func TestMathBinary(t *testing.T) { // fmt.Println(r) si := lv * r for c, v := range vals { - ov := tensor.AsFloat32Tensor(cellout).Values[si+c] + ov := tensor.AsFloat32(cellout).Values[si+c] testEqual(t, fun(v, v), float64(ov)) } } From 04084f42cba7a9867918085986d0e7ef4efe671f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 28 Sep 2024 10:26:34 -0700 Subject: [PATCH 156/311] fix extra goal/math_test --- goal/goal.go | 1 - goal/interpreter/interpreter.go | 1 + goal/testdata/test.goal | 5 ----- goal/transpile/math.go | 18 ++++++++++++------ goal/transpile/state.go | 2 +- goal/transpile/transpile_test.go | 3 ++- 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/goal/goal.go b/goal/goal.go index edf7f517f8..7e57e493ad 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -102,7 +102,6 @@ func NewGoal() *Goal { }, } gl.TrState.FuncToVar = true - gl.TrState.MathRecord = true gl.Config.StdIO.SetFromOS() gl.SSH = sshclient.NewConfig(&gl.Config) gl.SSHClients = make(map[string]*sshclient.Client) diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index 23f5101cde..f234e1fbb9 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -213,6 +213,7 @@ func (in *Interpreter) SaveHistory() error { // Interactive runs an interactive shell that allows the user to input goal. // Must have done in.Config() prior to calling. func (in *Interpreter) Interactive() error { + in.Goal.TrState.MathRecord = true rl, err := readline.NewFromConfig(&readline.Config{ AutoComplete: &goal.ReadlineCompleter{Goal: in.Goal}, Undo: true, diff --git a/goal/testdata/test.goal b/goal/testdata/test.goal index 4eeb0850ee..fdd2a23b3e 100644 --- a/goal/testdata/test.goal +++ b/goal/testdata/test.goal @@ -9,8 +9,3 @@ fmt.Println(x) fmt.Println(nd) fmt.Println(sz) fmt.Println(sh) - -if 1 == 1 { - fmt.Println(true) -} - diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 86e7a41d76..b82db3aa7e 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -143,17 +143,23 @@ func (mp *mathParse) curArgIsInts() bool { } // startFunc is called when starting a new function. -// empty is "dummy" assign case using Inc -func (mp *mathParse) startFunc(name string) *funcInfo { +// empty is "dummy" assign case using Inc. +// optional noLookup indicates to not lookup type and just +// push the name -- for internal cases to prevent arg conversions. +func (mp *mathParse) startFunc(name string, noLookup ...bool) *funcInfo { fi := &funcInfo{} sname := name if name == "" { sname = "tmath.Inc" } - if tf, err := tensor.FuncByName(sname); err == nil { - fi.Func = *tf + if len(noLookup) == 1 && noLookup[0] { + fi.Name = name } else { - fi.Name = name // not clear what point is + if tf, err := tensor.FuncByName(sname); err == nil { + fi.Func = *tf + } else { + fi.Name = name + } } mp.funcs.Push(fi) if name != "" { @@ -653,7 +659,7 @@ func (fw *funWrap) wrapFunc(mp *mathParse) bool { wrapFun = "tensor.NewStringFromValues" ellip = true } - mp.startFunc(wrapFun) + mp.startFunc(wrapFun, true) // don't lookup -- don't auto-convert args mp.out.Add(token.LPAREN) return ellip } diff --git a/goal/transpile/state.go b/goal/transpile/state.go index 4dd8a77f69..e7a77d9a53 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -56,7 +56,7 @@ type State struct { // NewState returns a new transpiling state; mostly for testing func NewState() *State { - st := &State{FuncToVar: true, MathRecord: true} + st := &State{FuncToVar: true} return st } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 7e4e240d0e..1799e41399 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# a[..., 2:]", `tensor.Reslice(a, tensor.Ellipsis, tensor.Slice { Start:2 } )`}, + {"# sh := x.shape", `sh := tensor.NewIntFromValues(x.Shape().Sizes ...)`}, } st := NewState() st.MathRecord = false @@ -226,6 +226,7 @@ func TestMath(t *testing.T) { {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, + {"# sh := x.shape", `sh := tensor.NewIntFromValues(x.Shape().Sizes ...)`}, {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, From 71e244551303f811144e2bc07c2d4e16ca9642a2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 28 Sep 2024 10:52:32 -0700 Subject: [PATCH 157/311] add get and set aliases for datafs Get and Set, and add datafs docs to readme. --- goal/README.md | 23 ++++++++++++++++++----- goal/transpile/math.go | 2 ++ goal/transpile/transpile_test.go | 5 ++++- tensor/datafs/list.go | 2 +- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/goal/README.md b/goal/README.md index b73b66a439..5547db8a38 100644 --- a/goal/README.md +++ b/goal/README.md @@ -403,12 +403,25 @@ todo: huge amount of work needed to support complex numbers throughout! | | |`np.fft.ifft(a)` | `ifft(a)` | inverse Fourier transform of `a` | | | |`signal.resample(x, np.ceil(len(x)/q))` | `decimate(x, q)` | downsample with low-pass filtering | -## TODO +## datafs -* more creation routines -* update readme table +The [datafs](../tensor/datafs) data filesystem provides a global filesystem-like workspace for storing tensor data, and Goal has special commands and functions to facilitate interacting with it. In an interactive `goal` shell, when you do `##` to switch into math mode, the prompt changes to show your current directory in the datafs, not the regular OS filesystem, and the final prompt character turns into a `#`. + +Use `get` and `set` (aliases for `datafs.Get` and `datafs.Set`) to retrieve and store data in the datafs: + +* `x := get("path/to/item")` retrieves the tensor data value at given path, which can then be used directly in an expression or saved to a new variable as in this example. + +* `set("path/to/item", x)` saves tensor data to given path, overwriting any existing value for that item if it already exists, and creating a new one if not. `x` can be any data expression. + +You can use the standard shell commands to navigate around the data filesystem: + +* `cd ` to change the current working directory. By default, new variables created in the shell are also recorded into the current working directory for later access. + +* `ls [-l,r] [dir]` list the contents of a directory; without arguments, it shows the current directory. The `-l` option shows each element on a separate line with its shape. `-r` does a recursive list through subdirectories. + +* `mkdir ` makes a new subdirectory. + +TODO: other commands, etc. -* # # surrounding in go, shell modes -* make a simple tutorial example showing basic ops diff --git a/goal/transpile/math.go b/goal/transpile/math.go index b82db3aa7e..7a80ad3fc8 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -795,6 +795,8 @@ var numpyFuncs = map[string]funWrap{ "linspace": {"tensor.NewFloat64SpacedLinear", ""}, "reshape": {"tensor.Reshape", ""}, "copy": {"tensor.Clone", ""}, + "get": {"datafs.Get", ""}, + "set": {"datafs.Set", ""}, "flatten": {"tensor.Flatten", "nofun"}, "squeeze": {"tensor.Squeeze", "nofun"}, } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 1799e41399..a13c552514 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# sh := x.shape", `sh := tensor.NewIntFromValues(x.Shape().Sizes ...)`}, + {`# set("item", 5)`, `datafs.Set("item", tensor.NewIntScalar(5))`}, } st := NewState() st.MathRecord = false @@ -266,6 +266,9 @@ func TestMath(t *testing.T) { {"# if a[1,2] == 2 {", `if tmath.Equal(tensor.Reslice(a, 1, 2), tensor.NewIntScalar(2)).Bool1D(0) {`}, {"# for i := 0; i < 3; i++ {", `for i := tensor.NewIntScalar(0); tmath.Less(i, tensor.NewIntScalar(3)).Bool1D(0); tmath.Inc(i) {`}, {"# for i, v := range a {", `for i := 0; i < a.Len(); i++ { v := a .Float1D(i)`}, + {`# x := get("item")`, `x := datafs.Get("item")`}, + {`# set("item", x)`, `datafs.Set("item", x)`}, + {`# set("item", 5)`, `datafs.Set("item", tensor.NewIntScalar(5))`}, } st := NewState() diff --git a/tensor/datafs/list.go b/tensor/datafs/list.go index 2932bf99db..7b69bee809 100644 --- a/tensor/datafs/list.go +++ b/tensor/datafs/list.go @@ -63,7 +63,7 @@ func (d *Data) ListLong(recursive bool, ident int) string { b.WriteString(it.ListLong(recursive, ident+1)) } } else { - b.WriteString(it.name + "\t" + it.String() + "\n") + b.WriteString(it.String() + "\n") } } return b.String() From 04a7224f13d7be2bdb70aa42f7d14956cd038721 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 28 Sep 2024 10:57:57 -0700 Subject: [PATCH 158/311] add test for # # wrapping -- works. all good for now! --- goal/transpile/transpile_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index a13c552514..1b87942d2e 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {`# set("item", 5)`, `datafs.Set("item", tensor.NewIntScalar(5))`}, + {`fmt.Println(#zeros(3,4)#)`, `fmt.Println(tensor.NewFloat64(3, 4))`}, } st := NewState() st.MathRecord = false @@ -269,6 +269,7 @@ func TestMath(t *testing.T) { {`# x := get("item")`, `x := datafs.Get("item")`}, {`# set("item", x)`, `datafs.Set("item", x)`}, {`# set("item", 5)`, `datafs.Set("item", tensor.NewIntScalar(5))`}, + {`fmt.Println(#zeros(3,4)#)`, `fmt.Println(tensor.NewFloat64(3, 4))`}, } st := NewState() From 132514f37f2a9ac149d7600798ddb83fb91cfed1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 17:21:21 -0700 Subject: [PATCH 159/311] fix FloatFromValues -> Float64FromValues --- goal/transpile/math.go | 2 +- goal/transpile/transpile_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 7a80ad3fc8..240a825168 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -758,7 +758,7 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { } // todo: look for sub-arrays etc. typ := "float64" - fun := "Float" + fun := "Float64" switch kind { case token.FLOAT: case token.INT: diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 1b87942d2e..e593bd01bb 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -224,6 +224,7 @@ func TestMath(t *testing.T) { {"# a := x ** 2", `a := tmath.Pow(x, tensor.NewIntScalar(2))`}, {"# a = -x", `a = tmath.Negate(x)`}, {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, + {"# a := [1.,2,3,4]", `a := tensor.NewFloat64FromValues([]float64 { 1., 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, {"# sh := x.shape", `sh := tensor.NewIntFromValues(x.Shape().Sizes ...)`}, From 786fde18f4f725d36dc69882cf5fde628b533c9f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 17:57:44 -0700 Subject: [PATCH 160/311] properly promote float types in binary expressions -- if either side is a float, the result is a float; all float1 tmath functions automatically produce float64 output. --- tensor/funcs.go | 31 ++++++++++++++++--- tensor/tmath/math.go | 74 ++++++++++++++++++++++---------------------- 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/tensor/funcs.go b/tensor/funcs.go index 3452243d95..c68a6c22fe 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -154,6 +154,22 @@ func FuncByName(name string) (*Func, error) { // Additional generic non-Tensor inputs are supported up to 2, // with Gen1 and Gen2 versions. +// FloatPromoteType returns the DataType for Tensor(s) that promotes +// the Float type if any of the elements are of that type. +// Otherwise it returns the type of the first tensor. +func FloatPromoteType(tsr ...Tensor) reflect.Kind { + ft := tsr[0].DataType() + for i := 1; i < len(tsr); i++ { + t := tsr[i].DataType() + if t == reflect.Float64 { + ft = t + } else if t == reflect.Float32 && ft != reflect.Float64 { + ft = t + } + } + return ft +} + // CallOut1 adds output [Values] tensor for function. func CallOut1(fun func(a Tensor, out Values) error, a Tensor) Values { out := NewOfType(a.DataType()) @@ -161,14 +177,21 @@ func CallOut1(fun func(a Tensor, out Values) error, a Tensor) Values { return out } +// CallOut1Float64 adds Float64 output [Values] tensor for function. +func CallOut1Float64(fun func(a Tensor, out Values) error, a Tensor) Values { + out := NewFloat64() + errors.Log(fun(a, out)) + return out +} + func CallOut2(fun func(a, b Tensor, out Values) error, a, b Tensor) Values { - out := NewOfType(a.DataType()) + out := NewOfType(FloatPromoteType(a, b)) errors.Log(fun(a, b, out)) return out } func CallOut3(fun func(a, b, c Tensor, out Values) error, a, b, c Tensor) Values { - out := NewOfType(a.DataType()) + out := NewOfType(FloatPromoteType(a, b, c)) errors.Log(fun(a, b, c, out)) return out } @@ -192,13 +215,13 @@ func CallOut1Gen2[T any, S any](fun func(g T, h S, a Tensor, out Values) error, } func CallOut2Gen1[T any](fun func(g T, a, b Tensor, out Values) error, g T, a, b Tensor) Values { - out := NewOfType(a.DataType()) + out := NewOfType(FloatPromoteType(a, b)) errors.Log(fun(g, a, b, out)) return out } func CallOut2Gen2[T any, S any](fun func(g T, h S, a, b Tensor, out Values) error, g T, h S, a, b Tensor) Values { - out := NewOfType(a.DataType()) + out := NewOfType(FloatPromoteType(a, b)) errors.Log(fun(g, h, a, b, out)) return out } diff --git a/tensor/tmath/math.go b/tensor/tmath/math.go index 2b1ef9dbea..8f38efcf46 100644 --- a/tensor/tmath/math.go +++ b/tensor/tmath/math.go @@ -11,7 +11,7 @@ import ( ) func Abs(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AbsOut, in) + return tensor.CallOut1Float64(AbsOut, in) } func AbsOut(in tensor.Tensor, out tensor.Values) error { @@ -19,7 +19,7 @@ func AbsOut(in tensor.Tensor, out tensor.Values) error { } func Acos(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AcosOut, in) + return tensor.CallOut1Float64(AcosOut, in) } func AcosOut(in tensor.Tensor, out tensor.Values) error { @@ -27,7 +27,7 @@ func AcosOut(in tensor.Tensor, out tensor.Values) error { } func Acosh(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AcoshOut, in) + return tensor.CallOut1Float64(AcoshOut, in) } func AcoshOut(in tensor.Tensor, out tensor.Values) error { @@ -35,7 +35,7 @@ func AcoshOut(in tensor.Tensor, out tensor.Values) error { } func Asin(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AsinOut, in) + return tensor.CallOut1Float64(AsinOut, in) } func AsinOut(in tensor.Tensor, out tensor.Values) error { @@ -43,7 +43,7 @@ func AsinOut(in tensor.Tensor, out tensor.Values) error { } func Asinh(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AsinhOut, in) + return tensor.CallOut1Float64(AsinhOut, in) } func AsinhOut(in tensor.Tensor, out tensor.Values) error { @@ -51,7 +51,7 @@ func AsinhOut(in tensor.Tensor, out tensor.Values) error { } func Atan(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AtanOut, in) + return tensor.CallOut1Float64(AtanOut, in) } func AtanOut(in tensor.Tensor, out tensor.Values) error { @@ -59,7 +59,7 @@ func AtanOut(in tensor.Tensor, out tensor.Values) error { } func Atanh(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(AtanhOut, in) + return tensor.CallOut1Float64(AtanhOut, in) } func AtanhOut(in tensor.Tensor, out tensor.Values) error { @@ -67,7 +67,7 @@ func AtanhOut(in tensor.Tensor, out tensor.Values) error { } func Cbrt(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(CbrtOut, in) + return tensor.CallOut1Float64(CbrtOut, in) } func CbrtOut(in tensor.Tensor, out tensor.Values) error { @@ -75,7 +75,7 @@ func CbrtOut(in tensor.Tensor, out tensor.Values) error { } func Ceil(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(CeilOut, in) + return tensor.CallOut1Float64(CeilOut, in) } func CeilOut(in tensor.Tensor, out tensor.Values) error { @@ -83,7 +83,7 @@ func CeilOut(in tensor.Tensor, out tensor.Values) error { } func Cos(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(CosOut, in) + return tensor.CallOut1Float64(CosOut, in) } func CosOut(in tensor.Tensor, out tensor.Values) error { @@ -91,7 +91,7 @@ func CosOut(in tensor.Tensor, out tensor.Values) error { } func Cosh(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(CoshOut, in) + return tensor.CallOut1Float64(CoshOut, in) } func CoshOut(in tensor.Tensor, out tensor.Values) error { @@ -99,7 +99,7 @@ func CoshOut(in tensor.Tensor, out tensor.Values) error { } func Erf(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(ErfOut, in) + return tensor.CallOut1Float64(ErfOut, in) } func ErfOut(in tensor.Tensor, out tensor.Values) error { @@ -107,7 +107,7 @@ func ErfOut(in tensor.Tensor, out tensor.Values) error { } func Erfc(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(ErfcOut, in) + return tensor.CallOut1Float64(ErfcOut, in) } func ErfcOut(in tensor.Tensor, out tensor.Values) error { @@ -115,7 +115,7 @@ func ErfcOut(in tensor.Tensor, out tensor.Values) error { } func Erfcinv(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(ErfcinvOut, in) + return tensor.CallOut1Float64(ErfcinvOut, in) } func ErfcinvOut(in tensor.Tensor, out tensor.Values) error { @@ -123,7 +123,7 @@ func ErfcinvOut(in tensor.Tensor, out tensor.Values) error { } func Erfinv(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(ErfinvOut, in) + return tensor.CallOut1Float64(ErfinvOut, in) } func ErfinvOut(in tensor.Tensor, out tensor.Values) error { @@ -131,7 +131,7 @@ func ErfinvOut(in tensor.Tensor, out tensor.Values) error { } func Exp(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(ExpOut, in) + return tensor.CallOut1Float64(ExpOut, in) } func ExpOut(in tensor.Tensor, out tensor.Values) error { @@ -139,7 +139,7 @@ func ExpOut(in tensor.Tensor, out tensor.Values) error { } func Exp2(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Exp2Out, in) + return tensor.CallOut1Float64(Exp2Out, in) } func Exp2Out(in tensor.Tensor, out tensor.Values) error { @@ -147,7 +147,7 @@ func Exp2Out(in tensor.Tensor, out tensor.Values) error { } func Expm1(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Expm1Out, in) + return tensor.CallOut1Float64(Expm1Out, in) } func Expm1Out(in tensor.Tensor, out tensor.Values) error { @@ -155,7 +155,7 @@ func Expm1Out(in tensor.Tensor, out tensor.Values) error { } func Floor(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(FloorOut, in) + return tensor.CallOut1Float64(FloorOut, in) } func FloorOut(in tensor.Tensor, out tensor.Values) error { @@ -163,7 +163,7 @@ func FloorOut(in tensor.Tensor, out tensor.Values) error { } func Gamma(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(GammaOut, in) + return tensor.CallOut1Float64(GammaOut, in) } func GammaOut(in tensor.Tensor, out tensor.Values) error { @@ -171,7 +171,7 @@ func GammaOut(in tensor.Tensor, out tensor.Values) error { } func J0(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(J0Out, in) + return tensor.CallOut1Float64(J0Out, in) } func J0Out(in tensor.Tensor, out tensor.Values) error { @@ -179,7 +179,7 @@ func J0Out(in tensor.Tensor, out tensor.Values) error { } func J1(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(J1Out, in) + return tensor.CallOut1Float64(J1Out, in) } func J1Out(in tensor.Tensor, out tensor.Values) error { @@ -187,7 +187,7 @@ func J1Out(in tensor.Tensor, out tensor.Values) error { } func Log(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(LogOut, in) + return tensor.CallOut1Float64(LogOut, in) } func LogOut(in tensor.Tensor, out tensor.Values) error { @@ -195,7 +195,7 @@ func LogOut(in tensor.Tensor, out tensor.Values) error { } func Log10(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Log10Out, in) + return tensor.CallOut1Float64(Log10Out, in) } func Log10Out(in tensor.Tensor, out tensor.Values) error { @@ -203,7 +203,7 @@ func Log10Out(in tensor.Tensor, out tensor.Values) error { } func Log1p(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Log1pOut, in) + return tensor.CallOut1Float64(Log1pOut, in) } func Log1pOut(in tensor.Tensor, out tensor.Values) error { @@ -211,7 +211,7 @@ func Log1pOut(in tensor.Tensor, out tensor.Values) error { } func Log2(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Log2Out, in) + return tensor.CallOut1Float64(Log2Out, in) } func Log2Out(in tensor.Tensor, out tensor.Values) error { @@ -219,7 +219,7 @@ func Log2Out(in tensor.Tensor, out tensor.Values) error { } func Logb(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(LogbOut, in) + return tensor.CallOut1Float64(LogbOut, in) } func LogbOut(in tensor.Tensor, out tensor.Values) error { @@ -227,7 +227,7 @@ func LogbOut(in tensor.Tensor, out tensor.Values) error { } func Round(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(RoundOut, in) + return tensor.CallOut1Float64(RoundOut, in) } func RoundOut(in tensor.Tensor, out tensor.Values) error { @@ -235,7 +235,7 @@ func RoundOut(in tensor.Tensor, out tensor.Values) error { } func RoundToEven(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(RoundToEvenOut, in) + return tensor.CallOut1Float64(RoundToEvenOut, in) } func RoundToEvenOut(in tensor.Tensor, out tensor.Values) error { @@ -243,7 +243,7 @@ func RoundToEvenOut(in tensor.Tensor, out tensor.Values) error { } func Sin(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(SinOut, in) + return tensor.CallOut1Float64(SinOut, in) } func SinOut(in tensor.Tensor, out tensor.Values) error { @@ -251,7 +251,7 @@ func SinOut(in tensor.Tensor, out tensor.Values) error { } func Sinh(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(SinhOut, in) + return tensor.CallOut1Float64(SinhOut, in) } func SinhOut(in tensor.Tensor, out tensor.Values) error { @@ -259,7 +259,7 @@ func SinhOut(in tensor.Tensor, out tensor.Values) error { } func Sqrt(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(SqrtOut, in) + return tensor.CallOut1Float64(SqrtOut, in) } func SqrtOut(in tensor.Tensor, out tensor.Values) error { @@ -267,7 +267,7 @@ func SqrtOut(in tensor.Tensor, out tensor.Values) error { } func Tan(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(TanOut, in) + return tensor.CallOut1Float64(TanOut, in) } func TanOut(in tensor.Tensor, out tensor.Values) error { @@ -275,7 +275,7 @@ func TanOut(in tensor.Tensor, out tensor.Values) error { } func Tanh(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(TanhOut, in) + return tensor.CallOut1Float64(TanhOut, in) } func TanhOut(in tensor.Tensor, out tensor.Values) error { @@ -283,7 +283,7 @@ func TanhOut(in tensor.Tensor, out tensor.Values) error { } func Trunc(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(TruncOut, in) + return tensor.CallOut1Float64(TruncOut, in) } func TruncOut(in tensor.Tensor, out tensor.Values) error { @@ -291,7 +291,7 @@ func TruncOut(in tensor.Tensor, out tensor.Values) error { } func Y0(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Y0Out, in) + return tensor.CallOut1Float64(Y0Out, in) } func Y0Out(in tensor.Tensor, out tensor.Values) error { @@ -299,7 +299,7 @@ func Y0Out(in tensor.Tensor, out tensor.Values) error { } func Y1(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(Y1Out, in) + return tensor.CallOut1Float64(Y1Out, in) } func Y1Out(in tensor.Tensor, out tensor.Values) error { From 5d6195323ccd0da0ba783a821398e0b3a17d40b8 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 18:01:56 -0700 Subject: [PATCH 161/311] scalar values don't print shape. --- tensor/io.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensor/io.go b/tensor/io.go index 0f735b9438..26c428a56f 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -191,6 +191,13 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { format = "%7.3g\t" } } + if tsr.NumDims() == 1 && tsr.DimSize(0) == 1 { // scalar special case + if tsr.IsString() { + return fmt.Sprintf(format, tsr.String1D(0)) + } else { + return fmt.Sprintf(format, tsr.Float1D(0)) + } + } mxwd := 0 n := min(tsr.Len(), maxLen) for i := range n { From 5afcaf6d0e517bfbe0faa6ed38b697d368e15f00 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 22:31:13 -0700 Subject: [PATCH 162/311] change default tensor format to %g, %s --- tensor/io.go | 8 +++--- tensor/slices_test.go | 29 ++++++++------------- tensor/tensor_test.go | 60 +++++++++++++++++++++---------------------- 3 files changed, 44 insertions(+), 53 deletions(-) diff --git a/tensor/io.go b/tensor/io.go index 26c428a56f..25a3e2cc16 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -172,7 +172,7 @@ func label(nm string, sh *Shape) string { // when it exceeds that length. If maxLen = 0, [MaxSprintLength] is used. // The format is the per-element format string, which should include // any delimiter or spacing between elements (which will apply to last -// element too). If empty it uses compact defaults for the data type. +// element too). If empty it uses general %g or %s defaults with a tab separator. func Sprintf(tsr Tensor, maxLen int, format string) string { if maxLen == 0 { maxLen = MaxSprintLength @@ -183,12 +183,12 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { if defFmt { switch { case tsr.IsString(): - format = "%15s\t" + format = "%s\t" case reflectx.KindIsInt(tsr.DataType()): isint = true - format = "%7g\t" + format = "%g\t" default: - format = "%7.3g\t" + format = "%g\t" } } if tsr.NumDims() == 1 && tsr.DimSize(0) == 1 { // scalar special case diff --git a/tensor/slices_test.go b/tensor/slices_test.go index 77582d4242..7ac9498de8 100644 --- a/tensor/slices_test.go +++ b/tensor/slices_test.go @@ -71,54 +71,47 @@ func TestSlicedExpr(t *testing.T) { } } - res := `[3, 4] - [0]: [1]: [2]: [3]: -[0]: 0 1 2 3 -[1]: 10 11 12 13 -[2]: 20 21 22 23 -` - assert.Equal(t, res, ft.String()) + rf := []float64{0, 1, 2, 3, 10, 11, 12, 13, 20, 21, 22, 23} + assert.Equal(t, rf, ft.Values) // fmt.Println(ft) - res = `[1] 12 -` sl := Reslice(ft, 1, 2) - assert.Equal(t, res, sl.String()) + assert.Equal(t, "12 ", sl.String()) - res = `[4] 10 11 12 13 + res := `[4] 10 11 12 13 ` sl = Reslice(ft, 1) assert.Equal(t, res, sl.String()) - res = `[3] 2 12 22 + res = `[3] 2 12 22 ` sl = Reslice(ft, Ellipsis, 2) assert.Equal(t, res, sl.String()) res = `[3, 4] [0]: [1]: [2]: [3]: -[0]: 3 2 1 0 -[1]: 13 12 11 10 -[2]: 23 22 21 20 +[0]: 3 2 1 0 +[1]: 13 12 11 10 +[2]: 23 22 21 20 ` sl = Reslice(ft, Ellipsis, Slice{Step: -1}) assert.Equal(t, res, sl.String()) res = `[1, 4] [0]: [1]: [2]: [3]: -[0]: 10 11 12 13 +[0]: 10 11 12 13 ` sl = Reslice(ft, NewAxis, 1) assert.Equal(t, res, sl.String()) res = `[1, 3] [0]: [1]: [2]: -[0]: 1 11 21 +[0]: 1 11 21 ` sl = Reslice(ft, NewAxis, FullAxis, 1) // keeps result as a column vector assert.Equal(t, res, sl.String()) - res = `[3] 1 11 21 + res = `[3] 1 11 21 ` sl = Reslice(ft, FullAxis, 1) assert.Equal(t, res, sl.String()) diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 8b6b05f4df..ecd2a1212e 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -134,17 +134,16 @@ func TestSliced(t *testing.T) { res := `[3, 4] [0]: [1]: [2]: [3]: -[0]: 0 1 2 3 -[1]: 10 11 12 13 -[2]: 20 21 22 23 +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 ` assert.Equal(t, res, ft.String()) - // fmt.Println(ft) res = `[2, 2] [0]: [1]: -[0]: 23 22 -[1]: 13 12 +[0]: 23 22 +[1]: 13 12 ` sl := NewSliced(ft, []int{2, 1}, []int{3, 2}) assert.Equal(t, res, sl.String()) @@ -152,9 +151,9 @@ func TestSliced(t *testing.T) { vl := sl.AsValues() assert.Equal(t, res, vl.String()) res = `[3, 1] -[0]: 2 -[1]: 12 -[2]: 22 +[0]: 2 +[1]: 12 +[2]: 22 ` sl2 := Reslice(ft, FullAxis, Slice{2, 3, 0}) assert.Equal(t, res, sl2.String()) @@ -175,9 +174,9 @@ func TestMasked(t *testing.T) { res := `[3, 4] [0]: [1]: [2]: [3]: -[0]: 0 1 2 3 -[1]: 10 11 12 13 -[2]: 20 21 22 23 +[0]: 0 1 2 3 +[1]: 10 11 12 13 +[2]: 20 21 22 23 ` assert.Equal(t, res, ms.String()) @@ -187,13 +186,13 @@ func TestMasked(t *testing.T) { }) res = `[3, 4] [0]: [1]: [2]: [3]: -[0]: NaN NaN 2 NaN -[1]: NaN NaN 12 NaN -[2]: NaN NaN 22 NaN +[0]: NaN NaN 2 NaN +[1]: NaN NaN 12 NaN +[2]: NaN NaN 22 NaN ` assert.Equal(t, res, ms.String()) - res = `[3] 2 12 22 + res = `[3] 2 12 22 ` vl := ms.AsValues() assert.Equal(t, res, vl.String()) @@ -223,8 +222,8 @@ func TestIndexed(t *testing.T) { res := `[2, 2, 2] [0 0]: [0 1]: [0 0]: [0 1]: -[0]: 1 1 11 11 -[0]: 2 2 22 22 +[0]: 1 1 11 11 +[0]: 2 2 22 22 ` assert.Equal(t, res, ix.String()) @@ -243,38 +242,37 @@ func TestReshaped(t *testing.T) { res := `[4, 3] [0]: [1]: [2]: -[0]: 0 1 2 -[1]: 3 10 11 -[2]: 12 13 20 -[3]: 21 22 23 +[0]: 0 1 2 +[1]: 3 10 11 +[2]: 12 13 20 +[3]: 21 22 23 ` rs := NewReshaped(ft, 4, 3) assert.Equal(t, res, rs.String()) res = `[1, 3, 4] [0 0]: [0 1]: [0 2]: [0 3]: -[0]: 0 1 2 3 -[0]: 10 11 12 13 -[0]: 20 21 22 23 +[0]: 0 1 2 3 +[0]: 10 11 12 13 +[0]: 20 21 22 23 ` rs = NewReshaped(ft, int(NewAxis), 3, 4) assert.Equal(t, res, rs.String()) res = `[12] -[0]: 0 1 2 3 10 11 12 13 20 21 22 23 +[0]: 0 1 2 3 10 11 12 13 20 21 22 23 ` rs = NewReshaped(ft, -1) assert.Equal(t, res, rs.String()) res = `[4, 3] [0]: [1]: [2]: -[0]: 0 1 2 -[1]: 3 10 11 -[2]: 12 13 20 -[3]: 21 22 23 +[0]: 0 1 2 +[1]: 3 10 11 +[2]: 12 13 20 +[3]: 21 22 23 ` rs = NewReshaped(ft, 4, -1) - // fmt.Println(rs) assert.Equal(t, res, rs.String()) err := rs.SetShapeSizes(5, -1) From a42f5c0c4fb9d4e8db626db67b9acb455d2d4c0c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 22:39:25 -0700 Subject: [PATCH 163/311] tmath.Div division operator always produces floating point, per python --- goal/README.md | 2 +- tensor/funcs.go | 6 ++++++ tensor/tmath/ops.go | 5 +++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/goal/README.md b/goal/README.md index 5547db8a38..e586642a47 100644 --- a/goal/README.md +++ b/goal/README.md @@ -354,7 +354,7 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | ------------ | ----------- | ------ | ------ | ---------------- | | `tmath.Add(a,b)` | same: |`a + b` | `a .+ b` | element-wise addition; Goal does this string-wise for string tensors | | `tmath.Mul(a,b)` | same: |`a * b` | `a .* b` | element-wise multiply | -| `tmath.Div(a,b)` | same: |`a/b` | `a./b` | element-wise divide | +| `tmath.Div(a,b)` | same: |`a/b` | `a./b` | element-wise divide. _important:_ this always produces a floating point result. | | `tmath.Mod(a,b)` | same: |`a%b` | `a./b` | element-wise modulous (works for float and int) | | `tmath.Pow(a,3)` | same: | `a**3` | `a.^3` | element-wise exponentiation | | `tmath.Cos(a)` | same: | `cos(a)` | `cos(a)` | element-wise function application | diff --git a/tensor/funcs.go b/tensor/funcs.go index c68a6c22fe..91a5360e7d 100644 --- a/tensor/funcs.go +++ b/tensor/funcs.go @@ -184,6 +184,12 @@ func CallOut1Float64(fun func(a Tensor, out Values) error, a Tensor) Values { return out } +func CallOut2Float64(fun func(a, b Tensor, out Values) error, a, b Tensor) Values { + out := NewFloat64() + errors.Log(fun(a, b, out)) + return out +} + func CallOut2(fun func(a, b Tensor, out Values) error, a, b Tensor) Values { out := NewOfType(FloatPromoteType(a, b)) errors.Log(fun(a, b, out)) diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index a6930290c4..6b8a5bc924 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -96,9 +96,10 @@ func MulOut(a, b tensor.Tensor, out tensor.Values) error { return tensor.FloatBinaryFuncOut(1, func(a, b float64) float64 { return a * b }, a, b, out) } -// Div divides tensors into output. +// Div divides tensors into output. always does floating point division, +// even with integer operands. func Div(a, b tensor.Tensor) tensor.Tensor { - return tensor.CallOut2(DivOut, a, b) + return tensor.CallOut2Float64(DivOut, a, b) } // DivOut divides two tensors into output. From 07689b9669177ff352ac40f70b04cb544f3cc6c4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 22:51:52 -0700 Subject: [PATCH 164/311] tensor print fix --- tensor/datafs/commands.go | 2 +- tensor/io.go | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tensor/datafs/commands.go b/tensor/datafs/commands.go index 203c4c39bb..bb7397121c 100644 --- a/tensor/datafs/commands.go +++ b/tensor/datafs/commands.go @@ -24,7 +24,7 @@ var ( ) func init() { - CurRoot, _ = NewDir("root") + CurRoot, _ = NewDir("data") CurDir = CurRoot } diff --git a/tensor/io.go b/tensor/io.go index 25a3e2cc16..66e5662487 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -179,13 +179,11 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { } colWd := 1 // column width in tabs defFmt := format == "" - isint := false if defFmt { switch { case tsr.IsString(): format = "%s\t" case reflectx.KindIsInt(tsr.DataType()): - isint = true format = "%g\t" default: format = "%g\t" @@ -212,13 +210,7 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { } } colWd = int(math32.IntMultipleGE(float32(mxwd), 8)) / 8 - if colWd > 1 && !tsr.IsString() && defFmt { // should be 2 - if isint { - format = "%15g\t" - } else { - format = "%15.7g\t" - } - } + fmt.Println("col wd:", colWd) sh := tsr.Shape() oddRow := false rows, cols, _, _ := Projection2DShape(sh, oddRow) @@ -260,7 +252,7 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { b.WriteString(s) nt := int(math32.IntMultipleGE(float32(len(s)), 8)) / 8 if nt < colWd { - b.WriteString(strings.Repeat("\t", nt-colWd)) + b.WriteString(strings.Repeat("\t", colWd-nt)) } } b.WriteString("\n") From d60a50d56d4c1da4ce88ee876e577f90d505feb0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 29 Sep 2024 23:12:28 -0700 Subject: [PATCH 165/311] add pi constant --- goal/transpile/math.go | 10 ++++++++-- yaegicore/symbols/cogentcore_org-core-tensor.go | 3 +++ yaegicore/symbols/make | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 240a825168..a5d09a1916 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -903,12 +903,18 @@ func (mp *mathParse) callName(cf *ast.CallExpr, funName, pkgName string) { mp.addToken(token.LPAREN) } +// basic ident replacements +var consts = map[string]string{ + "newaxis": "tensor.NewAxis", + "pi": "tensor.NewFloat64Scalar(math.Pi)", +} + func (mp *mathParse) ident(id *ast.Ident) { if id == nil { return } - if id.Name == "newaxis" { - mp.out.Add(token.IDENT, "tensor.NewAxis") + if cn, ok := consts[id.Name]; ok { + mp.out.Add(token.IDENT, cn) mp.idx++ return } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 33a0028169..797f2c443c 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -42,8 +42,10 @@ func init() { "BoolToInt": reflect.ValueOf(tensor.BoolToInt), "Calc": reflect.ValueOf(tensor.Calc), "CallOut1": reflect.ValueOf(tensor.CallOut1), + "CallOut1Float64": reflect.ValueOf(tensor.CallOut1Float64), "CallOut2": reflect.ValueOf(tensor.CallOut2), "CallOut2Bool": reflect.ValueOf(tensor.CallOut2Bool), + "CallOut2Float64": reflect.ValueOf(tensor.CallOut2Float64), "CallOut3": reflect.ValueOf(tensor.CallOut3), "Cells1D": reflect.ValueOf(tensor.Cells1D), "CellsSize": reflect.ValueOf(tensor.CellsSize), @@ -64,6 +66,7 @@ func init() { "FloatBinaryFuncOut": reflect.ValueOf(tensor.FloatBinaryFuncOut), "FloatFunc": reflect.ValueOf(tensor.FloatFunc), "FloatFuncOut": reflect.ValueOf(tensor.FloatFuncOut), + "FloatPromoteType": reflect.ValueOf(tensor.FloatPromoteType), "FloatSetFunc": reflect.ValueOf(tensor.FloatSetFunc), "FullAxis": reflect.ValueOf(tensor.FullAxis), "FuncByName": reflect.ValueOf(tensor.FuncByName), diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index c2fe416f10..7f253a1c03 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -1,6 +1,6 @@ #!/usr/bin/env cosh -yaegi extract fmt strconv strings image image/color image/draw time log/slog reflect +yaegi extract fmt strconv strings math image image/color image/draw time log/slog reflect command extract { for _, pkg := range args { From c393a8e22120bd3c33812a5408a97cf28bcf1101 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 00:30:46 -0700 Subject: [PATCH 166/311] start on vector package with basic vector functions: Inner, Sum, Dot, NormL2, NormL1 --- tensor/vector/README.md | 5 + tensor/vector/vector.go | 60 +++++++++ tensor/vector/vector_test.go | 25 ++++ .../cogentcore_org-core-tensor-vector.go | 20 +++ yaegicore/symbols/make | 2 +- yaegicore/symbols/math.go | 116 ++++++++++++++++++ 6 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 tensor/vector/README.md create mode 100644 tensor/vector/vector.go create mode 100644 tensor/vector/vector_test.go create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-vector.go create mode 100644 yaegicore/symbols/math.go diff --git a/tensor/vector/README.md b/tensor/vector/README.md new file mode 100644 index 0000000000..e821fce072 --- /dev/null +++ b/tensor/vector/README.md @@ -0,0 +1,5 @@ +# vector + +vector provides standard vector math functions that always operate on 1D views of tensor inputs, regardless of the original tensor shape. + + diff --git a/tensor/vector/vector.go b/tensor/vector/vector.go new file mode 100644 index 0000000000..6366ed0d1d --- /dev/null +++ b/tensor/vector/vector.go @@ -0,0 +1,60 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package vector provides standard vector math functions that +// always operate on 1D views of tensor inputs regardless of the original +// vector shape. +package vector + +import ( + "math" + + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/tmath" +) + +// Inner multiplies two vectors element-wise, returning the vector. +func Inner(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2Float64(InnerOut, a, b) +} + +// InnerOut multiplies two vectors element-wise. +func InnerOut(a, b tensor.Tensor, out tensor.Values) error { + return tmath.MulOut(tensor.As1D(a), tensor.As1D(b), out) +} + +// Sum returns the sum of all values in the tensor, as a scalar. +func Sum(a tensor.Tensor) tensor.Values { + n := a.Len() + sum := 0.0 + tensor.Vectorize(func(tsr ...tensor.Tensor) int { return n }, + func(idx int, tsr ...tensor.Tensor) { + sum += tsr[0].Float1D(idx) + }, a) + return tensor.NewFloat64Scalar(sum) +} + +// Dot performs a dot product, returning a scalar dot product. +func Dot(a, b tensor.Tensor) tensor.Values { + return Sum(Inner(a, b)) +} + +// NormL2 returns the length of the vector as the L2Norm: +// square root of the sum of squared values of the vector, as a scalar. +func NormL2(a tensor.Tensor) tensor.Values { + sum := Sum(Inner(a, a)).Float1D(0) + return tensor.NewFloat64Scalar(math.Sqrt(sum)) +} + +// NormL1 returns the length of the vector as the L1Norm: +// sum of the absolute values of the tensor, as a scalar. +func NormL1(a tensor.Tensor) tensor.Values { + n := a.Len() + sum := 0.0 + tensor.Vectorize(func(tsr ...tensor.Tensor) int { return n }, + func(idx int, tsr ...tensor.Tensor) { + sum += math.Abs(tsr[0].Float1D(idx)) + }, a) + return tensor.NewFloat64Scalar(sum) +} diff --git a/tensor/vector/vector_test.go b/tensor/vector/vector_test.go new file mode 100644 index 0000000000..49a04deba6 --- /dev/null +++ b/tensor/vector/vector_test.go @@ -0,0 +1,25 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package vector + +import ( + "testing" + + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" +) + +func TestVector(t *testing.T) { + v := tensor.NewFloat64FromValues(1, 2, 3) + ip := Inner(v, v).(*tensor.Float64) + assert.Equal(t, []float64{1, 4, 9}, ip.Values) + + smv := Sum(ip).(*tensor.Float64) + assert.Equal(t, 14.0, smv.Values[0]) + + dpv := Dot(v, v).(*tensor.Float64) + assert.Equal(t, 14.0, dpv.Values[0]) + +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go b/yaegicore/symbols/cogentcore_org-core-tensor-vector.go new file mode 100644 index 0000000000..bf64eb8e99 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-vector.go @@ -0,0 +1,20 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/vector'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/vector" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/vector/vector"] = map[string]reflect.Value{ + // function, constant and variable definitions + "Dot": reflect.ValueOf(vector.Dot), + "Inner": reflect.ValueOf(vector.Inner), + "InnerOut": reflect.ValueOf(vector.InnerOut), + "NormL1": reflect.ValueOf(vector.NormL1), + "NormL2": reflect.ValueOf(vector.NormL2), + "Sum": reflect.ValueOf(vector.Sum), + } +} diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 7f253a1c03..83f777f772 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -8,4 +8,4 @@ command extract { } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo base/num +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/vector tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo base/num diff --git a/yaegicore/symbols/math.go b/yaegicore/symbols/math.go new file mode 100644 index 0000000000..5b846c5773 --- /dev/null +++ b/yaegicore/symbols/math.go @@ -0,0 +1,116 @@ +// Code generated by 'yaegi extract math'. DO NOT EDIT. + +//go:build go1.22 +// +build go1.22 + +package symbols + +import ( + "go/constant" + "go/token" + "math" + "reflect" +) + +func init() { + Symbols["math/math"] = map[string]reflect.Value{ + // function, constant and variable definitions + "Abs": reflect.ValueOf(math.Abs), + "Acos": reflect.ValueOf(math.Acos), + "Acosh": reflect.ValueOf(math.Acosh), + "Asin": reflect.ValueOf(math.Asin), + "Asinh": reflect.ValueOf(math.Asinh), + "Atan": reflect.ValueOf(math.Atan), + "Atan2": reflect.ValueOf(math.Atan2), + "Atanh": reflect.ValueOf(math.Atanh), + "Cbrt": reflect.ValueOf(math.Cbrt), + "Ceil": reflect.ValueOf(math.Ceil), + "Copysign": reflect.ValueOf(math.Copysign), + "Cos": reflect.ValueOf(math.Cos), + "Cosh": reflect.ValueOf(math.Cosh), + "Dim": reflect.ValueOf(math.Dim), + "E": reflect.ValueOf(constant.MakeFromLiteral("2.71828182845904523536028747135266249775724709369995957496696762566337824315673231520670375558666729784504486779277967997696994772644702281675346915668215131895555530285035761295375777990557253360748291015625", token.FLOAT, 0)), + "Erf": reflect.ValueOf(math.Erf), + "Erfc": reflect.ValueOf(math.Erfc), + "Erfcinv": reflect.ValueOf(math.Erfcinv), + "Erfinv": reflect.ValueOf(math.Erfinv), + "Exp": reflect.ValueOf(math.Exp), + "Exp2": reflect.ValueOf(math.Exp2), + "Expm1": reflect.ValueOf(math.Expm1), + "FMA": reflect.ValueOf(math.FMA), + "Float32bits": reflect.ValueOf(math.Float32bits), + "Float32frombits": reflect.ValueOf(math.Float32frombits), + "Float64bits": reflect.ValueOf(math.Float64bits), + "Float64frombits": reflect.ValueOf(math.Float64frombits), + "Floor": reflect.ValueOf(math.Floor), + "Frexp": reflect.ValueOf(math.Frexp), + "Gamma": reflect.ValueOf(math.Gamma), + "Hypot": reflect.ValueOf(math.Hypot), + "Ilogb": reflect.ValueOf(math.Ilogb), + "Inf": reflect.ValueOf(math.Inf), + "IsInf": reflect.ValueOf(math.IsInf), + "IsNaN": reflect.ValueOf(math.IsNaN), + "J0": reflect.ValueOf(math.J0), + "J1": reflect.ValueOf(math.J1), + "Jn": reflect.ValueOf(math.Jn), + "Ldexp": reflect.ValueOf(math.Ldexp), + "Lgamma": reflect.ValueOf(math.Lgamma), + "Ln10": reflect.ValueOf(constant.MakeFromLiteral("2.30258509299404568401799145468436420760110148862877297603332784146804725494827975466552490443295866962642372461496758838959542646932914211937012833592062802600362869664962772731087170541286468505859375", token.FLOAT, 0)), + "Ln2": reflect.ValueOf(constant.MakeFromLiteral("0.6931471805599453094172321214581765680755001343602552541206800092715999496201383079363438206637927920954189307729314303884387720696314608777673678644642390655170150035209453154294578780536539852619171142578125", token.FLOAT, 0)), + "Log": reflect.ValueOf(math.Log), + "Log10": reflect.ValueOf(math.Log10), + "Log10E": reflect.ValueOf(constant.MakeFromLiteral("0.43429448190325182765112891891660508229439700580366656611445378416636798190620320263064286300825210972160277489744884502676719847561509639618196799746596688688378591625127711495224502868950366973876953125", token.FLOAT, 0)), + "Log1p": reflect.ValueOf(math.Log1p), + "Log2": reflect.ValueOf(math.Log2), + "Log2E": reflect.ValueOf(constant.MakeFromLiteral("1.44269504088896340735992468100189213742664595415298593413544940772066427768997545329060870636212628972710992130324953463427359402479619301286929040235571747101382214539290471666532766903401352465152740478515625", token.FLOAT, 0)), + "Logb": reflect.ValueOf(math.Logb), + "Max": reflect.ValueOf(math.Max), + "MaxFloat32": reflect.ValueOf(constant.MakeFromLiteral("340282346638528859811704183484516925440", token.FLOAT, 0)), + "MaxFloat64": reflect.ValueOf(constant.MakeFromLiteral("179769313486231570814527423731704356798070567525844996598917476803157260780028538760589558632766878171540458953514382464234321326889464182768467546703537516986049910576551282076245490090389328944075868508455133942304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368", token.FLOAT, 0)), + "MaxInt": reflect.ValueOf(constant.MakeFromLiteral("9223372036854775807", token.INT, 0)), + "MaxInt16": reflect.ValueOf(constant.MakeFromLiteral("32767", token.INT, 0)), + "MaxInt32": reflect.ValueOf(constant.MakeFromLiteral("2147483647", token.INT, 0)), + "MaxInt64": reflect.ValueOf(constant.MakeFromLiteral("9223372036854775807", token.INT, 0)), + "MaxInt8": reflect.ValueOf(constant.MakeFromLiteral("127", token.INT, 0)), + "MaxUint": reflect.ValueOf(constant.MakeFromLiteral("18446744073709551615", token.INT, 0)), + "MaxUint16": reflect.ValueOf(constant.MakeFromLiteral("65535", token.INT, 0)), + "MaxUint32": reflect.ValueOf(constant.MakeFromLiteral("4294967295", token.INT, 0)), + "MaxUint64": reflect.ValueOf(constant.MakeFromLiteral("18446744073709551615", token.INT, 0)), + "MaxUint8": reflect.ValueOf(constant.MakeFromLiteral("255", token.INT, 0)), + "Min": reflect.ValueOf(math.Min), + "MinInt": reflect.ValueOf(constant.MakeFromLiteral("-9223372036854775808", token.INT, 0)), + "MinInt16": reflect.ValueOf(constant.MakeFromLiteral("-32768", token.INT, 0)), + "MinInt32": reflect.ValueOf(constant.MakeFromLiteral("-2147483648", token.INT, 0)), + "MinInt64": reflect.ValueOf(constant.MakeFromLiteral("-9223372036854775808", token.INT, 0)), + "MinInt8": reflect.ValueOf(constant.MakeFromLiteral("-128", token.INT, 0)), + "Mod": reflect.ValueOf(math.Mod), + "Modf": reflect.ValueOf(math.Modf), + "NaN": reflect.ValueOf(math.NaN), + "Nextafter": reflect.ValueOf(math.Nextafter), + "Nextafter32": reflect.ValueOf(math.Nextafter32), + "Phi": reflect.ValueOf(constant.MakeFromLiteral("1.6180339887498948482045868343656381177203091798057628621354486119746080982153796619881086049305501566952211682590824739205931370737029882996587050475921915678674035433959321750307935872115194797515869140625", token.FLOAT, 0)), + "Pi": reflect.ValueOf(constant.MakeFromLiteral("3.141592653589793238462643383279502884197169399375105820974944594789982923695635954704435713335896673485663389728754819466702315787113662862838515639906529162340867271374644786874341662041842937469482421875", token.FLOAT, 0)), + "Pow": reflect.ValueOf(math.Pow), + "Pow10": reflect.ValueOf(math.Pow10), + "Remainder": reflect.ValueOf(math.Remainder), + "Round": reflect.ValueOf(math.Round), + "RoundToEven": reflect.ValueOf(math.RoundToEven), + "Signbit": reflect.ValueOf(math.Signbit), + "Sin": reflect.ValueOf(math.Sin), + "Sincos": reflect.ValueOf(math.Sincos), + "Sinh": reflect.ValueOf(math.Sinh), + "SmallestNonzeroFloat32": reflect.ValueOf(constant.MakeFromLiteral("1.40129846432481707092372958328991613128026194187651577175706828388979108268586060148663818836212158203125e-45", token.FLOAT, 0)), + "SmallestNonzeroFloat64": reflect.ValueOf(constant.MakeFromLiteral("4.940656458412465441765687928682213723650598026143247644255856825006755072702087518652998363616359923797965646954457177309266567103559397963987747960107818781263007131903114045278458171678489821036887186360569987307230500063874091535649843873124733972731696151400317153853980741262385655911710266585566867681870395603106249319452715914924553293054565444011274801297099995419319894090804165633245247571478690147267801593552386115501348035264934720193790268107107491703332226844753335720832431936092382893458368060106011506169809753078342277318329247904982524730776375927247874656084778203734469699533647017972677717585125660551199131504891101451037862738167250955837389733598993664809941164205702637090279242767544565229087538682506419718265533447265625e-324", token.FLOAT, 0)), + "Sqrt": reflect.ValueOf(math.Sqrt), + "Sqrt2": reflect.ValueOf(constant.MakeFromLiteral("1.414213562373095048801688724209698078569671875376948073176679739576083351575381440094441524123797447886801949755143139115339040409162552642832693297721230919563348109313505318596071447245776653289794921875", token.FLOAT, 0)), + "SqrtE": reflect.ValueOf(constant.MakeFromLiteral("1.64872127070012814684865078781416357165377610071014801157507931167328763229187870850146925823776361770041160388013884200789716007979526823569827080974091691342077871211546646890155898290686309337615966796875", token.FLOAT, 0)), + "SqrtPhi": reflect.ValueOf(constant.MakeFromLiteral("1.2720196495140689642524224617374914917156080418400962486166403754616080542166459302584536396369727769747312116100875915825863540562126478288118732191412003988041797518382391984914647764526307582855224609375", token.FLOAT, 0)), + "SqrtPi": reflect.ValueOf(constant.MakeFromLiteral("1.772453850905516027298167483341145182797549456122387128213807789740599698370237052541269446184448945647349951047154197675245574635259260134350885938555625028620527962319730619356050738133490085601806640625", token.FLOAT, 0)), + "Tan": reflect.ValueOf(math.Tan), + "Tanh": reflect.ValueOf(math.Tanh), + "Trunc": reflect.ValueOf(math.Trunc), + "Y0": reflect.ValueOf(math.Y0), + "Y1": reflect.ValueOf(math.Y1), + "Yn": reflect.ValueOf(math.Yn), + } +} From 1b21ad19cf381e371b7fe932b41bdce100d59da2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 00:42:39 -0700 Subject: [PATCH 167/311] vector: improved docs and tests --- tensor/vector/vector.go | 14 +++++++++----- tensor/vector/vector_test.go | 6 ++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tensor/vector/vector.go b/tensor/vector/vector.go index 6366ed0d1d..29a3083c4a 100644 --- a/tensor/vector/vector.go +++ b/tensor/vector/vector.go @@ -14,12 +14,14 @@ import ( "cogentcore.org/core/tensor/tmath" ) -// Inner multiplies two vectors element-wise, returning the vector. +// Inner multiplies two vectors element-wise, using a 1D vector +// view of the two vectors, returning the output 1D vector. func Inner(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2Float64(InnerOut, a, b) } -// InnerOut multiplies two vectors element-wise. +// InnerOut multiplies two vectors element-wise, using a 1D vector +// view of the two vectors, filling in values in the output 1D vector. func InnerOut(a, b tensor.Tensor, out tensor.Values) error { return tmath.MulOut(tensor.As1D(a), tensor.As1D(b), out) } @@ -35,16 +37,18 @@ func Sum(a tensor.Tensor) tensor.Values { return tensor.NewFloat64Scalar(sum) } -// Dot performs a dot product, returning a scalar dot product. +// Dot performs the dot product: the [Sum] of the [Inner] product +// of the two tensors, returning a scalar value. func Dot(a, b tensor.Tensor) tensor.Values { return Sum(Inner(a, b)) } // NormL2 returns the length of the vector as the L2Norm: // square root of the sum of squared values of the vector, as a scalar. +// This is the Sqrt of the [Dot] product of the vector with itself. func NormL2(a tensor.Tensor) tensor.Values { - sum := Sum(Inner(a, a)).Float1D(0) - return tensor.NewFloat64Scalar(math.Sqrt(sum)) + dot := Dot(a, a).Float1D(0) + return tensor.NewFloat64Scalar(math.Sqrt(dot)) } // NormL1 returns the length of the vector as the L1Norm: diff --git a/tensor/vector/vector_test.go b/tensor/vector/vector_test.go index 49a04deba6..bb19e89148 100644 --- a/tensor/vector/vector_test.go +++ b/tensor/vector/vector_test.go @@ -5,6 +5,7 @@ package vector import ( + "math" "testing" "cogentcore.org/core/tensor" @@ -22,4 +23,9 @@ func TestVector(t *testing.T) { dpv := Dot(v, v).(*tensor.Float64) assert.Equal(t, 14.0, dpv.Values[0]) + nl2v := NormL2(v).(*tensor.Float64) + assert.Equal(t, math.Sqrt(14.0), nl2v.Values[0]) + + nl1v := NormL1(v).(*tensor.Float64) + assert.Equal(t, 6.0, nl1v.Values[0]) } From 8a20759a3c733fb5095bf898813ed17fb89cfab7 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 11:32:20 -0700 Subject: [PATCH 168/311] remove debug print in sprintf and change params order with format first per actual sprintf --- tensor/bool.go | 2 +- tensor/indexed.go | 2 +- tensor/io.go | 3 +-- tensor/masked.go | 2 +- tensor/number.go | 2 +- tensor/reshaped.go | 2 +- tensor/rows.go | 2 +- tensor/sliced.go | 2 +- tensor/string.go | 2 +- 9 files changed, 9 insertions(+), 10 deletions(-) diff --git a/tensor/bool.go b/tensor/bool.go index 7233563d9c..3f4d0efcd2 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -63,7 +63,7 @@ func BoolToInt(bv bool) int { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Bool) String() string { return Sprintf(tsr, 0, "") } +func (tsr *Bool) String() string { return Sprintf("", tsr, 0) } // Label satisfies the core.Labeler interface for a summary description of the tensor func (tsr *Bool) Label() string { diff --git a/tensor/indexed.go b/tensor/indexed.go index 4beb62b8b6..90a5d1de2a 100644 --- a/tensor/indexed.go +++ b/tensor/indexed.go @@ -80,7 +80,7 @@ func (ix *Indexed) SourceIndexesFrom1D(oned int) []int { } func (ix *Indexed) Label() string { return label(ix.Metadata().Name(), ix.Shape()) } -func (ix *Indexed) String() string { return Sprintf(ix, 0, "") } +func (ix *Indexed) String() string { return Sprintf("", ix, 0) } func (ix *Indexed) Metadata() *metadata.Data { return ix.Tensor.Metadata() } func (ix *Indexed) IsString() bool { return ix.Tensor.IsString() } func (ix *Indexed) DataType() reflect.Kind { return ix.Tensor.DataType() } diff --git a/tensor/io.go b/tensor/io.go index 66e5662487..b4d0aba067 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -173,7 +173,7 @@ func label(nm string, sh *Shape) string { // The format is the per-element format string, which should include // any delimiter or spacing between elements (which will apply to last // element too). If empty it uses general %g or %s defaults with a tab separator. -func Sprintf(tsr Tensor, maxLen int, format string) string { +func Sprintf(format string, tsr Tensor, maxLen int) string { if maxLen == 0 { maxLen = MaxSprintLength } @@ -210,7 +210,6 @@ func Sprintf(tsr Tensor, maxLen int, format string) string { } } colWd = int(math32.IntMultipleGE(float32(mxwd), 8)) / 8 - fmt.Println("col wd:", colWd) sh := tsr.Shape() oddRow := false rows, cols, _, _ := Projection2DShape(sh, oddRow) diff --git a/tensor/masked.go b/tensor/masked.go index dee9053d4a..39f42cd9f5 100644 --- a/tensor/masked.go +++ b/tensor/masked.go @@ -89,7 +89,7 @@ func (ms *Masked) SyncShape() { } func (ms *Masked) Label() string { return label(ms.Metadata().Name(), ms.Shape()) } -func (ms *Masked) String() string { return Sprintf(ms, 0, "") } +func (ms *Masked) String() string { return Sprintf("", ms, 0) } func (ms *Masked) Metadata() *metadata.Data { return ms.Tensor.Metadata() } func (ms *Masked) IsString() bool { return ms.Tensor.IsString() } func (ms *Masked) DataType() reflect.Kind { return ms.Tensor.DataType() } diff --git a/tensor/number.go b/tensor/number.go index b6fc9940e4..8d9b9d66fa 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -95,7 +95,7 @@ func NewNumberFromValues[T num.Number](vals ...T) *Number[T] { } // String satisfies the fmt.Stringer interface for string of tensor data. -func (tsr *Number[T]) String() string { return Sprintf(tsr, 0, "") } +func (tsr *Number[T]) String() string { return Sprintf("", tsr, 0) } func (tsr *Number[T]) IsString() bool { return false } diff --git a/tensor/reshaped.go b/tensor/reshaped.go index df4f126ffb..da3d924f47 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -143,7 +143,7 @@ func (rs *Reshaped) SetShapeSizes(sizes ...int) error { } func (rs *Reshaped) Label() string { return label(rs.Metadata().Name(), rs.Shape()) } -func (rs *Reshaped) String() string { return Sprintf(rs, 0, "") } +func (rs *Reshaped) String() string { return Sprintf("", rs, 0) } func (rs *Reshaped) Metadata() *metadata.Data { return rs.Tensor.Metadata() } func (rs *Reshaped) IsString() bool { return rs.Tensor.IsString() } func (rs *Reshaped) DataType() reflect.Kind { return rs.Tensor.DataType() } diff --git a/tensor/rows.go b/tensor/rows.go index ec80da635f..5f98d0dec8 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -86,7 +86,7 @@ func (rw *Rows) NumRows() int { return len(rw.Indexes) } -func (rw *Rows) String() string { return Sprintf(rw.Tensor, 0, "") } +func (rw *Rows) String() string { return Sprintf("", rw.Tensor, 0) } func (rw *Rows) Label() string { return rw.Tensor.Label() } func (rw *Rows) Metadata() *metadata.Data { return rw.Tensor.Metadata() } func (rw *Rows) NumDims() int { return rw.Tensor.NumDims() } diff --git a/tensor/sliced.go b/tensor/sliced.go index 00429f2247..7a5cad28e0 100644 --- a/tensor/sliced.go +++ b/tensor/sliced.go @@ -223,7 +223,7 @@ func (sl *Sliced) IndexesNeeded(d int) { } func (sl *Sliced) Label() string { return label(sl.Metadata().Name(), sl.Shape()) } -func (sl *Sliced) String() string { return Sprintf(sl, 0, "") } +func (sl *Sliced) String() string { return Sprintf("", sl, 0) } func (sl *Sliced) Metadata() *metadata.Data { return sl.Tensor.Metadata() } func (sl *Sliced) IsString() bool { return sl.Tensor.IsString() } func (sl *Sliced) DataType() reflect.Kind { return sl.Tensor.DataType() } diff --git a/tensor/string.go b/tensor/string.go index 8aab94accc..ed4cacf06d 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -50,7 +50,7 @@ func Float64ToString(val float64) string { // String satisfies the fmt.Stringer interface for string of tensor data. func (tsr *String) String() string { - return Sprintf(tsr, 0, "") + return Sprintf("", tsr, 0) } func (tsr *String) IsString() bool { From b79ca78a59f432b5aa467c8aec0f48632d76b3ad Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 12:48:35 -0700 Subject: [PATCH 169/311] fix += etc --- goal/transpile/math.go | 19 +++++++++++-------- goal/transpile/transpile_test.go | 3 ++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index a5d09a1916..785472ed8c 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -83,7 +83,8 @@ type funcInfo struct { curArg int } -// mathParse has the parsing state +// mathParse has the parsing state, active only during a parsing pass +// on one specific chunk of code and tokens. type mathParse struct { state *State code string // code string @@ -566,13 +567,15 @@ func (mp *mathParse) defineStmt(as *ast.AssignStmt) { } func (mp *mathParse) assignStmt(as *ast.AssignStmt) { - if _, ok := as.Lhs[0].(*ast.Ident); ok { - mp.exprList(as.Lhs) - mp.addToken(as.Tok) - mp.startFunc("") - mp.exprList(as.Rhs) - mp.endFunc() - return + if as.Tok == token.ASSIGN { + if _, ok := as.Lhs[0].(*ast.Ident); ok { + mp.exprList(as.Lhs) + mp.addToken(as.Tok) + mp.startFunc("") + mp.exprList(as.Rhs) + mp.endFunc() + return + } } fn := "" switch as.Tok { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index e593bd01bb..2acd0487fd 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {`fmt.Println(#zeros(3,4)#)`, `fmt.Println(tensor.NewFloat64(3, 4))`}, + {"# x += 1", `tmath.AddAssign(x, tensor.NewIntScalar(1))`}, } st := NewState() st.MathRecord = false @@ -223,6 +223,7 @@ func TestMath(t *testing.T) { {"# a = x + y", `a = tmath.Add(x, y)`}, {"# a := x ** 2", `a := tmath.Pow(x, tensor.NewIntScalar(2))`}, {"# a = -x", `a = tmath.Negate(x)`}, + {"# x += 1", `tmath.AddAssign(x, tensor.NewIntScalar(1))`}, {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, {"# a := [1.,2,3,4]", `a := tensor.NewFloat64FromValues([]float64 { 1., 2, 3, 4 } ...)`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, From d498db9e6f45d8fd16f3da43040213925a139a58 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 13:10:21 -0700 Subject: [PATCH 170/311] define always does a type conversion to tensor.Tensor so all variables are maximally assignable. --- goal/transpile/math.go | 8 +++-- goal/transpile/transpile_test.go | 56 ++++++++++++++++---------------- 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 785472ed8c..a706c61045 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -150,8 +150,8 @@ func (mp *mathParse) curArgIsInts() bool { func (mp *mathParse) startFunc(name string, noLookup ...bool) *funcInfo { fi := &funcInfo{} sname := name - if name == "" { - sname = "tmath.Inc" + if name == "" || name == "tensor.Tensor" { + sname = "tmath.Inc" // one arg tensor fun } if len(noLookup) == 1 && noLookup[0] { fi.Name = name @@ -554,8 +554,10 @@ func (mp *mathParse) defineStmt(as *ast.AssignStmt) { firstStmt := mp.idx == 0 mp.exprList(as.Lhs) mp.addToken(as.Tok) - mp.startFunc("") // dummy single arg tensor function + mp.startFunc("tensor.Tensor") + mp.out.Add(token.LPAREN) mp.exprList(as.Rhs) + mp.out.Add(token.RPAREN) mp.endFunc() if firstStmt && mp.state.MathRecord { nvar, ok := as.Lhs[0].(*ast.Ident) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 2acd0487fd..655b69f589 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -217,29 +217,29 @@ func TestCur(t *testing.T) { func TestMath(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x := 1", `x := tensor.NewIntScalar(1)`}, - {"# x := a + 1", `x := tmath.Add(a, tensor.NewIntScalar(1))`}, + {"# x := 1", `x := tensor.Tensor(tensor.NewIntScalar(1))`}, + {"# x := a + 1", `x := tensor.Tensor(tmath.Add(a, tensor.NewIntScalar(1)))`}, {"# x = x * 4", `x = tmath.Mul(x, tensor.NewIntScalar(4))`}, {"# a = x + y", `a = tmath.Add(x, y)`}, - {"# a := x ** 2", `a := tmath.Pow(x, tensor.NewIntScalar(2))`}, + {"# a := x ** 2", `a := tensor.Tensor(tmath.Pow(x, tensor.NewIntScalar(2)))`}, {"# a = -x", `a = tmath.Negate(x)`}, {"# x += 1", `tmath.AddAssign(x, tensor.NewIntScalar(1))`}, - {"# a := [1,2,3,4]", `a := tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...)`}, - {"# a := [1.,2,3,4]", `a := tensor.NewFloat64FromValues([]float64 { 1., 2, 3, 4 } ...)`}, + {"# a := [1,2,3,4]", `a := tensor.Tensor(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...))`}, + {"# a := [1.,2,3,4]", `a := tensor.Tensor(tensor.NewFloat64FromValues([]float64 { 1., 2, 3, 4 } ...))`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, - {"# sh := x.shape", `sh := tensor.NewIntFromValues(x.Shape().Sizes ...)`}, - {"# a := zeros(3, 4)", `a := tensor.NewFloat64(3, 4)`}, - {"# a := full(5.5, 3, 4)", `a := tensor.NewFloat64Full(5.5, 3, 4)`}, - {"# a := zeros(sh)", `a := tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, - {"# a := arange(36)", `a := tensor.NewIntRange(36)`}, - {"# a := arange(36, 0, -1)", `a := tensor.NewIntRange(36, 0, - 1)`}, - {"# a := linspace(0, 5, 6, true)", `a := tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, - {"# a := reshape(x, 6, 6)", `a := tensor.Reshape(x, 6, 6)`}, - {"# a := reshape(x, [6, 6])", `a := tensor.Reshape(x, 6, 6)`}, - {"# a := reshape(x, sh)", `a := tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, - {"# a := reshape(arange(36), 6, 6)", `a := tensor.Reshape(tensor.NewIntRange(36), 6, 6)`}, - {"# y := a.reshape(6, 6)", `y := tensor.Reshape(a, 6, 6)`}, + {"# x.shape", `tensor.NewIntFromValues(x.Shape().Sizes ...)`}, + {"# zeros(3, 4)", `tensor.NewFloat64(3, 4)`}, + {"# full(5.5, 3, 4)", `tensor.NewFloat64Full(5.5, 3, 4)`}, + {"# zeros(sh)", `tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, + {"# arange(36)", `tensor.NewIntRange(36)`}, + {"# arange(36, 0, -1)", `tensor.NewIntRange(36, 0, - 1)`}, + {"# linspace(0, 5, 6, true)", `tensor.NewFloat64SpacedLinear(tensor.NewIntScalar(0), tensor.NewIntScalar(5), 6, true)`}, + {"# reshape(x, 6, 6)", `tensor.Reshape(x, 6, 6)`}, + {"# reshape(x, [6, 6])", `tensor.Reshape(x, 6, 6)`}, + {"# reshape(x, sh)", `tensor.Reshape(x, tensor.AsIntSlice(sh) ...)`}, + {"# reshape(arange(36), 6, 6)", `tensor.Reshape(tensor.NewIntRange(36), 6, 6)`}, + {"# a.reshape(6, 6)", `tensor.Reshape(a, 6, 6)`}, {"# a[1, 2]", `tensor.Reslice(a, 1, 2)`}, {"# a[:, 2]", `tensor.Reslice(a, tensor.FullAxis, 2)`}, {"# a[1:3:1, 2]", `tensor.Reslice(a, tensor.Slice { Start:1, Stop:3, Step:1 } , 2)`}, @@ -250,25 +250,25 @@ func TestMath(t *testing.T) { {"# a[..., 2:]", `tensor.Reslice(a, tensor.Ellipsis, tensor.Slice { Start:2 } )`}, {"# a[:, 2] = b", `tmath.Assign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, {"# a[:, 2] += b", `tmath.AddAssign(tensor.Reslice(a, tensor.FullAxis, 2), b)`}, - {"# c := cos(a)", `c := tmath.Cos(a)`}, - {"# m := stats.Mean(a)", `m := stats.Mean(a)`}, - {"# m := (stats.Mean(a))", `m := (stats.Mean(a))`}, - {"# m := stats.Mean(reshape(a,36))", `m := stats.Mean(tensor.Reshape(a, 36))`}, + {"# cos(a)", `tmath.Cos(a)`}, + {"# stats.Mean(a)", `stats.Mean(a)`}, + {"# (stats.Mean(a))", `(stats.Mean(a))`}, + {"# stats.Mean(reshape(a,36))", `stats.Mean(tensor.Reshape(a, 36))`}, {"# z = a[1:5,1:5] - stats.Mean(ra)", `z = tmath.Sub(tensor.Reslice(a, tensor.Slice { Start:1, Stop:5 } , tensor.Slice { Start:1, Stop:5 } ), stats.Mean(ra))`}, - {"# m := metric.Matrix(metric.Cosine, a)", `m := metric.Matrix(metric.Cosine, a)`}, - {"# b := a > 5", `b := tmath.Greater(a, tensor.NewIntScalar(5))`}, - {"# b := !a", `b := tmath.Not(a)`}, - {"# b := a[a > 5]", `b := tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5)))`}, - {"# b := a[a > 5].flatten()", `b := tensor.Flatten(tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5))))`}, + {"# metric.Matrix(metric.Cosine, a)", `metric.Matrix(metric.Cosine, a)`}, + {"# a > 5", `tmath.Greater(a, tensor.NewIntScalar(5))`}, + {"# !a", `tmath.Not(a)`}, + {"# a[a > 5]", `tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5)))`}, + {"# a[a > 5].flatten()", `tensor.Flatten(tensor.Mask(a, tmath.Greater(a, tensor.NewIntScalar(5))))`}, {"# a[:3, 2].copy()", `tensor.Clone(tensor.Reslice(a, tensor.Slice { Stop:3 } , 2))`}, {"# a[:3, 2].reshape(4,2)", `tensor.Reshape(tensor.Reslice(a, tensor.Slice { Stop:3 } , 2), 4, 2)`}, {"# a > 5 || a < 1", `tmath.Or(tmath.Greater(a, tensor.NewIntScalar(5)), tmath.Less(a, tensor.NewIntScalar(1)))`}, {"# fmt.Println(a)", `fmt.Println(a)`}, {"# }", `}`}, {"# if a[1,2] == 2 {", `if tmath.Equal(tensor.Reslice(a, 1, 2), tensor.NewIntScalar(2)).Bool1D(0) {`}, - {"# for i := 0; i < 3; i++ {", `for i := tensor.NewIntScalar(0); tmath.Less(i, tensor.NewIntScalar(3)).Bool1D(0); tmath.Inc(i) {`}, + {"# for i := 0; i < 3; i++ {", `for i := tensor.Tensor(tensor.NewIntScalar(0)); tmath.Less(i, tensor.NewIntScalar(3)).Bool1D(0); tmath.Inc(i) {`}, {"# for i, v := range a {", `for i := 0; i < a.Len(); i++ { v := a .Float1D(i)`}, - {`# x := get("item")`, `x := datafs.Get("item")`}, + {`# x := get("item")`, `x := tensor.Tensor(datafs.Get("item"))`}, {`# set("item", x)`, `datafs.Set("item", x)`}, {`# set("item", 5)`, `datafs.Set("item", tensor.NewIntScalar(5))`}, {`fmt.Println(#zeros(3,4)#)`, `fmt.Println(tensor.NewFloat64(3, 4))`}, From 263b2625f59845105408aace12b54ce5b3fd3b8d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 14:06:31 -0700 Subject: [PATCH 171/311] rename vector.Inner -> vector.Mul --- tensor/vector/vector.go | 14 +++++++------- .../symbols/cogentcore_org-core-tensor-vector.go | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tensor/vector/vector.go b/tensor/vector/vector.go index 29a3083c4a..85f8bfbc85 100644 --- a/tensor/vector/vector.go +++ b/tensor/vector/vector.go @@ -14,15 +14,15 @@ import ( "cogentcore.org/core/tensor/tmath" ) -// Inner multiplies two vectors element-wise, using a 1D vector +// Mul multiplies two vectors element-wise, using a 1D vector // view of the two vectors, returning the output 1D vector. -func Inner(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2Float64(InnerOut, a, b) +func Mul(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2Float64(MulOut, a, b) } -// InnerOut multiplies two vectors element-wise, using a 1D vector +// MulOut multiplies two vectors element-wise, using a 1D vector // view of the two vectors, filling in values in the output 1D vector. -func InnerOut(a, b tensor.Tensor, out tensor.Values) error { +func MulOut(a, b tensor.Tensor, out tensor.Values) error { return tmath.MulOut(tensor.As1D(a), tensor.As1D(b), out) } @@ -37,10 +37,10 @@ func Sum(a tensor.Tensor) tensor.Values { return tensor.NewFloat64Scalar(sum) } -// Dot performs the dot product: the [Sum] of the [Inner] product +// Dot performs the vector dot product: the [Sum] of the [Mul] product // of the two tensors, returning a scalar value. func Dot(a, b tensor.Tensor) tensor.Values { - return Sum(Inner(a, b)) + return Sum(Mul(a, b)) } // NormL2 returns the length of the vector as the L2Norm: diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go b/yaegicore/symbols/cogentcore_org-core-tensor-vector.go index bf64eb8e99..5a1d61788f 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-vector.go @@ -10,11 +10,11 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/vector/vector"] = map[string]reflect.Value{ // function, constant and variable definitions - "Dot": reflect.ValueOf(vector.Dot), - "Inner": reflect.ValueOf(vector.Inner), - "InnerOut": reflect.ValueOf(vector.InnerOut), - "NormL1": reflect.ValueOf(vector.NormL1), - "NormL2": reflect.ValueOf(vector.NormL2), - "Sum": reflect.ValueOf(vector.Sum), + "Dot": reflect.ValueOf(vector.Dot), + "Mul": reflect.ValueOf(vector.Mul), + "MulOut": reflect.ValueOf(vector.MulOut), + "NormL1": reflect.ValueOf(vector.NormL1), + "NormL2": reflect.ValueOf(vector.NormL2), + "Sum": reflect.ValueOf(vector.Sum), } } From 0c8d146f38b36067018aff0d92adca33186262a2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 14:51:02 -0700 Subject: [PATCH 172/311] use consistent NormL1 and NormL2 throughout the stats, metric, vector code --- goal/README.md | 2 +- tensor/stats/cluster/clust_test.go | 2 +- tensor/stats/metric/README.md | 12 +- tensor/stats/metric/enumgen.go | 6 +- tensor/stats/metric/funcs.go | 111 ++++++++++++------ tensor/stats/metric/metric_test.go | 20 ++-- tensor/stats/metric/metrics.go | 34 +++--- tensor/stats/stats/README.md | 5 +- tensor/stats/stats/enumgen.go | 10 +- tensor/stats/stats/funcs.go | 26 ++-- tensor/stats/stats/stats.go | 14 +-- tensor/stats/stats/stats_test.go | 20 ++-- tensor/vector/vector.go | 4 +- tensor/vector/vector_test.go | 2 +- ...cogentcore_org-core-tensor-stats-metric.go | 24 ++-- .../cogentcore_org-core-tensor-stats-stats.go | 15 ++- 16 files changed, 167 insertions(+), 140 deletions(-) diff --git a/goal/README.md b/goal/README.md index e586642a47..2db95fb460 100644 --- a/goal/README.md +++ b/goal/README.md @@ -390,7 +390,7 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | | |`a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | | | |`a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | | | |`np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | -| `stats.L2Norm(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | +| `stats.NormL2(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | | | |`cg` | `conjgrad` | conjugate gradients solver | ## FFT and complex numbers diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index 2338df4b02..d6ff053a98 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -34,7 +34,7 @@ func TestClust(t *testing.T) { t.Error(err) } in := dt.Column("Input") - out := metric.Matrix(metric.Euclidean, in) + out := metric.Matrix(metric.NormL2, in) cl := Cluster(Avg.String(), out, dt.Column("Name")) diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 8f0f3133ca..728a880eeb 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -9,7 +9,7 @@ The metric functions always operate on the outermost _row_ dimension, and it is * To obtain a single summary metric across all values, use `tensor.As1D`. -* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the metric for each such cell computed across the outer rows dimension. For the `Euclidean` metric for example, each cell has the difference for that cell value across all the rows between the two tensors. See [Matrix functions](#matrix-functions) below for a function that computes the distances _between each cell pattern and all the others_, as a distance or similarity matrix. +* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the metric for each such cell computed across the outer rows dimension. For the `NormL2` metric for example, each cell has the difference for that cell value across all the rows between the two tensors. See [Matrix functions](#matrix-functions) below for a function that computes the distances _between each cell pattern and all the others_, as a distance or similarity matrix. * Use `tensor.NewRowCellsView` to reshape any tensor into a 2D rows x cells shape, with the cells starting at a given dimension. Thus, any number of outer dimensions can be collapsed into the outer row dimension, and the remaining dimensions become the cells. @@ -17,11 +17,11 @@ The metric functions always operate on the outermost _row_ dimension, and it is ### Value _increases_ with increasing distance (i.e., difference metric) -* `Euclidean` or `L2Norm`: the square root of the sum of squares differences between tensor values. +* `NormL2`: the square root of the sum of squares differences between tensor values. * `SumSquares`: the sum of squares differences between tensor values. -* `Abs`or `L2Norm`: the sum of the absolute value of differences between tensor values. +* `Abs`or `NormL2`: the sum of the absolute value of differences between tensor values. * `Hamming`: the sum of 1s for every element that is different, i.e., "city block" distance. -* `EuclideanBinTol`: the `Euclidean` square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. +* `NormL2BinTol`: the `NormL2` square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. * `SumSquaresBinTol`: the `SumSquares` differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. * `InvCosine`: is 1-`Cosine`, which is useful to convert it to an Increasing metric where more different vectors have larger metric values. * `InvCorrelation`: is 1-`Correlation`, which is useful to convert it to an Increasing metric where more different vectors have larger metric values. @@ -29,10 +29,10 @@ The metric functions always operate on the outermost _row_ dimension, and it is ### Value _decreases_ with increasing distance (i.e., similarity metric) -* `InnerProduct`: the sum of the co-products of the tensor values. +* `DotProduct`: the sum of the co-products of the tensor values. * `Covariance`: the co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. * `Correlation`: the standardized `Covariance` in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the `Cosine` of mean-normalized vectors. -* `Cosine`: the high-dimensional angle between two vectors, in range (-1..1) as the normalized `InnerProduct`: inner product / sqrt(ssA * ssB). See also `Correlation`. +* `Cosine`: the high-dimensional angle between two vectors, in range (-1..1) as the normalized `DotProduct`: inner product / sqrt(ssA * ssB). See also `Correlation`. Here is general info about these functions: diff --git a/tensor/stats/metric/enumgen.go b/tensor/stats/metric/enumgen.go index 409eb8e9fa..cc8a8fb09d 100644 --- a/tensor/stats/metric/enumgen.go +++ b/tensor/stats/metric/enumgen.go @@ -11,11 +11,11 @@ var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} // MetricsN is the highest valid value for type Metrics, plus one. const MetricsN Metrics = 13 -var _MetricsValueMap = map[string]Metrics{`Euclidean`: 0, `SumSquares`: 1, `Abs`: 2, `Hamming`: 3, `EuclideanBinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `InnerProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} +var _MetricsValueMap = map[string]Metrics{`NormL2`: 0, `SumSquares`: 1, `NormL1`: 2, `Hamming`: 3, `NormL2BinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `DotProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} -var _MetricsDescMap = map[Metrics]string{0: `Euclidean is the square root of the sum of squares differences between tensor values, aka the L2Norm.`, 1: `SumSquares is the sum of squares differences between tensor values.`, 2: `Abs is the sum of the absolute value of differences between tensor values, aka the L1Norm.`, 3: `Hamming is the sum of 1s for every element that is different, i.e., "city block" distance.`, 4: `EuclideanBinTol is the [Euclidean] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 5: `SumSquaresBinTol is the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 6: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 7: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 8: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 9: `InnerProduct is the sum of the co-products of the tensor values.`, 10: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 11: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 12: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [InnerProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} +var _MetricsDescMap = map[Metrics]string{0: `NormL2 is the square root of the sum of squares differences between tensor values, aka the L2 Norm.`, 1: `SumSquares is the sum of squares differences between tensor values.`, 2: `NormL1 is the sum of the absolute value of differences between tensor values, the L1 Norm.`, 3: `Hamming is the sum of 1s for every element that is different, i.e., "city block" distance.`, 4: `NormL2BinTol is the [NormL2] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 5: `SumSquaresBinTol is the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 6: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 7: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 8: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 9: `DotProduct is the sum of the co-products of the tensor values.`, 10: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 11: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 12: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [DotProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} -var _MetricsMap = map[Metrics]string{0: `Euclidean`, 1: `SumSquares`, 2: `Abs`, 3: `Hamming`, 4: `EuclideanBinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `InnerProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} +var _MetricsMap = map[Metrics]string{0: `NormL2`, 1: `SumSquares`, 2: `NormL1`, 3: `Hamming`, 4: `NormL2BinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `DotProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} // String returns the string representation of this Metrics value. func (i Metrics) String() string { return enums.String(i, _MetricsMap) } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index d0158cfa0a..aa122aede9 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -84,9 +84,9 @@ func SumSquares(a, b tensor.Tensor) tensor.Values { return out } -// EuclideanOut computes the Euclidean square root of the sum of squares -// differences between tensor values, aka the L2 Norm. -func EuclideanOut(a, b tensor.Tensor, out tensor.Values) error { +// NormL2Out computes the L2 Norm: square root of the sum of squares +// differences between tensor values, aka the Euclidean distance. +func NormL2Out(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresScaleOut64(a, b, out) if err != nil { return err @@ -107,16 +107,17 @@ func EuclideanOut(a, b tensor.Tensor, out tensor.Values) error { return nil } -// Euclidean computes the sum of squares differences between tensor values, +// NormL2 computes the L2 Norm: square root of the sum of squares +// differences between tensor values, aka the Euclidean distance. // See [MetricFunc] for general information. -func Euclidean(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(EuclideanOut, a, b) +func NormL2(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(NormL2Out, a, b) } -// AbsOut computes the sum of the absolute value of differences between the -// tensor values, aka the L1 Norm. +// NormL1Out computes the sum of the absolute value of differences between the +// tensor values, the L1 Norm. // See [MetricOutFunc] for general information. -func AbsOut(a, b tensor.Tensor, out tensor.Values) error { +func NormL1Out(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -128,10 +129,11 @@ func AbsOut(a, b tensor.Tensor, out tensor.Values) error { return err } -// Abs computes the sum of squares differences between tensor values, +// NormL1 computes the sum of the absolute value of differences between the +// tensor values, the L1 Norm. // See [MetricFunc] for general information. -func Abs(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(AbsOut, a, b) +func NormL1(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(NormL1Out, a, b) } // HammingOut computes the sum of 1s for every element that is different, @@ -152,7 +154,8 @@ func HammingOut(a, b tensor.Tensor, out tensor.Values) error { return err } -// Hamming computes the sum of squares differences between tensor values, +// Hamming computes the sum of 1s for every element that is different, +// i.e., "city block" distance. // See [MetricFunc] for general information. func Hamming(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(HammingOut, a, b) @@ -177,10 +180,10 @@ func SumSquaresBinTolScaleOut64(a, b tensor.Tensor, out tensor.Values) (scale64, return } -// EuclideanBinTolOut computes the Euclidean square root of the sum of squares -// differences between tensor values, with binary tolerance: +// NormL2BinTolOut computes the L2 Norm square root of the sum of squares +// differences between tensor values (aka Euclidean distance), with binary tolerance: // differences < 0.5 are thresholded to 0. -func EuclideanBinTolOut(a, b tensor.Tensor, out tensor.Values) error { +func NormL2BinTolOut(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) if err != nil { return err @@ -201,10 +204,12 @@ func EuclideanBinTolOut(a, b tensor.Tensor, out tensor.Values) error { return nil } -// EuclideanBinTol computes the sum of squares differences between tensor values, +// NormL2BinTol computes the L2 Norm square root of the sum of squares +// differences between tensor values (aka Euclidean distance), with binary tolerance: +// differences < 0.5 are thresholded to 0. // See [MetricFunc] for general information. -func EuclideanBinTol(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(EuclideanBinTolOut, a, b) +func NormL2BinTol(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(NormL2BinTolOut, a, b) } // SumSquaresBinTolOut computes the sum of squares differences between tensor values, @@ -231,6 +236,7 @@ func SumSquaresBinTolOut(a, b tensor.Tensor, out tensor.Values) error { } // SumSquaresBinTol computes the sum of squares differences between tensor values, +// with binary tolerance: differences < 0.5 are thresholded to 0. // See [MetricFunc] for general information. func SumSquaresBinTol(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(SumSquaresBinTolOut, a, b) @@ -263,15 +269,21 @@ func CrossEntropyOut(a, b tensor.Tensor, out tensor.Values) error { return err } -// CrossEntropy computes the sum of squares differences between tensor values, +// CrossEntropy is a standard measure of the difference between two +// probabilty distributions, reflecting the additional entropy (uncertainty) associated +// with measuring probabilities under distribution b when in fact they come from +// distribution a. It is also the entropy of a plus the divergence between a from b, +// using Kullback-Leibler (KL) divergence. It is computed as: +// a * log(a/b) + (1-a) * log(1-a/1-b). // See [MetricFunc] for general information. func CrossEntropy(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(CrossEntropyOut, a, b) } -// InnerProductOut computes the sum of the co-products of the two on-NaN tensor values. +// DotProductOut computes the sum of the element-wise products of the +// two tensors (aka the inner product). // See [MetricOutFunc] for general information. -func InnerProductOut(a, b tensor.Tensor, out tensor.Values) error { +func DotProductOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -283,10 +295,11 @@ func InnerProductOut(a, b tensor.Tensor, out tensor.Values) error { return err } -// InnerProduct computes the sum of squares differences between tensor values, +// DotProductOut computes the sum of the element-wise products of the +// two tensors (aka the inner product). // See [MetricFunc] for general information. -func InnerProduct(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(InnerProductOut, a, b) +func DotProduct(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(DotProductOut, a, b) } // CovarianceOut computes the co-variance between two vectors, @@ -318,7 +331,9 @@ func CovarianceOut(a, b tensor.Tensor, out tensor.Values) error { return nil } -// Covariance computes the sum of squares differences between tensor values, +// Covariance computes the co-variance between two vectors, +// i.e., the mean of the co-product of each vector element minus +// the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))]. // See [MetricFunc] for general information. func Covariance(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(CovarianceOut, a, b) @@ -371,14 +386,19 @@ func CorrelationOut64(a, b tensor.Tensor, out tensor.Values) (tensor.Tensor, err // in range (-1..1) as the mean of the co-product of each vector // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). -// (i.e., the standardized [CovarianceFunc]). -// Equivalent to the [CosineFunc] of mean-normalized vectors. +// (i.e., the standardized [Covariance]). +// Equivalent to the [Cosine] of mean-normalized vectors. func CorrelationOut(a, b tensor.Tensor, out tensor.Values) error { _, err := CorrelationOut64(a, b, out) return err } -// Correlation computes the sum of squares differences between tensor values, +// Correlation computes the correlation between two vectors, +// in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized [Covariance]). +// Equivalent to the [Cosine] of mean-normalized vectors. // See [MetricFunc] for general information. func Correlation(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(CorrelationOut, a, b) @@ -389,7 +409,7 @@ func Correlation(a, b tensor.Tensor) tensor.Values { // element minus the mean of that vector, normalized by the product of their // standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). // (i.e., the standardized covariance). -// Equivalent to the [CosineFunc] of mean-normalized vectors. +// Equivalent to the [Cosine] of mean-normalized vectors. // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. func InvCorrelationOut(a, b tensor.Tensor, out tensor.Values) error { @@ -405,15 +425,22 @@ func InvCorrelationOut(a, b tensor.Tensor, out tensor.Values) error { return nil } -// InvCorrelation computes the sum of squares differences between tensor values, +// InvCorrelation computes 1 minus the correlation between two vectors, +// in range (-1..1) as the mean of the co-product of each vector +// element minus the mean of that vector, normalized by the product of their +// standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). +// (i.e., the standardized covariance). +// Equivalent to the [Cosine] of mean-normalized vectors. +// This is useful for a difference measure instead of similarity, +// where more different vectors have larger metric values. // See [MetricFunc] for general information. func InvCorrelation(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(InvCorrelationOut, a, b) } // CosineOut64 computes the high-dimensional angle between two vectors, -// in range (-1..1) as the normalized [InnerProductFunc]: -// inner product / sqrt(ssA * ssB). See also [CorrelationFunc]. +// in range (-1..1) as the normalized [Dot]: +// dot product / sqrt(ssA * ssB). See also [Correlation]. func CosineOut64(a, b tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { ss64, avar64, bvar64, err := Vectorize3Out64(NFunc, func(idx int, tsr ...tensor.Tensor) { Vec3outFunc(idx, tsr[0], tsr[1], tsr[2], tsr[3], tsr[4], 0, func(a, b, ss, avar, bvar float64) (float64, float64, float64) { @@ -440,22 +467,24 @@ func CosineOut64(a, b tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { } // CosineOut computes the high-dimensional angle between two vectors, -// in range (-1..1) as the normalized inner product: -// inner product / sqrt(ssA * ssB). See also [CorrelationFunc] +// in range (-1..1) as the normalized dot product: +// dot product / sqrt(ssA * ssB). See also [Correlation] func CosineOut(a, b tensor.Tensor, out tensor.Values) error { _, err := CosineOut64(a, b, out) return err } -// Cosine computes the sum of squares differences between tensor values, +// Cosine computes the high-dimensional angle between two vectors, +// in range (-1..1) as the normalized dot product: +// dot product / sqrt(ssA * ssB). See also [Correlation] // See [MetricFunc] for general information. func Cosine(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(CosineOut, a, b) } // InvCosineOut computes 1 minus the cosine between two vectors, -// in range (-1..1) as the normalized inner product: -// inner product / sqrt(ssA * ssB). +// in range (-1..1) as the normalized dot product: +// dot product / sqrt(ssA * ssB). // This is useful for a difference measure instead of similarity, // where more different vectors have larger metric values. func InvCosineOut(a, b tensor.Tensor, out tensor.Values) error { @@ -471,7 +500,11 @@ func InvCosineOut(a, b tensor.Tensor, out tensor.Values) error { return nil } -// InvCosine computes the sum of squares differences between tensor values, +// InvCosine computes 1 minus the cosine between two vectors, +// in range (-1..1) as the normalized dot product: +// dot product / sqrt(ssA * ssB). +// This is useful for a difference measure instead of similarity, +// where more different vectors have larger metric values. // See [MetricFunc] for general information. func InvCosine(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(InvCosineOut, a, b) diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index f3d0c23dea..9f7f7228f2 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -25,17 +25,17 @@ func TestFuncs(t *testing.T) { btsr := tensor.NewNumberFromValues(b64...) out := tensor.NewFloat64(1) - EuclideanOut(atsr, btsr, out) - assert.InDelta(t, results[MetricEuclidean], out.Values[0], tol) + NormL2Out(atsr, btsr, out) + assert.InDelta(t, results[MetricNormL2], out.Values[0], tol) SumSquaresOut(atsr, btsr, out) assert.InDelta(t, results[MetricSumSquares], out.Values[0], tol) - EuclideanBinTolOut(atsr, btsr, out) - assert.InDelta(t, results[MetricEuclideanBinTol], out.Values[0], tol) + NormL2BinTolOut(atsr, btsr, out) + assert.InDelta(t, results[MetricNormL2BinTol], out.Values[0], tol) - AbsOut(atsr, btsr, out) - assert.InDelta(t, results[MetricAbs], out.Values[0], tol) + NormL1Out(atsr, btsr, out) + assert.InDelta(t, results[MetricNormL1], out.Values[0], tol) HammingOut(atsr, btsr, out) assert.Equal(t, results[MetricHamming], out.Values[0]) @@ -55,8 +55,8 @@ func TestFuncs(t *testing.T) { CrossEntropyOut(atsr, btsr, out) assert.InDelta(t, results[MetricCrossEntropy], out.Values[0], tol) - InnerProductOut(atsr, btsr, out) - assert.InDelta(t, results[MetricInnerProduct], out.Values[0], tol) + DotProductOut(atsr, btsr, out) + assert.InDelta(t, results[MetricDotProduct], out.Values[0], tol) CosineOut(atsr, btsr, out) assert.InDelta(t, results[MetricCosine], out.Values[0], tol) @@ -64,7 +64,7 @@ func TestFuncs(t *testing.T) { InvCosineOut(atsr, btsr, out) assert.InDelta(t, results[MetricInvCosine], out.Values[0], tol) - for met := MetricEuclidean; met < MetricsN; met++ { + for met := MetricNormL2; met < MetricsN; met++ { out := met.Call(atsr, btsr) assert.InDelta(t, results[met], out.Float1D(0), tol) } @@ -79,7 +79,7 @@ func TestMatrix(t *testing.T) { assert.NoError(t, err) in := dt.Column("Input") out := tensor.NewFloat64() - err = MatrixOut(Euclidean, in, out) + err = MatrixOut(NormL2, in, out) assert.NoError(t, err) // fmt.Println(out.Tensor) for i, v := range simres { diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index c87f371577..eeb33d93c2 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -12,15 +12,15 @@ import ( ) func init() { - tensor.AddFunc(MetricEuclidean.FuncName(), Euclidean) + tensor.AddFunc(MetricNormL2.FuncName(), NormL2) tensor.AddFunc(MetricSumSquares.FuncName(), SumSquares) - tensor.AddFunc(MetricAbs.FuncName(), Abs) + tensor.AddFunc(MetricNormL1.FuncName(), NormL1) tensor.AddFunc(MetricHamming.FuncName(), Hamming) - tensor.AddFunc(MetricEuclideanBinTol.FuncName(), EuclideanBinTol) + tensor.AddFunc(MetricNormL2BinTol.FuncName(), NormL2BinTol) tensor.AddFunc(MetricSumSquaresBinTol.FuncName(), SumSquaresBinTol) tensor.AddFunc(MetricInvCosine.FuncName(), InvCosine) tensor.AddFunc(MetricInvCorrelation.FuncName(), InvCorrelation) - tensor.AddFunc(MetricInnerProduct.FuncName(), InnerProduct) + tensor.AddFunc(MetricDotProduct.FuncName(), DotProduct) tensor.AddFunc(MetricCrossEntropy.FuncName(), CrossEntropy) tensor.AddFunc(MetricCovariance.FuncName(), Covariance) tensor.AddFunc(MetricCorrelation.FuncName(), Correlation) @@ -31,25 +31,25 @@ func init() { type Metrics int32 //enums:enum -trim-prefix Metric const ( - // Euclidean is the square root of the sum of squares differences - // between tensor values, aka the L2Norm. - MetricEuclidean Metrics = iota + // NormL2 is the square root of the sum of squares differences + // between tensor values, aka the L2 Norm. + MetricNormL2 Metrics = iota // SumSquares is the sum of squares differences between tensor values. MetricSumSquares - // Abs is the sum of the absolute value of differences - // between tensor values, aka the L1Norm. - MetricAbs + // NormL1 is the sum of the absolute value of differences + // between tensor values, the L1 Norm. + MetricNormL1 // Hamming is the sum of 1s for every element that is different, // i.e., "city block" distance. MetricHamming - // EuclideanBinTol is the [Euclidean] square root of the sum of squares + // NormL2BinTol is the [NormL2] square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. - MetricEuclideanBinTol + MetricNormL2BinTol // SumSquaresBinTol is the [SumSquares] differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. @@ -74,8 +74,8 @@ const ( ///////////////////////////////////////////////////////////////////////// // Everything below here is !Increasing -- larger = closer, not farther - // InnerProduct is the sum of the co-products of the tensor values. - MetricInnerProduct + // DotProduct is the sum of the co-products of the tensor values. + MetricDotProduct // Covariance is co-variance between two vectors, // i.e., the mean of the co-product of each vector element minus @@ -90,7 +90,7 @@ const ( MetricCorrelation // Cosine is high-dimensional angle between two vectors, - // in range (-1..1) as the normalized [InnerProduct]: + // in range (-1..1) as the normalized [DotProduct]: // inner product / sqrt(ssA * ssB). See also [Correlation]. MetricCosine ) @@ -114,11 +114,11 @@ func (m Metrics) Call(a, b tensor.Tensor) tensor.Values { } // Increasing returns true if the distance metric is such that metric -// values increase as a function of distance (e.g., Euclidean) +// values increase as a function of distance (e.g., NormL2) // and false if metric values decrease as a function of distance // (e.g., Cosine, Correlation) func (m Metrics) Increasing() bool { - if m >= MetricInnerProduct { + if m >= MetricDotProduct { return false } return true diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index bcd092b1a8..dc99b8b8f6 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -28,8 +28,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Count`: count of number of elements * `Sum`: sum of elements -* `SumAbs`: sum of absolute-value-of elements (same as L1Norm) -* `L1Norm`: L1 Norm: sum of absolute values (same as SumAbs) +* `NormL1`: L1 Norm: sum of absolute values * `Prod`: product of elements * `Min`: minimum value * `Max`: maximum value @@ -40,7 +39,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Std`: sample standard deviation (sqrt of Var) * `Sem`: sample standard error of the mean (Std divided by sqrt(n)) * `SumSq`: sum of squared element values -* `L2Norm`: L2 Norm: square-root of sum-of-squares +* `NormL2`: L2 Norm: square-root of sum-of-squares * `VarPop`: population variance (squared diffs from mean, divided by n) * `StdPop`: population standard deviation (sqrt of VarPop) * `SemPop`: population standard error of the mean (StdPop divided by sqrt(n)) diff --git a/tensor/stats/stats/enumgen.go b/tensor/stats/stats/enumgen.go index 6e2cc87399..b4db1f6c7c 100644 --- a/tensor/stats/stats/enumgen.go +++ b/tensor/stats/stats/enumgen.go @@ -6,16 +6,16 @@ import ( "cogentcore.org/core/enums" ) -var _StatsValues = []Stats{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20} +var _StatsValues = []Stats{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19} // StatsN is the highest valid value for type Stats, plus one. -const StatsN Stats = 21 +const StatsN Stats = 20 -var _StatsValueMap = map[string]Stats{`Count`: 0, `Sum`: 1, `SumAbs`: 2, `L1Norm`: 3, `Prod`: 4, `Min`: 5, `Max`: 6, `MinAbs`: 7, `MaxAbs`: 8, `Mean`: 9, `Var`: 10, `Std`: 11, `Sem`: 12, `SumSq`: 13, `L2Norm`: 14, `VarPop`: 15, `StdPop`: 16, `SemPop`: 17, `Median`: 18, `Q1`: 19, `Q3`: 20} +var _StatsValueMap = map[string]Stats{`Count`: 0, `Sum`: 1, `NormL1`: 2, `Prod`: 3, `Min`: 4, `Max`: 5, `MinAbs`: 6, `MaxAbs`: 7, `Mean`: 8, `Var`: 9, `Std`: 10, `Sem`: 11, `SumSq`: 12, `NormL2`: 13, `VarPop`: 14, `StdPop`: 15, `SemPop`: 16, `Median`: 17, `Q1`: 18, `Q3`: 19} -var _StatsDescMap = map[Stats]string{0: `count of number of elements.`, 1: `sum of elements.`, 2: `sum of absolute-value-of elements (= L1Norm).`, 3: `L1 Norm: sum of absolute values (= SumAbs).`, 4: `product of elements.`, 5: `minimum value.`, 6: `maximum value.`, 7: `minimum of absolute values.`, 8: `maximum of absolute values.`, 9: `mean value = sum / count.`, 10: `sample variance (squared deviations from mean, divided by n-1).`, 11: `sample standard deviation (sqrt of Var).`, 12: `sample standard error of the mean (Std divided by sqrt(n)).`, 13: `sum of squared values.`, 14: `L2 Norm: square-root of sum-of-squares.`, 15: `population variance (squared diffs from mean, divided by n).`, 16: `population standard deviation (sqrt of VarPop).`, 17: `population standard error of the mean (StdPop divided by sqrt(n)).`, 18: `middle value in sorted ordering.`, 19: `Q1 first quartile = 25%ile value = .25 quantile value.`, 20: `Q3 third quartile = 75%ile value = .75 quantile value.`} +var _StatsDescMap = map[Stats]string{0: `count of number of elements.`, 1: `sum of elements.`, 2: `L1 Norm: sum of absolute values of elements.`, 3: `product of elements.`, 4: `minimum value.`, 5: `maximum value.`, 6: `minimum of absolute values.`, 7: `maximum of absolute values.`, 8: `mean value = sum / count.`, 9: `sample variance (squared deviations from mean, divided by n-1).`, 10: `sample standard deviation (sqrt of Var).`, 11: `sample standard error of the mean (Std divided by sqrt(n)).`, 12: `sum of squared values.`, 13: `L2 Norm: square-root of sum-of-squares.`, 14: `population variance (squared diffs from mean, divided by n).`, 15: `population standard deviation (sqrt of VarPop).`, 16: `population standard error of the mean (StdPop divided by sqrt(n)).`, 17: `middle value in sorted ordering.`, 18: `Q1 first quartile = 25%ile value = .25 quantile value.`, 19: `Q3 third quartile = 75%ile value = .75 quantile value.`} -var _StatsMap = map[Stats]string{0: `Count`, 1: `Sum`, 2: `SumAbs`, 3: `L1Norm`, 4: `Prod`, 5: `Min`, 6: `Max`, 7: `MinAbs`, 8: `MaxAbs`, 9: `Mean`, 10: `Var`, 11: `Std`, 12: `Sem`, 13: `SumSq`, 14: `L2Norm`, 15: `VarPop`, 16: `StdPop`, 17: `SemPop`, 18: `Median`, 19: `Q1`, 20: `Q3`} +var _StatsMap = map[Stats]string{0: `Count`, 1: `Sum`, 2: `NormL1`, 3: `Prod`, 4: `Min`, 5: `Max`, 6: `MinAbs`, 7: `MaxAbs`, 8: `Mean`, 9: `Var`, 10: `Std`, 11: `Sem`, 12: `SumSq`, 13: `NormL2`, 14: `VarPop`, 15: `StdPop`, 16: `SemPop`, 17: `Median`, 18: `Q1`, 19: `Q3`} // String returns the string representation of this Stats value. func (i Stats) String() string { return enums.String(i, _StatsMap) } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 1bc5134d2d..6da443f43c 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -82,17 +82,17 @@ func Sum(in tensor.Tensor) tensor.Values { return out } -// SumAbs computes the sum of absolute-value-of tensor values. +// NormL1 computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. -func SumAbs(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(SumAbsOut, in) +func NormL1(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(NormL1Out, in) } -// SumAbsOut computes the sum of absolute-value-of tensor values. +// NormL1Out computes the sum of absolute-value-of tensor values. // This is also known as the L1 norm. // See [StatsFunc] for general information. -func SumAbsOut(in tensor.Tensor, out tensor.Values) error { +func NormL1Out(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) @@ -485,10 +485,10 @@ func SumSqOut(in tensor.Tensor, out tensor.Values) error { return err } -// L2NormOut64 computes the square root of the sum of squares of tensor values, +// NormL2Out64 computes the square root of the sum of squares of tensor values, // known as the L2 norm, and returns the Float64 output values for // use in subsequent computations. -func L2NormOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { +func NormL2Out64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { scale64, ss64, err := SumSqScaleOut64(in, out) if err != nil { return nil, err @@ -509,19 +509,19 @@ func L2NormOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { return scale64, nil } -// L2Norm computes the square root of the sum of squares of tensor values, +// NormL2 computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsFunc] for general information. -func L2Norm(in tensor.Tensor) tensor.Values { +func NormL2(in tensor.Tensor) tensor.Values { out := tensor.NewOfType(in.DataType()) - errors.Log1(L2NormOut64(in, out)) + errors.Log1(NormL2Out64(in, out)) return out } -// L2NormOut computes the square root of the sum of squares of tensor values, +// NormL2Out computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsOutFunc] for general information. -func L2NormOut(in tensor.Tensor, out tensor.Values) error { - _, err := L2NormOut64(in, out) +func NormL2Out(in tensor.Tensor, out tensor.Values) error { + _, err := NormL2Out64(in, out) return err } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 6994bb1326..9e0e1c8933 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -16,8 +16,7 @@ import ( func init() { tensor.AddFunc(StatCount.FuncName(), Count) tensor.AddFunc(StatSum.FuncName(), Sum) - tensor.AddFunc(StatSumAbs.FuncName(), SumAbs) - tensor.AddFunc(StatL1Norm.FuncName(), SumAbs) + tensor.AddFunc(StatNormL1.FuncName(), NormL1) tensor.AddFunc(StatProd.FuncName(), Prod) tensor.AddFunc(StatMin.FuncName(), Min) tensor.AddFunc(StatMax.FuncName(), Max) @@ -28,7 +27,7 @@ func init() { tensor.AddFunc(StatStd.FuncName(), Std) tensor.AddFunc(StatSem.FuncName(), Sem) tensor.AddFunc(StatSumSq.FuncName(), SumSq) - tensor.AddFunc(StatL2Norm.FuncName(), L2Norm) + tensor.AddFunc(StatNormL2.FuncName(), NormL2) tensor.AddFunc(StatVarPop.FuncName(), VarPop) tensor.AddFunc(StatStdPop.FuncName(), StdPop) tensor.AddFunc(StatSemPop.FuncName(), SemPop) @@ -48,11 +47,8 @@ const ( // sum of elements. StatSum - // sum of absolute-value-of elements (= L1Norm). - StatSumAbs - - // L1 Norm: sum of absolute values (= SumAbs). - StatL1Norm + // L1 Norm: sum of absolute values of elements. + StatNormL1 // product of elements. StatProd @@ -85,7 +81,7 @@ const ( StatSumSq // L2 Norm: square-root of sum-of-squares. - StatL2Norm + StatNormL2 // population variance (squared diffs from mean, divided by n). StatVarPop diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index 8c16304d48..7d6a3fb86d 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -17,7 +17,7 @@ func TestFuncs64(t *testing.T) { ix := tensor.NewNumberFromValues(vals...) out := tensor.NewFloat64(1) - results := []float64{11, 5.5, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} + results := []float64{11, 5.5, 5.5, 0, 0, 1, 0, 1, 0.5, 0.11, math.Sqrt(0.11), math.Sqrt(0.11) / math.Sqrt(11), 3.85, math.Sqrt(3.85), 0.1, math.Sqrt(0.1), math.Sqrt(0.1) / math.Sqrt(11), 0.5, 0.25, 0.75} tol := 1.0e-8 @@ -27,8 +27,8 @@ func TestFuncs64(t *testing.T) { SumOut(ix, out) assert.Equal(t, results[StatSum], out.Values[0]) - SumAbsOut(ix, out) - assert.Equal(t, results[StatSumAbs], out.Values[0]) + NormL1Out(ix, out) + assert.Equal(t, results[StatNormL1], out.Values[0]) ProdOut(ix, out) assert.Equal(t, results[StatProd], out.Values[0]) @@ -69,8 +69,8 @@ func TestFuncs64(t *testing.T) { SumSqOut(ix, out) assert.InDelta(t, results[StatSumSq], out.Values[0], tol) - L2NormOut(ix, out) - assert.InDelta(t, results[StatL2Norm], out.Values[0], tol) + NormL2Out(ix, out) + assert.InDelta(t, results[StatNormL2], out.Values[0], tol) MedianOut(ix, out) assert.InDelta(t, results[StatMedian], out.Values[0], tol) @@ -93,7 +93,7 @@ func TestFuncsInt(t *testing.T) { ix := tensor.NewRows(tsr) out := tensor.NewInt(1) - results := []int{11, 550, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} + results := []int{11, 550, 550, 0, 0, 100, 0, 100, 50, 1100, int(math.Sqrt(1100)), int(math.Sqrt(1100) / math.Sqrt(11)), 38500, 196, 1000, int(math.Sqrt(1000)), int(math.Sqrt(1000) / math.Sqrt(11))} CountOut(ix, out) assert.Equal(t, results[StatCount], out.Values[0]) @@ -101,8 +101,8 @@ func TestFuncsInt(t *testing.T) { SumOut(ix, out) assert.Equal(t, results[StatSum], out.Values[0]) - SumAbsOut(ix, out) - assert.Equal(t, results[StatSumAbs], out.Values[0]) + NormL1Out(ix, out) + assert.Equal(t, results[StatNormL1], out.Values[0]) ProdOut(ix, out) assert.Equal(t, results[StatProd], out.Values[0]) @@ -143,8 +143,8 @@ func TestFuncsInt(t *testing.T) { SumSqOut(ix, out) assert.Equal(t, results[StatSumSq], out.Values[0]) - L2NormOut(ix, out) - assert.Equal(t, results[StatL2Norm], out.Values[0]) + NormL2Out(ix, out) + assert.Equal(t, results[StatNormL2], out.Values[0]) for stat := StatCount; stat <= StatSemPop; stat++ { out := stat.Call(ix) diff --git a/tensor/vector/vector.go b/tensor/vector/vector.go index 85f8bfbc85..431d0a4272 100644 --- a/tensor/vector/vector.go +++ b/tensor/vector/vector.go @@ -43,7 +43,7 @@ func Dot(a, b tensor.Tensor) tensor.Values { return Sum(Mul(a, b)) } -// NormL2 returns the length of the vector as the L2Norm: +// NormL2 returns the length of the vector as the L2 Norm: // square root of the sum of squared values of the vector, as a scalar. // This is the Sqrt of the [Dot] product of the vector with itself. func NormL2(a tensor.Tensor) tensor.Values { @@ -51,7 +51,7 @@ func NormL2(a tensor.Tensor) tensor.Values { return tensor.NewFloat64Scalar(math.Sqrt(dot)) } -// NormL1 returns the length of the vector as the L1Norm: +// NormL1 returns the length of the vector as the L1 Norm: // sum of the absolute values of the tensor, as a scalar. func NormL1(a tensor.Tensor) tensor.Values { n := a.Len() diff --git a/tensor/vector/vector_test.go b/tensor/vector/vector_test.go index bb19e89148..69e26e5979 100644 --- a/tensor/vector/vector_test.go +++ b/tensor/vector/vector_test.go @@ -14,7 +14,7 @@ import ( func TestVector(t *testing.T) { v := tensor.NewFloat64FromValues(1, 2, 3) - ip := Inner(v, v).(*tensor.Float64) + ip := Mul(v, v).(*tensor.Float64) assert.Equal(t, []float64{1, 4, 9}, ip.Values) smv := Sum(ip).(*tensor.Float64) diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go index 9d2a09cad6..3c02914d98 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -10,8 +10,6 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/stats/metric/metric"] = map[string]reflect.Value{ // function, constant and variable definitions - "Abs": reflect.ValueOf(metric.Abs), - "AbsOut": reflect.ValueOf(metric.AbsOut), "AsMetricFunc": reflect.ValueOf(metric.AsMetricFunc), "AsMetricOutFunc": reflect.ValueOf(metric.AsMetricOutFunc), "ClosestRow": reflect.ValueOf(metric.ClosestRow), @@ -29,36 +27,38 @@ func init() { "CrossEntropyOut": reflect.ValueOf(metric.CrossEntropyOut), "CrossMatrix": reflect.ValueOf(metric.CrossMatrix), "CrossMatrixOut": reflect.ValueOf(metric.CrossMatrixOut), - "Euclidean": reflect.ValueOf(metric.Euclidean), - "EuclideanBinTol": reflect.ValueOf(metric.EuclideanBinTol), - "EuclideanBinTolOut": reflect.ValueOf(metric.EuclideanBinTolOut), - "EuclideanOut": reflect.ValueOf(metric.EuclideanOut), + "DotProduct": reflect.ValueOf(metric.DotProduct), + "DotProductOut": reflect.ValueOf(metric.DotProductOut), "Hamming": reflect.ValueOf(metric.Hamming), "HammingOut": reflect.ValueOf(metric.HammingOut), - "InnerProduct": reflect.ValueOf(metric.InnerProduct), - "InnerProductOut": reflect.ValueOf(metric.InnerProductOut), "InvCorrelation": reflect.ValueOf(metric.InvCorrelation), "InvCorrelationOut": reflect.ValueOf(metric.InvCorrelationOut), "InvCosine": reflect.ValueOf(metric.InvCosine), "InvCosineOut": reflect.ValueOf(metric.InvCosineOut), "Matrix": reflect.ValueOf(metric.Matrix), "MatrixOut": reflect.ValueOf(metric.MatrixOut), - "MetricAbs": reflect.ValueOf(metric.MetricAbs), "MetricCorrelation": reflect.ValueOf(metric.MetricCorrelation), "MetricCosine": reflect.ValueOf(metric.MetricCosine), "MetricCovariance": reflect.ValueOf(metric.MetricCovariance), "MetricCrossEntropy": reflect.ValueOf(metric.MetricCrossEntropy), - "MetricEuclidean": reflect.ValueOf(metric.MetricEuclidean), - "MetricEuclideanBinTol": reflect.ValueOf(metric.MetricEuclideanBinTol), + "MetricDotProduct": reflect.ValueOf(metric.MetricDotProduct), "MetricHamming": reflect.ValueOf(metric.MetricHamming), - "MetricInnerProduct": reflect.ValueOf(metric.MetricInnerProduct), "MetricInvCorrelation": reflect.ValueOf(metric.MetricInvCorrelation), "MetricInvCosine": reflect.ValueOf(metric.MetricInvCosine), + "MetricNormL1": reflect.ValueOf(metric.MetricNormL1), + "MetricNormL2": reflect.ValueOf(metric.MetricNormL2), + "MetricNormL2BinTol": reflect.ValueOf(metric.MetricNormL2BinTol), "MetricSumSquares": reflect.ValueOf(metric.MetricSumSquares), "MetricSumSquaresBinTol": reflect.ValueOf(metric.MetricSumSquaresBinTol), "MetricsN": reflect.ValueOf(metric.MetricsN), "MetricsValues": reflect.ValueOf(metric.MetricsValues), "NFunc": reflect.ValueOf(metric.NFunc), + "NormL1": reflect.ValueOf(metric.NormL1), + "NormL1Out": reflect.ValueOf(metric.NormL1Out), + "NormL2": reflect.ValueOf(metric.NormL2), + "NormL2BinTol": reflect.ValueOf(metric.NormL2BinTol), + "NormL2BinTolOut": reflect.ValueOf(metric.NormL2BinTolOut), + "NormL2Out": reflect.ValueOf(metric.NormL2Out), "PCA": reflect.ValueOf(metric.PCA), "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), "ProjectOnMatrixColumnOut": reflect.ValueOf(metric.ProjectOnMatrixColumnOut), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index ac20b76d89..6b8437582c 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -26,9 +26,6 @@ func init() { "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), "GroupStats": reflect.ValueOf(stats.GroupStats), "Groups": reflect.ValueOf(stats.Groups), - "L2Norm": reflect.ValueOf(stats.L2Norm), - "L2NormOut": reflect.ValueOf(stats.L2NormOut), - "L2NormOut64": reflect.ValueOf(stats.L2NormOut64), "Max": reflect.ValueOf(stats.Max), "MaxAbs": reflect.ValueOf(stats.MaxAbs), "MaxAbsOut": reflect.ValueOf(stats.MaxAbsOut), @@ -43,6 +40,11 @@ func init() { "MinAbsOut": reflect.ValueOf(stats.MinAbsOut), "MinOut": reflect.ValueOf(stats.MinOut), "NFunc": reflect.ValueOf(stats.NFunc), + "NormL1": reflect.ValueOf(stats.NormL1), + "NormL1Out": reflect.ValueOf(stats.NormL1Out), + "NormL2": reflect.ValueOf(stats.NormL2), + "NormL2Out": reflect.ValueOf(stats.NormL2Out), + "NormL2Out64": reflect.ValueOf(stats.NormL2Out64), "Prod": reflect.ValueOf(stats.Prod), "ProdOut": reflect.ValueOf(stats.ProdOut), "Q1": reflect.ValueOf(stats.Q1), @@ -56,14 +58,14 @@ func init() { "SemPop": reflect.ValueOf(stats.SemPop), "SemPopOut": reflect.ValueOf(stats.SemPopOut), "StatCount": reflect.ValueOf(stats.StatCount), - "StatL1Norm": reflect.ValueOf(stats.StatL1Norm), - "StatL2Norm": reflect.ValueOf(stats.StatL2Norm), "StatMax": reflect.ValueOf(stats.StatMax), "StatMaxAbs": reflect.ValueOf(stats.StatMaxAbs), "StatMean": reflect.ValueOf(stats.StatMean), "StatMedian": reflect.ValueOf(stats.StatMedian), "StatMin": reflect.ValueOf(stats.StatMin), "StatMinAbs": reflect.ValueOf(stats.StatMinAbs), + "StatNormL1": reflect.ValueOf(stats.StatNormL1), + "StatNormL2": reflect.ValueOf(stats.StatNormL2), "StatProd": reflect.ValueOf(stats.StatProd), "StatQ1": reflect.ValueOf(stats.StatQ1), "StatQ3": reflect.ValueOf(stats.StatQ3), @@ -72,7 +74,6 @@ func init() { "StatStd": reflect.ValueOf(stats.StatStd), "StatStdPop": reflect.ValueOf(stats.StatStdPop), "StatSum": reflect.ValueOf(stats.StatSum), - "StatSumAbs": reflect.ValueOf(stats.StatSumAbs), "StatSumSq": reflect.ValueOf(stats.StatSumSq), "StatVar": reflect.ValueOf(stats.StatVar), "StatVarPop": reflect.ValueOf(stats.StatVarPop), @@ -85,8 +86,6 @@ func init() { "StdPopOut": reflect.ValueOf(stats.StdPopOut), "StripPackage": reflect.ValueOf(stats.StripPackage), "Sum": reflect.ValueOf(stats.Sum), - "SumAbs": reflect.ValueOf(stats.SumAbs), - "SumAbsOut": reflect.ValueOf(stats.SumAbsOut), "SumOut": reflect.ValueOf(stats.SumOut), "SumOut64": reflect.ValueOf(stats.SumOut64), "SumSq": reflect.ValueOf(stats.SumSq), From 45e35a27a35fddd044b70a614988c15140463e54 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 30 Sep 2024 15:10:10 -0700 Subject: [PATCH 173/311] Actually L1Norm and L2Norm are clearer -- use those throughout. --- goal/README.md | 2 +- tensor/README.md | 2 +- tensor/stats/cluster/clust_test.go | 2 +- tensor/stats/metric/README.md | 10 +++---- tensor/stats/metric/enumgen.go | 6 ++-- tensor/stats/metric/funcs.go | 30 +++++++++---------- tensor/stats/metric/metric_test.go | 16 +++++----- tensor/stats/metric/metrics.go | 20 ++++++------- tensor/stats/stats/README.md | 6 ++-- tensor/stats/stats/enumgen.go | 4 +-- tensor/stats/stats/funcs.go | 28 ++++++++--------- tensor/stats/stats/stats.go | 8 ++--- tensor/stats/stats/stats_test.go | 16 +++++----- tensor/vector/vector.go | 10 +++---- tensor/vector/vector_test.go | 4 +-- ...cogentcore_org-core-tensor-stats-metric.go | 18 +++++------ .../cogentcore_org-core-tensor-stats-stats.go | 14 ++++----- .../cogentcore_org-core-tensor-vector.go | 4 +-- 18 files changed, 99 insertions(+), 101 deletions(-) diff --git a/goal/README.md b/goal/README.md index 2db95fb460..e586642a47 100644 --- a/goal/README.md +++ b/goal/README.md @@ -390,7 +390,7 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | | |`a.max(0)` | `max(a)` | maximum element of each column of tensor `a` | | | |`a.max(1)` | `max(a,[],2)` | maximum element of each row of tensor `a` | | | |`np.maximum(a, b)` | `max(a,b)` | compares a and b element-wise, and returns the maximum value from each pair | -| `stats.NormL2(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | +| `stats.L2Norm(a)` | `np.sqrt(v @ v)` or `np.linalg.norm(v)` | `norm(v)` | L2 norm of vector v | | | |`cg` | `conjgrad` | conjugate gradients solver | ## FFT and complex numbers diff --git a/tensor/README.md b/tensor/README.md index f61593a8cc..30ea46945d 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -10,7 +10,7 @@ In addition, there are five important "view" implementations of `Tensor` that wr * `Sliced` provides a sub-sliced view into the wrapped `Tensor` source, using an indexed list along each dimension. Thus, it can provide a reordered and filtered view onto the raw data, and it has a well-defined shape in terms of the number of indexes per dimension. This corresponds to the NumPy basic sliced indexing model. -* `Masked` provides a `Bool` masked view onto each element in the wrapped `Tensor`, where the two maintain the same shape). Any cell with a `false` value in the bool mask returns a `NaN` (missing data), and `Set` functions are no-ops, such that the tensor functions automatically only process the mask-filtered data. +* `Masked` provides a `Bool` masked view onto each element in the wrapped `Tensor`, where the two maintain the same shape). Any cell with a `false` value in the bool mask returns a `NaN` (missing data), and `Set` functions are no-ops. The [stats](stats) packages treat `NaN` as missing data, but [tmath](tmath), [vector](vector), and [matrix](matrix) packages do not, so it is best to call `.AsValues()` on masked data prior to operating on it, in a basic math context (i.e., `copy` in Goal). * `Indexed` has a tensor of indexes into the source data, where the final, innermost dimension of the indexes is the same size as the number of dimensions in the wrapped source tensor. The overall shape of this view is that of the remaining outer dimensions of the Indexes tensor, and like other views, assignment and return values are taken from the corresponding indexed value in the wrapped source tensor. diff --git a/tensor/stats/cluster/clust_test.go b/tensor/stats/cluster/clust_test.go index d6ff053a98..bdfc4df69b 100644 --- a/tensor/stats/cluster/clust_test.go +++ b/tensor/stats/cluster/clust_test.go @@ -34,7 +34,7 @@ func TestClust(t *testing.T) { t.Error(err) } in := dt.Column("Input") - out := metric.Matrix(metric.NormL2, in) + out := metric.Matrix(metric.L2Norm, in) cl := Cluster(Avg.String(), out, dt.Column("Name")) diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 728a880eeb..7b9a80d09f 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -9,7 +9,7 @@ The metric functions always operate on the outermost _row_ dimension, and it is * To obtain a single summary metric across all values, use `tensor.As1D`. -* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the metric for each such cell computed across the outer rows dimension. For the `NormL2` metric for example, each cell has the difference for that cell value across all the rows between the two tensors. See [Matrix functions](#matrix-functions) below for a function that computes the distances _between each cell pattern and all the others_, as a distance or similarity matrix. +* For `RowMajor` data that is naturally organized as a single outer _rows_ dimension with the remaining inner dimensions comprising the _cells_, the results are the metric for each such cell computed across the outer rows dimension. For the `L2Norm` metric for example, each cell has the difference for that cell value across all the rows between the two tensors. See [Matrix functions](#matrix-functions) below for a function that computes the distances _between each cell pattern and all the others_, as a distance or similarity matrix. * Use `tensor.NewRowCellsView` to reshape any tensor into a 2D rows x cells shape, with the cells starting at a given dimension. Thus, any number of outer dimensions can be collapsed into the outer row dimension, and the remaining dimensions become the cells. @@ -17,11 +17,11 @@ The metric functions always operate on the outermost _row_ dimension, and it is ### Value _increases_ with increasing distance (i.e., difference metric) -* `NormL2`: the square root of the sum of squares differences between tensor values. +* `L2Norm`: the square root of the sum of squares differences between tensor values. * `SumSquares`: the sum of squares differences between tensor values. -* `Abs`or `NormL2`: the sum of the absolute value of differences between tensor values. +* `Abs`or `L2Norm`: the sum of the absolute value of differences between tensor values. * `Hamming`: the sum of 1s for every element that is different, i.e., "city block" distance. -* `NormL2BinTol`: the `NormL2` square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. +* `L2NormBinTol`: the `L2Norm` square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. * `SumSquaresBinTol`: the `SumSquares` differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0. * `InvCosine`: is 1-`Cosine`, which is useful to convert it to an Increasing metric where more different vectors have larger metric values. * `InvCorrelation`: is 1-`Correlation`, which is useful to convert it to an Increasing metric where more different vectors have larger metric values. @@ -38,7 +38,7 @@ Here is general info about these functions: The output must be a `tensor.Values` tensor, and it is automatically shaped to hold the stat value(s) for the "cells" in higher-dimensional tensors, and a single scalar value for a 1D input tensor. -All metric functions skip over NaN's, as a missing value. +All metric functions skip over `NaN`'s, as a missing value. Metric functions cannot be computed in parallel, e.g., using VectorizeThreaded or GPU, due to shared writing to the same output values. Special implementations are required if that is needed. diff --git a/tensor/stats/metric/enumgen.go b/tensor/stats/metric/enumgen.go index cc8a8fb09d..c9802a2353 100644 --- a/tensor/stats/metric/enumgen.go +++ b/tensor/stats/metric/enumgen.go @@ -11,11 +11,11 @@ var _MetricsValues = []Metrics{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12} // MetricsN is the highest valid value for type Metrics, plus one. const MetricsN Metrics = 13 -var _MetricsValueMap = map[string]Metrics{`NormL2`: 0, `SumSquares`: 1, `NormL1`: 2, `Hamming`: 3, `NormL2BinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `DotProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} +var _MetricsValueMap = map[string]Metrics{`L2Norm`: 0, `SumSquares`: 1, `L1Norm`: 2, `Hamming`: 3, `L2NormBinTol`: 4, `SumSquaresBinTol`: 5, `InvCosine`: 6, `InvCorrelation`: 7, `CrossEntropy`: 8, `DotProduct`: 9, `Covariance`: 10, `Correlation`: 11, `Cosine`: 12} -var _MetricsDescMap = map[Metrics]string{0: `NormL2 is the square root of the sum of squares differences between tensor values, aka the L2 Norm.`, 1: `SumSquares is the sum of squares differences between tensor values.`, 2: `NormL1 is the sum of the absolute value of differences between tensor values, the L1 Norm.`, 3: `Hamming is the sum of 1s for every element that is different, i.e., "city block" distance.`, 4: `NormL2BinTol is the [NormL2] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 5: `SumSquaresBinTol is the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 6: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 7: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 8: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 9: `DotProduct is the sum of the co-products of the tensor values.`, 10: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 11: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 12: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [DotProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} +var _MetricsDescMap = map[Metrics]string{0: `L2Norm is the square root of the sum of squares differences between tensor values, aka the L2 Norm.`, 1: `SumSquares is the sum of squares differences between tensor values.`, 2: `L1Norm is the sum of the absolute value of differences between tensor values, the L1 Norm.`, 3: `Hamming is the sum of 1s for every element that is different, i.e., "city block" distance.`, 4: `L2NormBinTol is the [L2Norm] square root of the sum of squares differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 5: `SumSquaresBinTol is the [SumSquares] differences between tensor values, with binary tolerance: differences < 0.5 are thresholded to 0.`, 6: `InvCosine is 1-[Cosine], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 7: `InvCorrelation is 1-[Correlation], which is useful to convert it to an Increasing metric where more different vectors have larger metric values.`, 8: `CrossEntropy is a standard measure of the difference between two probabilty distributions, reflecting the additional entropy (uncertainty) associated with measuring probabilities under distribution b when in fact they come from distribution a. It is also the entropy of a plus the divergence between a from b, using Kullback-Leibler (KL) divergence. It is computed as: a * log(a/b) + (1-a) * log(1-a/1-b).`, 9: `DotProduct is the sum of the co-products of the tensor values.`, 10: `Covariance is co-variance between two vectors, i.e., the mean of the co-product of each vector element minus the mean of that vector: cov(A,B) = E[(A - E(A))(B - E(B))].`, 11: `Correlation is the standardized [Covariance] in the range (-1..1), computed as the mean of the co-product of each vector element minus the mean of that vector, normalized by the product of their standard deviations: cor(A,B) = E[(A - E(A))(B - E(B))] / sigma(A) sigma(B). Equivalent to the [Cosine] of mean-normalized vectors.`, 12: `Cosine is high-dimensional angle between two vectors, in range (-1..1) as the normalized [DotProduct]: inner product / sqrt(ssA * ssB). See also [Correlation].`} -var _MetricsMap = map[Metrics]string{0: `NormL2`, 1: `SumSquares`, 2: `NormL1`, 3: `Hamming`, 4: `NormL2BinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `DotProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} +var _MetricsMap = map[Metrics]string{0: `L2Norm`, 1: `SumSquares`, 2: `L1Norm`, 3: `Hamming`, 4: `L2NormBinTol`, 5: `SumSquaresBinTol`, 6: `InvCosine`, 7: `InvCorrelation`, 8: `CrossEntropy`, 9: `DotProduct`, 10: `Covariance`, 11: `Correlation`, 12: `Cosine`} // String returns the string representation of this Metrics value. func (i Metrics) String() string { return enums.String(i, _MetricsMap) } diff --git a/tensor/stats/metric/funcs.go b/tensor/stats/metric/funcs.go index aa122aede9..d00f47077e 100644 --- a/tensor/stats/metric/funcs.go +++ b/tensor/stats/metric/funcs.go @@ -84,9 +84,9 @@ func SumSquares(a, b tensor.Tensor) tensor.Values { return out } -// NormL2Out computes the L2 Norm: square root of the sum of squares +// L2NormOut computes the L2 Norm: square root of the sum of squares // differences between tensor values, aka the Euclidean distance. -func NormL2Out(a, b tensor.Tensor, out tensor.Values) error { +func L2NormOut(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresScaleOut64(a, b, out) if err != nil { return err @@ -107,17 +107,17 @@ func NormL2Out(a, b tensor.Tensor, out tensor.Values) error { return nil } -// NormL2 computes the L2 Norm: square root of the sum of squares +// L2Norm computes the L2 Norm: square root of the sum of squares // differences between tensor values, aka the Euclidean distance. // See [MetricFunc] for general information. -func NormL2(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(NormL2Out, a, b) +func L2Norm(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(L2NormOut, a, b) } -// NormL1Out computes the sum of the absolute value of differences between the +// L1NormOut computes the sum of the absolute value of differences between the // tensor values, the L1 Norm. // See [MetricOutFunc] for general information. -func NormL1Out(a, b tensor.Tensor, out tensor.Values) error { +func L1NormOut(a, b tensor.Tensor, out tensor.Values) error { if err := tensor.MustBeSameShape(a, b); err != nil { return err } @@ -129,11 +129,11 @@ func NormL1Out(a, b tensor.Tensor, out tensor.Values) error { return err } -// NormL1 computes the sum of the absolute value of differences between the +// L1Norm computes the sum of the absolute value of differences between the // tensor values, the L1 Norm. // See [MetricFunc] for general information. -func NormL1(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(NormL1Out, a, b) +func L1Norm(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(L1NormOut, a, b) } // HammingOut computes the sum of 1s for every element that is different, @@ -180,10 +180,10 @@ func SumSquaresBinTolScaleOut64(a, b tensor.Tensor, out tensor.Values) (scale64, return } -// NormL2BinTolOut computes the L2 Norm square root of the sum of squares +// L2NormBinTolOut computes the L2 Norm square root of the sum of squares // differences between tensor values (aka Euclidean distance), with binary tolerance: // differences < 0.5 are thresholded to 0. -func NormL2BinTolOut(a, b tensor.Tensor, out tensor.Values) error { +func L2NormBinTolOut(a, b tensor.Tensor, out tensor.Values) error { scale64, ss64, err := SumSquaresBinTolScaleOut64(a, b, out) if err != nil { return err @@ -204,12 +204,12 @@ func NormL2BinTolOut(a, b tensor.Tensor, out tensor.Values) error { return nil } -// NormL2BinTol computes the L2 Norm square root of the sum of squares +// L2NormBinTol computes the L2 Norm square root of the sum of squares // differences between tensor values (aka Euclidean distance), with binary tolerance: // differences < 0.5 are thresholded to 0. // See [MetricFunc] for general information. -func NormL2BinTol(a, b tensor.Tensor) tensor.Values { - return tensor.CallOut2(NormL2BinTolOut, a, b) +func L2NormBinTol(a, b tensor.Tensor) tensor.Values { + return tensor.CallOut2(L2NormBinTolOut, a, b) } // SumSquaresBinTolOut computes the sum of squares differences between tensor values, diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index 9f7f7228f2..caa43d5f80 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -25,17 +25,17 @@ func TestFuncs(t *testing.T) { btsr := tensor.NewNumberFromValues(b64...) out := tensor.NewFloat64(1) - NormL2Out(atsr, btsr, out) - assert.InDelta(t, results[MetricNormL2], out.Values[0], tol) + L2NormOut(atsr, btsr, out) + assert.InDelta(t, results[MetricL2Norm], out.Values[0], tol) SumSquaresOut(atsr, btsr, out) assert.InDelta(t, results[MetricSumSquares], out.Values[0], tol) - NormL2BinTolOut(atsr, btsr, out) - assert.InDelta(t, results[MetricNormL2BinTol], out.Values[0], tol) + L2NormBinTolOut(atsr, btsr, out) + assert.InDelta(t, results[MetricL2NormBinTol], out.Values[0], tol) - NormL1Out(atsr, btsr, out) - assert.InDelta(t, results[MetricNormL1], out.Values[0], tol) + L1NormOut(atsr, btsr, out) + assert.InDelta(t, results[MetricL1Norm], out.Values[0], tol) HammingOut(atsr, btsr, out) assert.Equal(t, results[MetricHamming], out.Values[0]) @@ -64,7 +64,7 @@ func TestFuncs(t *testing.T) { InvCosineOut(atsr, btsr, out) assert.InDelta(t, results[MetricInvCosine], out.Values[0], tol) - for met := MetricNormL2; met < MetricsN; met++ { + for met := MetricL2Norm; met < MetricsN; met++ { out := met.Call(atsr, btsr) assert.InDelta(t, results[met], out.Float1D(0), tol) } @@ -79,7 +79,7 @@ func TestMatrix(t *testing.T) { assert.NoError(t, err) in := dt.Column("Input") out := tensor.NewFloat64() - err = MatrixOut(NormL2, in, out) + err = MatrixOut(L2Norm, in, out) assert.NoError(t, err) // fmt.Println(out.Tensor) for i, v := range simres { diff --git a/tensor/stats/metric/metrics.go b/tensor/stats/metric/metrics.go index eeb33d93c2..2dcf0790d4 100644 --- a/tensor/stats/metric/metrics.go +++ b/tensor/stats/metric/metrics.go @@ -12,11 +12,11 @@ import ( ) func init() { - tensor.AddFunc(MetricNormL2.FuncName(), NormL2) + tensor.AddFunc(MetricL2Norm.FuncName(), L2Norm) tensor.AddFunc(MetricSumSquares.FuncName(), SumSquares) - tensor.AddFunc(MetricNormL1.FuncName(), NormL1) + tensor.AddFunc(MetricL1Norm.FuncName(), L1Norm) tensor.AddFunc(MetricHamming.FuncName(), Hamming) - tensor.AddFunc(MetricNormL2BinTol.FuncName(), NormL2BinTol) + tensor.AddFunc(MetricL2NormBinTol.FuncName(), L2NormBinTol) tensor.AddFunc(MetricSumSquaresBinTol.FuncName(), SumSquaresBinTol) tensor.AddFunc(MetricInvCosine.FuncName(), InvCosine) tensor.AddFunc(MetricInvCorrelation.FuncName(), InvCorrelation) @@ -31,25 +31,25 @@ func init() { type Metrics int32 //enums:enum -trim-prefix Metric const ( - // NormL2 is the square root of the sum of squares differences + // L2Norm is the square root of the sum of squares differences // between tensor values, aka the L2 Norm. - MetricNormL2 Metrics = iota + MetricL2Norm Metrics = iota // SumSquares is the sum of squares differences between tensor values. MetricSumSquares - // NormL1 is the sum of the absolute value of differences + // L1Norm is the sum of the absolute value of differences // between tensor values, the L1 Norm. - MetricNormL1 + MetricL1Norm // Hamming is the sum of 1s for every element that is different, // i.e., "city block" distance. MetricHamming - // NormL2BinTol is the [NormL2] square root of the sum of squares + // L2NormBinTol is the [L2Norm] square root of the sum of squares // differences between tensor values, with binary tolerance: // differences < 0.5 are thresholded to 0. - MetricNormL2BinTol + MetricL2NormBinTol // SumSquaresBinTol is the [SumSquares] differences between tensor values, // with binary tolerance: differences < 0.5 are thresholded to 0. @@ -114,7 +114,7 @@ func (m Metrics) Call(a, b tensor.Tensor) tensor.Values { } // Increasing returns true if the distance metric is such that metric -// values increase as a function of distance (e.g., NormL2) +// values increase as a function of distance (e.g., L2Norm) // and false if metric values decrease as a function of distance // (e.g., Cosine, Correlation) func (m Metrics) Increasing() bool { diff --git a/tensor/stats/stats/README.md b/tensor/stats/stats/README.md index dc99b8b8f6..19773f0491 100644 --- a/tensor/stats/stats/README.md +++ b/tensor/stats/stats/README.md @@ -20,7 +20,7 @@ All stats are registered in the `tensor.Funcs` global list (for use in Goal), an stats.Mean.Call(in, out) ``` -All stats functions (and all tensor functions more generally) skip over NaN's as a missing value, so they are equivalent to the `nanmean` etc versions in NumPy. +All stats functions skip over `NaN`s as a missing value, so they are equivalent to the `nanmean` etc versions in NumPy. ## Stats @@ -28,7 +28,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Count`: count of number of elements * `Sum`: sum of elements -* `NormL1`: L1 Norm: sum of absolute values +* `L1Norm`: L1 Norm: sum of absolute values * `Prod`: product of elements * `Min`: minimum value * `Max`: maximum value @@ -39,7 +39,7 @@ The following statistics are supported (per the `Stats` enum in `stats.go`): * `Std`: sample standard deviation (sqrt of Var) * `Sem`: sample standard error of the mean (Std divided by sqrt(n)) * `SumSq`: sum of squared element values -* `NormL2`: L2 Norm: square-root of sum-of-squares +* `L2Norm`: L2 Norm: square-root of sum-of-squares * `VarPop`: population variance (squared diffs from mean, divided by n) * `StdPop`: population standard deviation (sqrt of VarPop) * `SemPop`: population standard error of the mean (StdPop divided by sqrt(n)) diff --git a/tensor/stats/stats/enumgen.go b/tensor/stats/stats/enumgen.go index b4db1f6c7c..1c5691e379 100644 --- a/tensor/stats/stats/enumgen.go +++ b/tensor/stats/stats/enumgen.go @@ -11,11 +11,11 @@ var _StatsValues = []Stats{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // StatsN is the highest valid value for type Stats, plus one. const StatsN Stats = 20 -var _StatsValueMap = map[string]Stats{`Count`: 0, `Sum`: 1, `NormL1`: 2, `Prod`: 3, `Min`: 4, `Max`: 5, `MinAbs`: 6, `MaxAbs`: 7, `Mean`: 8, `Var`: 9, `Std`: 10, `Sem`: 11, `SumSq`: 12, `NormL2`: 13, `VarPop`: 14, `StdPop`: 15, `SemPop`: 16, `Median`: 17, `Q1`: 18, `Q3`: 19} +var _StatsValueMap = map[string]Stats{`Count`: 0, `Sum`: 1, `L1Norm`: 2, `Prod`: 3, `Min`: 4, `Max`: 5, `MinAbs`: 6, `MaxAbs`: 7, `Mean`: 8, `Var`: 9, `Std`: 10, `Sem`: 11, `SumSq`: 12, `L2Norm`: 13, `VarPop`: 14, `StdPop`: 15, `SemPop`: 16, `Median`: 17, `Q1`: 18, `Q3`: 19} var _StatsDescMap = map[Stats]string{0: `count of number of elements.`, 1: `sum of elements.`, 2: `L1 Norm: sum of absolute values of elements.`, 3: `product of elements.`, 4: `minimum value.`, 5: `maximum value.`, 6: `minimum of absolute values.`, 7: `maximum of absolute values.`, 8: `mean value = sum / count.`, 9: `sample variance (squared deviations from mean, divided by n-1).`, 10: `sample standard deviation (sqrt of Var).`, 11: `sample standard error of the mean (Std divided by sqrt(n)).`, 12: `sum of squared values.`, 13: `L2 Norm: square-root of sum-of-squares.`, 14: `population variance (squared diffs from mean, divided by n).`, 15: `population standard deviation (sqrt of VarPop).`, 16: `population standard error of the mean (StdPop divided by sqrt(n)).`, 17: `middle value in sorted ordering.`, 18: `Q1 first quartile = 25%ile value = .25 quantile value.`, 19: `Q3 third quartile = 75%ile value = .75 quantile value.`} -var _StatsMap = map[Stats]string{0: `Count`, 1: `Sum`, 2: `NormL1`, 3: `Prod`, 4: `Min`, 5: `Max`, 6: `MinAbs`, 7: `MaxAbs`, 8: `Mean`, 9: `Var`, 10: `Std`, 11: `Sem`, 12: `SumSq`, 13: `NormL2`, 14: `VarPop`, 15: `StdPop`, 16: `SemPop`, 17: `Median`, 18: `Q1`, 19: `Q3`} +var _StatsMap = map[Stats]string{0: `Count`, 1: `Sum`, 2: `L1Norm`, 3: `Prod`, 4: `Min`, 5: `Max`, 6: `MinAbs`, 7: `MaxAbs`, 8: `Mean`, 9: `Var`, 10: `Std`, 11: `Sem`, 12: `SumSq`, 13: `L2Norm`, 14: `VarPop`, 15: `StdPop`, 16: `SemPop`, 17: `Median`, 18: `Q1`, 19: `Q3`} // String returns the string representation of this Stats value. func (i Stats) String() string { return enums.String(i, _StatsMap) } diff --git a/tensor/stats/stats/funcs.go b/tensor/stats/stats/funcs.go index 6da443f43c..d04c296d77 100644 --- a/tensor/stats/stats/funcs.go +++ b/tensor/stats/stats/funcs.go @@ -82,17 +82,15 @@ func Sum(in tensor.Tensor) tensor.Values { return out } -// NormL1 computes the sum of absolute-value-of tensor values. -// This is also known as the L1 norm. +// L1Norm computes the sum of absolute-value-of tensor values. // See [StatsFunc] for general information. -func NormL1(in tensor.Tensor) tensor.Values { - return tensor.CallOut1(NormL1Out, in) +func L1Norm(in tensor.Tensor) tensor.Values { + return tensor.CallOut1(L1NormOut, in) } -// NormL1Out computes the sum of absolute-value-of tensor values. -// This is also known as the L1 norm. +// L1NormOut computes the sum of absolute-value-of tensor values. // See [StatsFunc] for general information. -func NormL1Out(in tensor.Tensor, out tensor.Values) error { +func L1NormOut(in tensor.Tensor, out tensor.Values) error { _, err := VectorizeOut64(NFunc, func(idx int, tsr ...tensor.Tensor) { VecFunc(idx, tsr[0], tsr[1], 0, func(val, agg float64) float64 { return agg + math.Abs(val) @@ -485,10 +483,10 @@ func SumSqOut(in tensor.Tensor, out tensor.Values) error { return err } -// NormL2Out64 computes the square root of the sum of squares of tensor values, +// L2NormOut64 computes the square root of the sum of squares of tensor values, // known as the L2 norm, and returns the Float64 output values for // use in subsequent computations. -func NormL2Out64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { +func L2NormOut64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { scale64, ss64, err := SumSqScaleOut64(in, out) if err != nil { return nil, err @@ -509,19 +507,19 @@ func NormL2Out64(in tensor.Tensor, out tensor.Values) (tensor.Tensor, error) { return scale64, nil } -// NormL2 computes the square root of the sum of squares of tensor values, +// L2Norm computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsFunc] for general information. -func NormL2(in tensor.Tensor) tensor.Values { +func L2Norm(in tensor.Tensor) tensor.Values { out := tensor.NewOfType(in.DataType()) - errors.Log1(NormL2Out64(in, out)) + errors.Log1(L2NormOut64(in, out)) return out } -// NormL2Out computes the square root of the sum of squares of tensor values, +// L2NormOut computes the square root of the sum of squares of tensor values, // known as the L2 norm. // See [StatsOutFunc] for general information. -func NormL2Out(in tensor.Tensor, out tensor.Values) error { - _, err := NormL2Out64(in, out) +func L2NormOut(in tensor.Tensor, out tensor.Values) error { + _, err := L2NormOut64(in, out) return err } diff --git a/tensor/stats/stats/stats.go b/tensor/stats/stats/stats.go index 9e0e1c8933..42808da987 100644 --- a/tensor/stats/stats/stats.go +++ b/tensor/stats/stats/stats.go @@ -16,7 +16,7 @@ import ( func init() { tensor.AddFunc(StatCount.FuncName(), Count) tensor.AddFunc(StatSum.FuncName(), Sum) - tensor.AddFunc(StatNormL1.FuncName(), NormL1) + tensor.AddFunc(StatL1Norm.FuncName(), L1Norm) tensor.AddFunc(StatProd.FuncName(), Prod) tensor.AddFunc(StatMin.FuncName(), Min) tensor.AddFunc(StatMax.FuncName(), Max) @@ -27,7 +27,7 @@ func init() { tensor.AddFunc(StatStd.FuncName(), Std) tensor.AddFunc(StatSem.FuncName(), Sem) tensor.AddFunc(StatSumSq.FuncName(), SumSq) - tensor.AddFunc(StatNormL2.FuncName(), NormL2) + tensor.AddFunc(StatL2Norm.FuncName(), L2Norm) tensor.AddFunc(StatVarPop.FuncName(), VarPop) tensor.AddFunc(StatStdPop.FuncName(), StdPop) tensor.AddFunc(StatSemPop.FuncName(), SemPop) @@ -48,7 +48,7 @@ const ( StatSum // L1 Norm: sum of absolute values of elements. - StatNormL1 + StatL1Norm // product of elements. StatProd @@ -81,7 +81,7 @@ const ( StatSumSq // L2 Norm: square-root of sum-of-squares. - StatNormL2 + StatL2Norm // population variance (squared diffs from mean, divided by n). StatVarPop diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index 7d6a3fb86d..f6ae0066c1 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -27,8 +27,8 @@ func TestFuncs64(t *testing.T) { SumOut(ix, out) assert.Equal(t, results[StatSum], out.Values[0]) - NormL1Out(ix, out) - assert.Equal(t, results[StatNormL1], out.Values[0]) + L1NormOut(ix, out) + assert.Equal(t, results[StatL1Norm], out.Values[0]) ProdOut(ix, out) assert.Equal(t, results[StatProd], out.Values[0]) @@ -69,8 +69,8 @@ func TestFuncs64(t *testing.T) { SumSqOut(ix, out) assert.InDelta(t, results[StatSumSq], out.Values[0], tol) - NormL2Out(ix, out) - assert.InDelta(t, results[StatNormL2], out.Values[0], tol) + L2NormOut(ix, out) + assert.InDelta(t, results[StatL2Norm], out.Values[0], tol) MedianOut(ix, out) assert.InDelta(t, results[StatMedian], out.Values[0], tol) @@ -101,8 +101,8 @@ func TestFuncsInt(t *testing.T) { SumOut(ix, out) assert.Equal(t, results[StatSum], out.Values[0]) - NormL1Out(ix, out) - assert.Equal(t, results[StatNormL1], out.Values[0]) + L1NormOut(ix, out) + assert.Equal(t, results[StatL1Norm], out.Values[0]) ProdOut(ix, out) assert.Equal(t, results[StatProd], out.Values[0]) @@ -143,8 +143,8 @@ func TestFuncsInt(t *testing.T) { SumSqOut(ix, out) assert.Equal(t, results[StatSumSq], out.Values[0]) - NormL2Out(ix, out) - assert.Equal(t, results[StatNormL2], out.Values[0]) + L2NormOut(ix, out) + assert.Equal(t, results[StatL2Norm], out.Values[0]) for stat := StatCount; stat <= StatSemPop; stat++ { out := stat.Call(ix) diff --git a/tensor/vector/vector.go b/tensor/vector/vector.go index 431d0a4272..b6fd70d9df 100644 --- a/tensor/vector/vector.go +++ b/tensor/vector/vector.go @@ -38,22 +38,22 @@ func Sum(a tensor.Tensor) tensor.Values { } // Dot performs the vector dot product: the [Sum] of the [Mul] product -// of the two tensors, returning a scalar value. +// of the two tensors, returning a scalar value. Also known as the inner product. func Dot(a, b tensor.Tensor) tensor.Values { return Sum(Mul(a, b)) } -// NormL2 returns the length of the vector as the L2 Norm: +// L2Norm returns the length of the vector as the L2 Norm: // square root of the sum of squared values of the vector, as a scalar. // This is the Sqrt of the [Dot] product of the vector with itself. -func NormL2(a tensor.Tensor) tensor.Values { +func L2Norm(a tensor.Tensor) tensor.Values { dot := Dot(a, a).Float1D(0) return tensor.NewFloat64Scalar(math.Sqrt(dot)) } -// NormL1 returns the length of the vector as the L1 Norm: +// L1Norm returns the length of the vector as the L1 Norm: // sum of the absolute values of the tensor, as a scalar. -func NormL1(a tensor.Tensor) tensor.Values { +func L1Norm(a tensor.Tensor) tensor.Values { n := a.Len() sum := 0.0 tensor.Vectorize(func(tsr ...tensor.Tensor) int { return n }, diff --git a/tensor/vector/vector_test.go b/tensor/vector/vector_test.go index 69e26e5979..3fde1fa7e1 100644 --- a/tensor/vector/vector_test.go +++ b/tensor/vector/vector_test.go @@ -23,9 +23,9 @@ func TestVector(t *testing.T) { dpv := Dot(v, v).(*tensor.Float64) assert.Equal(t, 14.0, dpv.Values[0]) - nl2v := NormL2(v).(*tensor.Float64) + nl2v := L2Norm(v).(*tensor.Float64) assert.Equal(t, math.Sqrt(14.0), nl2v.Values[0]) - nl1v := NormL1(v).(*tensor.Float64) + nl1v := L1Norm(v).(*tensor.Float64) assert.Equal(t, 6.0, nl1v.Values[0]) } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go index 3c02914d98..6a72da3926 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -35,6 +35,12 @@ func init() { "InvCorrelationOut": reflect.ValueOf(metric.InvCorrelationOut), "InvCosine": reflect.ValueOf(metric.InvCosine), "InvCosineOut": reflect.ValueOf(metric.InvCosineOut), + "L1Norm": reflect.ValueOf(metric.L1Norm), + "L1NormOut": reflect.ValueOf(metric.L1NormOut), + "L2Norm": reflect.ValueOf(metric.L2Norm), + "L2NormBinTol": reflect.ValueOf(metric.L2NormBinTol), + "L2NormBinTolOut": reflect.ValueOf(metric.L2NormBinTolOut), + "L2NormOut": reflect.ValueOf(metric.L2NormOut), "Matrix": reflect.ValueOf(metric.Matrix), "MatrixOut": reflect.ValueOf(metric.MatrixOut), "MetricCorrelation": reflect.ValueOf(metric.MetricCorrelation), @@ -45,20 +51,14 @@ func init() { "MetricHamming": reflect.ValueOf(metric.MetricHamming), "MetricInvCorrelation": reflect.ValueOf(metric.MetricInvCorrelation), "MetricInvCosine": reflect.ValueOf(metric.MetricInvCosine), - "MetricNormL1": reflect.ValueOf(metric.MetricNormL1), - "MetricNormL2": reflect.ValueOf(metric.MetricNormL2), - "MetricNormL2BinTol": reflect.ValueOf(metric.MetricNormL2BinTol), + "MetricL1Norm": reflect.ValueOf(metric.MetricL1Norm), + "MetricL2Norm": reflect.ValueOf(metric.MetricL2Norm), + "MetricL2NormBinTol": reflect.ValueOf(metric.MetricL2NormBinTol), "MetricSumSquares": reflect.ValueOf(metric.MetricSumSquares), "MetricSumSquaresBinTol": reflect.ValueOf(metric.MetricSumSquaresBinTol), "MetricsN": reflect.ValueOf(metric.MetricsN), "MetricsValues": reflect.ValueOf(metric.MetricsValues), "NFunc": reflect.ValueOf(metric.NFunc), - "NormL1": reflect.ValueOf(metric.NormL1), - "NormL1Out": reflect.ValueOf(metric.NormL1Out), - "NormL2": reflect.ValueOf(metric.NormL2), - "NormL2BinTol": reflect.ValueOf(metric.NormL2BinTol), - "NormL2BinTolOut": reflect.ValueOf(metric.NormL2BinTolOut), - "NormL2Out": reflect.ValueOf(metric.NormL2Out), "PCA": reflect.ValueOf(metric.PCA), "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), "ProjectOnMatrixColumnOut": reflect.ValueOf(metric.ProjectOnMatrixColumnOut), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go index 6b8437582c..95b716c7a4 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go @@ -26,6 +26,11 @@ func init() { "GroupDescribe": reflect.ValueOf(stats.GroupDescribe), "GroupStats": reflect.ValueOf(stats.GroupStats), "Groups": reflect.ValueOf(stats.Groups), + "L1Norm": reflect.ValueOf(stats.L1Norm), + "L1NormOut": reflect.ValueOf(stats.L1NormOut), + "L2Norm": reflect.ValueOf(stats.L2Norm), + "L2NormOut": reflect.ValueOf(stats.L2NormOut), + "L2NormOut64": reflect.ValueOf(stats.L2NormOut64), "Max": reflect.ValueOf(stats.Max), "MaxAbs": reflect.ValueOf(stats.MaxAbs), "MaxAbsOut": reflect.ValueOf(stats.MaxAbsOut), @@ -40,11 +45,6 @@ func init() { "MinAbsOut": reflect.ValueOf(stats.MinAbsOut), "MinOut": reflect.ValueOf(stats.MinOut), "NFunc": reflect.ValueOf(stats.NFunc), - "NormL1": reflect.ValueOf(stats.NormL1), - "NormL1Out": reflect.ValueOf(stats.NormL1Out), - "NormL2": reflect.ValueOf(stats.NormL2), - "NormL2Out": reflect.ValueOf(stats.NormL2Out), - "NormL2Out64": reflect.ValueOf(stats.NormL2Out64), "Prod": reflect.ValueOf(stats.Prod), "ProdOut": reflect.ValueOf(stats.ProdOut), "Q1": reflect.ValueOf(stats.Q1), @@ -58,14 +58,14 @@ func init() { "SemPop": reflect.ValueOf(stats.SemPop), "SemPopOut": reflect.ValueOf(stats.SemPopOut), "StatCount": reflect.ValueOf(stats.StatCount), + "StatL1Norm": reflect.ValueOf(stats.StatL1Norm), + "StatL2Norm": reflect.ValueOf(stats.StatL2Norm), "StatMax": reflect.ValueOf(stats.StatMax), "StatMaxAbs": reflect.ValueOf(stats.StatMaxAbs), "StatMean": reflect.ValueOf(stats.StatMean), "StatMedian": reflect.ValueOf(stats.StatMedian), "StatMin": reflect.ValueOf(stats.StatMin), "StatMinAbs": reflect.ValueOf(stats.StatMinAbs), - "StatNormL1": reflect.ValueOf(stats.StatNormL1), - "StatNormL2": reflect.ValueOf(stats.StatNormL2), "StatProd": reflect.ValueOf(stats.StatProd), "StatQ1": reflect.ValueOf(stats.StatQ1), "StatQ3": reflect.ValueOf(stats.StatQ3), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go b/yaegicore/symbols/cogentcore_org-core-tensor-vector.go index 5a1d61788f..a5d83b1fc9 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-vector.go @@ -11,10 +11,10 @@ func init() { Symbols["cogentcore.org/core/tensor/vector/vector"] = map[string]reflect.Value{ // function, constant and variable definitions "Dot": reflect.ValueOf(vector.Dot), + "L1Norm": reflect.ValueOf(vector.L1Norm), + "L2Norm": reflect.ValueOf(vector.L2Norm), "Mul": reflect.ValueOf(vector.Mul), "MulOut": reflect.ValueOf(vector.MulOut), - "NormL1": reflect.ValueOf(vector.NormL1), - "NormL2": reflect.ValueOf(vector.NormL2), "Sum": reflect.ValueOf(vector.Sum), } } From 186fc34d2a15ade50be7e126f4c89041f029eb98 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 00:10:15 -0700 Subject: [PATCH 174/311] matrix multiplication fully working -- need @ wrapper --- tensor/align.go | 22 ++++++ tensor/matrix/matrix.go | 28 ++++++-- tensor/matrix/matrix_test.go | 70 +++++++++++++++++++ tensor/matrix/ops.go | 125 ++++++++++++++++++++++++++++++++++ tensor/stats/metric/matrix.go | 4 +- 5 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 tensor/matrix/matrix_test.go create mode 100644 tensor/matrix/ops.go diff --git a/tensor/align.go b/tensor/align.go index 79b046c6fd..9617d48c2c 100644 --- a/tensor/align.go +++ b/tensor/align.go @@ -111,6 +111,28 @@ func AlignForAssign(a, b Tensor) (as, bs *Shape, err error) { return } +// SplitAtInnerDims returns the sizes of the given tensor's shape +// with the given number of inner-most dimensions retained as is, +// and those above collapsed to a single dimension. +// If the total number of dimensions is < nInner the result is nil. +func SplitAtInnerDims(tsr Tensor, nInner int) []int { + sizes := tsr.ShapeSizes() + nd := len(sizes) + if nd < nInner { + return nil + } + rsz := make([]int, nInner+1) + split := nd - nInner + rows := sizes[:split] + copy(rsz[1:], sizes[split:]) + nr := 1 + for _, r := range rows { + nr *= r + } + rsz[0] = nr + return rsz +} + // FloatAssignFunc sets a to a binary function of a and b float64 values. func FloatAssignFunc(fun func(a, b float64) float64, a, b Tensor) error { as, bs, err := AlignForAssign(a, b) diff --git a/tensor/matrix/matrix.go b/tensor/matrix/matrix.go index 796f3db14c..1c43e00927 100644 --- a/tensor/matrix/matrix.go +++ b/tensor/matrix/matrix.go @@ -17,11 +17,17 @@ type Matrix struct { Tensor tensor.Tensor } +func StringCheck(tsr tensor.Tensor) error { + if tsr.IsString() { + return errors.New("matrix: tensor has string values; must be numeric") + } + return nil +} + // NewMatrix returns given [tensor.Tensor] as a [gonum] [mat.Matrix]. // It returns an error if the tensor is not 2D. func NewMatrix(tsr tensor.Tensor) (*Matrix, error) { - if tsr.IsString() { - err := errors.New("matrix.NewMatrix: tensor has string values; must be numeric") + if err := StringCheck(tsr); err != nil { return nil, err } nd := tsr.NumDims() @@ -85,9 +91,23 @@ func (sy *Symmetric) SymmetricDim() (r int) { return sy.Tensor.DimSize(0) } -// CopyDense copies a gonum mat.Dense matrix into given Tensor +// NewDense returns given [tensor.Float64] as a [gonum] [mat.Dense] +// Matrix, on which many of the matrix operations are defined. +// It functions similar to the [tensor.Values] type, as the output +// of matrix operations. The Dense type serves as a view onto +// the tensor's data, so operations directly modify it. +func NewDense(tsr *tensor.Float64) (*mat.Dense, error) { + nd := tsr.NumDims() + if nd != 2 { + err := errors.New("matrix.NewDense: tensor is not 2D") + return nil, err + } + return mat.NewDense(tsr.DimSize(0), tsr.DimSize(1), tsr.Values), nil +} + +// CopyFromDense copies a gonum mat.Dense matrix into given Tensor // using standard Float64 interface -func CopyDense(to tensor.Values, dm *mat.Dense) { +func CopyFromDense(to tensor.Values, dm *mat.Dense) { nr, nc := dm.Dims() to.SetShapeSizes(nr, nc) idx := 0 diff --git a/tensor/matrix/matrix_test.go b/tensor/matrix/matrix_test.go new file mode 100644 index 0000000000..1ac8477452 --- /dev/null +++ b/tensor/matrix/matrix_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "testing" + + "cogentcore.org/core/tensor" + "github.com/stretchr/testify/assert" +) + +func TestMatrix(t *testing.T) { + a := tensor.AsFloat64(tensor.Reshape(tensor.NewIntRange(1, 5), 2, 2)) + // fmt.Println(a) + + v := tensor.NewFloat64FromValues(2, 3) + _ = v + + o := Mul(a, a).(*tensor.Float64) + // fmt.Println(o) + + assert.Equal(t, []float64{7, 10, 15, 22}, o.Values) + + o = Mul(a, v).(*tensor.Float64) + // fmt.Println(o) + + assert.Equal(t, []float64{8, 18}, o.Values) + assert.Equal(t, []int{2, 1}, o.Shape().Sizes) + + o = Mul(v, a).(*tensor.Float64) + // fmt.Println(o) + + assert.Equal(t, []float64{11, 16}, o.Values) + assert.Equal(t, []int{1, 2}, o.Shape().Sizes) + + nr := 3 + b := tensor.NewFloat64(nr, 2, 2) + for r := range nr { + b.SetRowTensor(a, r) + } + // fmt.Println(b) + + o = Mul(b, a).(*tensor.Float64) + // fmt.Println(o) + assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) + assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) + + o = Mul(a, b).(*tensor.Float64) + // fmt.Println(o) + assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) + assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) + + o = Mul(b, b).(*tensor.Float64) + // fmt.Println(o) + assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) + assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) + + o = Mul(v, b).(*tensor.Float64) + // fmt.Println(o) + assert.Equal(t, []float64{11, 16, 11, 16, 11, 16}, o.Values) + assert.Equal(t, []int{3, 1, 2}, o.Shape().Sizes) + + o = Mul(b, v).(*tensor.Float64) + // fmt.Println(o) + assert.Equal(t, []float64{8, 18, 8, 18, 8, 18}, o.Values) + assert.Equal(t, []int{3, 2, 1}, o.Shape().Sizes) + +} diff --git a/tensor/matrix/ops.go b/tensor/matrix/ops.go new file mode 100644 index 0000000000..54eedfb37e --- /dev/null +++ b/tensor/matrix/ops.go @@ -0,0 +1,125 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" +) + +// CallOut2 calls an Out function with 2 input args. All matrix functions +// require *tensor.Float64 outputs. +func CallOut2(fun func(a, b tensor.Tensor, out *tensor.Float64) error, a, b tensor.Tensor) tensor.Values { + out := tensor.NewFloat64() + errors.Log(fun(a, b, out)) + return out +} + +// Mul performs matrix multiplication, using the following rules based +// on the shapes of the relevant tensors. If the tensor shapes are not +// suitable, an error is logged (see [MulOut] for a version returning the error). +// - If both arguments are 2-D they are multiplied like conventional matrices. +// - If either argument is N-D, N > 2, it is treated as a stack of matrices +// residing in the last two indexes and broadcast accordingly. +// - If the first argument is 1-D, it is promoted to a matrix by prepending +// a 1 to its dimensions. After matrix multiplication the prepended 1 is removed. +// - If the second argument is 1-D, it is promoted to a matrix by appending +// a 1 to its dimensions. After matrix multiplication the appended 1 is removed. +func Mul(a, b tensor.Tensor) tensor.Tensor { + return CallOut2(MulOut, a, b) +} + +// MulOut performs matrix multiplication, into the given output tensor, +// using the following rules based +// on the shapes of the relevant tensors. If the tensor shapes are not +// suitable, an error is returned. +// - If both arguments are 2-D they are multiplied like conventional matrices. +// The result has shape a.Rows, b.Columns. +// - If either argument is N-D, N > 2, it is treated as a stack of matrices +// residing in the last two indexes and broadcast accordingly. Both cannot +// be > 2 dimensional, unless their outer dimension size is 1 or the same. +// - If the first argument is 1-D, it is promoted to a matrix by prepending +// a 1 to its dimensions. After matrix multiplication the prepended 1 is removed. +// - If the second argument is 1-D, it is promoted to a matrix by appending +// a 1 to its dimensions. After matrix multiplication the appended 1 is removed. +func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { + if err := StringCheck(a); err != nil { + return err + } + if err := StringCheck(b); err != nil { + return err + } + na := a.NumDims() + nb := b.NumDims() + ea := a + eb := b + if na == 1 { + ea = tensor.Reshape(a, 1, a.DimSize(0)) + na = 2 + } + if nb == 1 { + eb = tensor.Reshape(b, b.DimSize(0), 1) + nb = 2 + } + if na > 2 { + asz := tensor.SplitAtInnerDims(a, 2) + if asz[0] == 1 { + ea = tensor.Reshape(a, asz[1:]...) + na = 2 + } + } + if nb > 2 { + bsz := tensor.SplitAtInnerDims(b, 2) + if bsz[0] == 1 { + eb = tensor.Reshape(b, bsz[1:]...) + nb = 2 + } + } + switch { + case na == nb && na == 2: + ma, _ := NewMatrix(ea) + mb, _ := NewMatrix(eb) + out.SetShapeSizes(ea.DimSize(0), eb.DimSize(1)) + do, _ := NewDense(out) + do.Mul(ma, mb) + case na > 2 && nb == 2: + mb, _ := NewMatrix(eb) + nr := ea.DimSize(0) + out.SetShapeSizes(nr, ea.DimSize(1), eb.DimSize(1)) + for r := range nr { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + do.Mul(ma, mb) + } + case nb > 2 && na == 2: + ma, _ := NewMatrix(ea) + nr := eb.DimSize(0) + out.SetShapeSizes(nr, ea.DimSize(0), eb.DimSize(2)) + for r := range nr { + sb := tensor.Reslice(eb, r, tensor.FullAxis, tensor.FullAxis) + mb, _ := NewMatrix(sb) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + do.Mul(ma, mb) + } + case na > 2 && nb > 2: + if ea.DimSize(0) != eb.DimSize(0) { + return errors.New("matrix.Mul: a and b input matricies are > 2 dimensional; must have same outer dimension sizes") + } + nr := ea.DimSize(0) + out.SetShapeSizes(nr, ea.DimSize(1), eb.DimSize(2)) + for r := range nr { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + sb := tensor.Reslice(eb, r, tensor.FullAxis, tensor.FullAxis) + mb, _ := NewMatrix(sb) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + do.Mul(ma, mb) + } + default: + return errors.New("matrix.Mul: input dimensions do not align") + } + return nil +} diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 37e94c32e3..c3f8718c0b 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -212,7 +212,7 @@ func PCA(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { } var ev mat.Dense eig.VectorsTo(&ev) - matrix.CopyDense(eigenvecs, &ev) + matrix.CopyFromDense(eigenvecs, &ev) fv := tensor.AsFloat64(vals) eig.Values(fv.Values) if fv != vals { @@ -243,7 +243,7 @@ func SVD(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { } var ev mat.Dense eig.UTo(&ev) - matrix.CopyDense(eigenvecs, &ev) + matrix.CopyFromDense(eigenvecs, &ev) fv := tensor.AsFloat64(vals) eig.Values(fv.Values) if fv != vals { From ead829304243043df67872e3a58b058a1f67124f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 01:10:36 -0700 Subject: [PATCH 175/311] @ operator for matrix multiplication --- goal/transpile/math.go | 15 +++++++++++++++ goal/transpile/parser.go | 17 +++++++++++++++-- goal/transpile/transpile_test.go | 2 +- yaegicore/symbols/cogentcore_org-core-tensor.go | 1 + yaegicore/symbols/make | 2 +- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index a706c61045..a79e6ba8b7 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -480,6 +480,18 @@ func (mp *mathParse) exprsAreBool(ex []ast.Expr) bool { } func (mp *mathParse) binaryExpr(ex *ast.BinaryExpr) { + if ex.Op == token.ILLEGAL { // @ = matmul + mp.startFunc("matrix.Mul") + mp.out.Add(token.LPAREN) + mp.expr(ex.X) + mp.out.Add(token.COMMA) + mp.idx++ + mp.expr(ex.Y) + mp.out.Add(token.RPAREN) + mp.endFunc() + return + } + fn := "" switch ex.Op { case token.ADD: @@ -538,6 +550,9 @@ func (mp *mathParse) unaryExpr(ex *ast.UnaryExpr) { fn = "Not" case token.SUB: fn = "Negate" + case token.ADD: + mp.expr(ex.X) + return default: // * goes to StarExpr -- not sure what else could happen here? mp.addToken(ex.Op) mp.expr(ex.X) diff --git a/goal/transpile/parser.go b/goal/transpile/parser.go index 920a05f13f..8a09e65ea3 100644 --- a/goal/transpile/parser.go +++ b/goal/transpile/parser.go @@ -139,7 +139,11 @@ type parser struct { func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode Mode) { p.file = fset.AddFile(filename, -1, len(src)) - eh := func(pos token.Position, msg string) { p.errors.Add(pos, msg) } + eh := func(pos token.Position, msg string) { + if !strings.Contains(msg, "@") { + p.errors.Add(pos, msg) + } + } p.scanner.Init(p.file, src, eh, scanner.ScanComments) p.top = true @@ -1984,6 +1988,10 @@ func (p *parser) tokPrec() (token.Token, int) { if p.inRhs && tok == token.ASSIGN { tok = token.EQL } + if p.tok == token.ILLEGAL && p.lit == "@" { + // fmt.Println("@ token") + return token.ILLEGAL, 5 + } return tok, tok.Precedence() } @@ -2010,7 +2018,12 @@ func (p *parser) parseBinaryExpr(x ast.Expr, prec1 int) ast.Expr { if oprec < prec1 { return x } - pos := p.expect(op) + pos := p.pos + if op == token.ILLEGAL { + p.next() + } else { + pos = p.expect(op) + } y := p.parseBinaryExpr(nil, oprec+1) x = &ast.BinaryExpr{X: x, OpPos: pos, Op: op, Y: y} } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 655b69f589..b612729be8 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x += 1", `tmath.AddAssign(x, tensor.NewIntScalar(1))`}, + {"# x @ a", `matrix.Mul(x, a)`}, } st := NewState() st.MathRecord = false diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 797f2c443c..8302eac053 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -143,6 +143,7 @@ func init() { "SlicesMagicN": reflect.ValueOf(tensor.SlicesMagicN), "SlicesMagicValues": reflect.ValueOf(tensor.SlicesMagicValues), "Space": reflect.ValueOf(tensor.Space), + "SplitAtInnerDims": reflect.ValueOf(tensor.SplitAtInnerDims), "Sprintf": reflect.ValueOf(tensor.Sprintf), "Squeeze": reflect.ValueOf(tensor.Squeeze), "StableSort": reflect.ValueOf(tensor.StableSort), diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 83f777f772..572582e352 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -8,4 +8,4 @@ command extract { } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/vector tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo base/num +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/vector tensor/matrix tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo base/num From 9c7ab28676ca65871ae03d7803240863ce2b5e4a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 02:42:31 -0700 Subject: [PATCH 176/311] determinant functions --- goal/README.md | 7 ++-- goal/transpile/math.go | 38 +++++++++++-------- goal/transpile/transpile_test.go | 4 +- tensor/matrix/matrix_test.go | 16 +++----- tensor/matrix/ops.go | 26 +++++++++++++ tensor/reshaped.go | 10 +++++ tensor/tensor_test.go | 11 ++++++ .../cogentcore_org-core-tensor-matrix.go | 28 ++++++++++++++ .../symbols/cogentcore_org-core-tensor.go | 1 + 9 files changed, 111 insertions(+), 30 deletions(-) create mode 100644 yaegicore/symbols/cogentcore_org-core-tensor-matrix.go diff --git a/goal/README.md b/goal/README.md index e586642a47..5e5c40c619 100644 --- a/goal/README.md +++ b/goal/README.md @@ -363,9 +363,10 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| | |`a @ b` | `a * b` | matrix multiply | -| | |`a.transpose() or a.T` | `a.'` | transpose of a | -| | |`a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | +| `matrix.Mul(a,b)` | same: |`a @ b` | `a * b` | matrix multiply | +| `tensor.Transpose(a)` | same: |`a.transpose() or a.T` | `a.'` | transpose of `a` | +| TODO: | |`a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | +| `matrix.Det(a)` | `matrix.Det(a)` | `np.linalg.det(a)` | ? | determinant of `a` | | | |`np.eye(3)` | `eye(3)` | 3x3 identity matrix | | | |`np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D tensor, `a` | | | |`np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index a79e6ba8b7..21b497af0c 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -656,6 +656,7 @@ var numpyProps = map[string]funWrap{ "len": {"Len()", "nis"}, "size": {"Len()", "nis"}, "shape": {"Shape().Sizes", "niv"}, + "T": {"", "tensor.Transpose"}, } // tensorFunc outputs the wrapping function and whether it needs ellipsis @@ -678,6 +679,8 @@ func (fw *funWrap) wrapFunc(mp *mathParse) bool { case "nsv": wrapFun = "tensor.NewStringFromValues" ellip = true + default: + wrapFun = fw.wrap } mp.startFunc(wrapFun, true) // don't lookup -- don't auto-convert args mp.out.Add(token.LPAREN) @@ -695,9 +698,13 @@ func (mp *mathParse) selectorExpr(ex *ast.SelectorExpr) { } ellip := fw.wrapFunc(mp) mp.expr(ex.X) - mp.addToken(token.PERIOD) - mp.out.Add(token.IDENT, fw.fun) - mp.idx++ + if fw.fun != "" { + mp.addToken(token.PERIOD) + mp.out.Add(token.IDENT, fw.fun) + mp.idx++ + } else { + mp.idx += 2 + } if ellip { mp.out.Add(token.ELLIPSIS) } @@ -807,18 +814,19 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { // nofun = do not accept a function version, just a method var numpyFuncs = map[string]funWrap{ // "array": {"tensor.NewFloatFromValues", ""}, // todo: probably not right, maybe don't have? - "zeros": {"tensor.NewFloat64", ""}, - "full": {"tensor.NewFloat64Full", ""}, - "ones": {"tensor.NewFloat64Ones", ""}, - "rand": {"tensor.NewFloat64Rand", ""}, - "arange": {"tensor.NewIntRange", ""}, - "linspace": {"tensor.NewFloat64SpacedLinear", ""}, - "reshape": {"tensor.Reshape", ""}, - "copy": {"tensor.Clone", ""}, - "get": {"datafs.Get", ""}, - "set": {"datafs.Set", ""}, - "flatten": {"tensor.Flatten", "nofun"}, - "squeeze": {"tensor.Squeeze", "nofun"}, + "zeros": {"tensor.NewFloat64", ""}, + "full": {"tensor.NewFloat64Full", ""}, + "ones": {"tensor.NewFloat64Ones", ""}, + "rand": {"tensor.NewFloat64Rand", ""}, + "arange": {"tensor.NewIntRange", ""}, + "linspace": {"tensor.NewFloat64SpacedLinear", ""}, + "transpose": {"tensor.Transpose", ""}, + "reshape": {"tensor.Reshape", ""}, + "copy": {"tensor.Clone", ""}, + "get": {"datafs.Get", ""}, + "set": {"datafs.Set", ""}, + "flatten": {"tensor.Flatten", "nofun"}, + "squeeze": {"tensor.Squeeze", "nofun"}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index b612729be8..8fed6d8c0d 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x @ a", `matrix.Mul(x, a)`}, + {"# x.T", `tensor.Transpose(x)`}, } st := NewState() st.MathRecord = false @@ -223,12 +223,14 @@ func TestMath(t *testing.T) { {"# a = x + y", `a = tmath.Add(x, y)`}, {"# a := x ** 2", `a := tensor.Tensor(tmath.Pow(x, tensor.NewIntScalar(2)))`}, {"# a = -x", `a = tmath.Negate(x)`}, + {"# x @ a", `matrix.Mul(x, a)`}, {"# x += 1", `tmath.AddAssign(x, tensor.NewIntScalar(1))`}, {"# a := [1,2,3,4]", `a := tensor.Tensor(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...))`}, {"# a := [1.,2,3,4]", `a := tensor.Tensor(tensor.NewFloat64FromValues([]float64 { 1., 2, 3, 4 } ...))`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, {"# x.shape", `tensor.NewIntFromValues(x.Shape().Sizes ...)`}, + {"# x.T", `tensor.Transpose(x)`}, {"# zeros(3, 4)", `tensor.NewFloat64(3, 4)`}, {"# full(5.5, 3, 4)", `tensor.NewFloat64Full(5.5, 3, 4)`}, {"# zeros(sh)", `tensor.NewFloat64(tensor.AsIntSlice(sh) ...)`}, diff --git a/tensor/matrix/matrix_test.go b/tensor/matrix/matrix_test.go index 1ac8477452..8846454204 100644 --- a/tensor/matrix/matrix_test.go +++ b/tensor/matrix/matrix_test.go @@ -19,19 +19,13 @@ func TestMatrix(t *testing.T) { _ = v o := Mul(a, a).(*tensor.Float64) - // fmt.Println(o) - assert.Equal(t, []float64{7, 10, 15, 22}, o.Values) o = Mul(a, v).(*tensor.Float64) - // fmt.Println(o) - assert.Equal(t, []float64{8, 18}, o.Values) assert.Equal(t, []int{2, 1}, o.Shape().Sizes) o = Mul(v, a).(*tensor.Float64) - // fmt.Println(o) - assert.Equal(t, []float64{11, 16}, o.Values) assert.Equal(t, []int{1, 2}, o.Shape().Sizes) @@ -43,28 +37,28 @@ func TestMatrix(t *testing.T) { // fmt.Println(b) o = Mul(b, a).(*tensor.Float64) - // fmt.Println(o) assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) o = Mul(a, b).(*tensor.Float64) - // fmt.Println(o) assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) o = Mul(b, b).(*tensor.Float64) - // fmt.Println(o) assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) o = Mul(v, b).(*tensor.Float64) - // fmt.Println(o) assert.Equal(t, []float64{11, 16, 11, 16, 11, 16}, o.Values) assert.Equal(t, []int{3, 1, 2}, o.Shape().Sizes) o = Mul(b, v).(*tensor.Float64) - // fmt.Println(o) assert.Equal(t, []float64{8, 18, 8, 18, 8, 18}, o.Values) assert.Equal(t, []int{3, 2, 1}, o.Shape().Sizes) + o = Mul(a, tensor.Transpose(a)).(*tensor.Float64) + assert.Equal(t, []float64{5, 11, 11, 25}, o.Values) + + d := Det(a) + assert.Equal(t, -2.0, d.Float1D(0)) } diff --git a/tensor/matrix/ops.go b/tensor/matrix/ops.go index 54eedfb37e..6f709072b7 100644 --- a/tensor/matrix/ops.go +++ b/tensor/matrix/ops.go @@ -7,6 +7,7 @@ package matrix import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" + "gonum.org/v1/gonum/mat" ) // CallOut2 calls an Out function with 2 input args. All matrix functions @@ -123,3 +124,28 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { } return nil } + +// Det returns the determinant of the given tensor. +// For a 2D matrix [[a, b], [c, d]] it this is ad - bc. +// See also [LogDet] for a version that is more numerically +// stable for large matricies. +func Det(a tensor.Tensor) tensor.Tensor { + m, err := NewMatrix(a) + if errors.Log(err) != nil { + return tensor.NewFloat64Scalar(0) + } + return tensor.NewFloat64Scalar(mat.Det(m)) +} + +// LogDet returns the determinant of the given tensor, +// as the log and sign of the value, which is more +// numerically stable. The return is a 1D vector of length 2, +// with the first value being the log, and the second the sign. +func LogDet(a tensor.Tensor) tensor.Tensor { + m, err := NewMatrix(a) + if errors.Log(err) != nil { + return tensor.NewFloat64Scalar(0) + } + l, s := mat.LogDet(m) + return tensor.NewFloat64FromValues(l, s) +} diff --git a/tensor/reshaped.go b/tensor/reshaped.go index da3d924f47..4a88464da1 100644 --- a/tensor/reshaped.go +++ b/tensor/reshaped.go @@ -65,6 +65,16 @@ func Reshape(tsr Tensor, sizes ...int) Tensor { return rs } +// Transpose returns a new [Reshaped] tensor with the strides +// switched so that rows and column dimensions are effectively +// reversed. +func Transpose(tsr Tensor) Tensor { + rs := &Reshaped{Tensor: tsr} + rs.Reshape.CopyFrom(tsr.Shape()) + rs.Reshape.Strides = ColumnMajorStrides(rs.Reshape.Sizes...) + return rs +} + // NewRowCellsView returns a 2D [Reshaped] view onto the given tensor, // with a single outer "row" dimension and a single inner "cells" dimension, // with the given 'split' dimension specifying where the cells start. diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index ecd2a1212e..8bee9c7598 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -277,6 +277,17 @@ func TestReshaped(t *testing.T) { err := rs.SetShapeSizes(5, -1) assert.Error(t, err) + + res = `[3, 4] + [0]: [3]: [2]: [1]: +[0]: 0 3 12 21 +[0]: 1 10 13 22 +[0]: 2 11 20 23 +` + tr := Transpose(ft) + // fmt.Println(tr) + assert.Equal(t, res, tr.String()) + } func TestSortFilter(t *testing.T) { diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go b/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go new file mode 100644 index 0000000000..7124f52e29 --- /dev/null +++ b/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go @@ -0,0 +1,28 @@ +// Code generated by 'yaegi extract cogentcore.org/core/tensor/matrix'. DO NOT EDIT. + +package symbols + +import ( + "cogentcore.org/core/tensor/matrix" + "reflect" +) + +func init() { + Symbols["cogentcore.org/core/tensor/matrix/matrix"] = map[string]reflect.Value{ + // function, constant and variable definitions + "CallOut2": reflect.ValueOf(matrix.CallOut2), + "CopyFromDense": reflect.ValueOf(matrix.CopyFromDense), + "Det": reflect.ValueOf(matrix.Det), + "LogDet": reflect.ValueOf(matrix.LogDet), + "Mul": reflect.ValueOf(matrix.Mul), + "MulOut": reflect.ValueOf(matrix.MulOut), + "NewDense": reflect.ValueOf(matrix.NewDense), + "NewMatrix": reflect.ValueOf(matrix.NewMatrix), + "NewSymmetric": reflect.ValueOf(matrix.NewSymmetric), + "StringCheck": reflect.ValueOf(matrix.StringCheck), + + // type definitions + "Matrix": reflect.ValueOf((*matrix.Matrix)(nil)), + "Symmetric": reflect.ValueOf((*matrix.Symmetric)(nil)), + } +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 8302eac053..4d73caf4b2 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -153,6 +153,7 @@ func init() { "StringToFloat64": reflect.ValueOf(tensor.StringToFloat64), "Tab": reflect.ValueOf(tensor.Tab), "ThreadingThreshold": reflect.ValueOf(&tensor.ThreadingThreshold).Elem(), + "Transpose": reflect.ValueOf(tensor.Transpose), "UnstableSort": reflect.ValueOf(tensor.UnstableSort), "Vectorize": reflect.ValueOf(tensor.Vectorize), "VectorizeOnThreads": reflect.ValueOf(tensor.VectorizeOnThreads), From e3242bf69dd2b3dbd3d32bbd1b087af6be85d713 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 03:21:26 -0700 Subject: [PATCH 177/311] multidimensional array literals processed correctly --- goal/transpile/math.go | 32 ++++++++++++++++++++++++++++++++ goal/transpile/transpile_test.go | 3 ++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/goal/transpile/math.go b/goal/transpile/math.go index 21b497af0c..c909bb71ba 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -801,6 +801,12 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { mp.idx++ // closing brace we're not using return } + var sh []int + mp.arrayShape(il.Indices, &sh) + if len(sh) > 1 { + mp.startFunc("tensor.Reshape") + mp.out.Add(token.LPAREN) + } mp.startFunc("tensor.New" + fun + "FromValues") mp.out.Add(token.LPAREN) mp.out.Add(token.IDENT, "[]"+typ) @@ -809,6 +815,32 @@ func (mp *mathParse) arrayLiteral(il *ast.IndexListExpr) { mp.addToken(token.RBRACE) mp.out.AddMulti(token.ELLIPSIS, token.RPAREN) mp.endFunc() + if len(sh) > 1 { + mp.out.Add(token.COMMA) + nsh := len(sh) + for i, s := range sh { + mp.out.Add(token.INT, fmt.Sprintf("%d", s)) + if i < nsh-1 { + mp.out.Add(token.COMMA) + } + } + mp.out.Add(token.RPAREN) + mp.endFunc() + } +} + +func (mp *mathParse) arrayShape(ex []ast.Expr, sh *[]int) { + n := len(ex) + if n == 0 { + return + } + *sh = append(*sh, n) + for i := range n { + if il, ok := ex[i].(*ast.IndexListExpr); ok { + mp.arrayShape(il.Indices, sh) + return + } + } } // nofun = do not accept a function version, just a method diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 8fed6d8c0d..eca158db40 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,8 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# x.T", `tensor.Transpose(x)`}, + {"# a := [1,2,3,4]", `a := tensor.Tensor(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...))`}, + {"# a := [[1,2], [3,4]]", `a := tensor.Tensor(tensor.Reshape(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...), 2, 2))`}, } st := NewState() st.MathRecord = false From aac3037b3dcf981947ff9d4974e8deb14cc9abbb Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 03:26:51 -0700 Subject: [PATCH 178/311] readme update --- goal/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/goal/README.md b/goal/README.md index 5e5c40c619..8061119a04 100644 --- a/goal/README.md +++ b/goal/README.md @@ -264,9 +264,9 @@ The following sections provide a full list of equivalents between the `tensor` G | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | -| `tensor.NewFloat64FromValues(` `[]float64{1, 2, 3})` | `[1., 2., 3.]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | -| TODO: (does reshape of flat literal) | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | -| TODO: | | `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | +| `tensor.NewFloat64FromValues(` `[]float64{1, 2, 3})` | `[1., 2., 3.]` | `np.array([1., 2., 3.])` | `[ 1 2 3 ]` | define a 1D tensor | +| | `[[1., 2., 3.], [4., 5., 6.]]` or: | `(np.array([[1., 2., 3.], [4., 5., 6.]])` | `[ 1 2 3; 4 5 6 ]` | define a 2x3 2D tensor | +| | | `[[a, b], [c, d]]` or `block([[a, b], [c, d]])` | `np.block([[a, b], [c, d]])` | `[ a b; c d ]` | construct a matrix from blocks `a`, `b`, `c`, and `d` | | `tensor.NewFloat64(3,4)` | `zeros(3,4)` | `np.zeros((3, 4))` | `zeros(3,4)` | 3x4 2D tensor of float64 zeros; Goal does not use "tuple" so no double parens | | `tensor.NewFloat64(3,4,5)` | `zeros(3, 4, 5)` | `np.zeros((3, 4, 5))` | `zeros(3,4,5)` | 3x4x5 three-dimensional tensor of float64 zeros | | `tensor.NewFloat64Ones(3,4)` | `ones(3, 4)` | `np.ones((3, 4))` | `ones(3,4)` | 3x4 2D tensor of 64-bit floating point ones | From bb9ec09e74917ad2b58cbbfb8de290fb9c3dd446 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 10:20:52 -0700 Subject: [PATCH 179/311] collapse extra dimension for vector @ matrix --- tensor/matrix/matrix_test.go | 8 ++++---- tensor/matrix/ops.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/tensor/matrix/matrix_test.go b/tensor/matrix/matrix_test.go index 8846454204..c97523a8f5 100644 --- a/tensor/matrix/matrix_test.go +++ b/tensor/matrix/matrix_test.go @@ -23,11 +23,11 @@ func TestMatrix(t *testing.T) { o = Mul(a, v).(*tensor.Float64) assert.Equal(t, []float64{8, 18}, o.Values) - assert.Equal(t, []int{2, 1}, o.Shape().Sizes) + assert.Equal(t, []int{2}, o.Shape().Sizes) o = Mul(v, a).(*tensor.Float64) assert.Equal(t, []float64{11, 16}, o.Values) - assert.Equal(t, []int{1, 2}, o.Shape().Sizes) + assert.Equal(t, []int{2}, o.Shape().Sizes) nr := 3 b := tensor.NewFloat64(nr, 2, 2) @@ -50,11 +50,11 @@ func TestMatrix(t *testing.T) { o = Mul(v, b).(*tensor.Float64) assert.Equal(t, []float64{11, 16, 11, 16, 11, 16}, o.Values) - assert.Equal(t, []int{3, 1, 2}, o.Shape().Sizes) + assert.Equal(t, []int{3, 2}, o.Shape().Sizes) o = Mul(b, v).(*tensor.Float64) assert.Equal(t, []float64{8, 18, 8, 18, 8, 18}, o.Values) - assert.Equal(t, []int{3, 2, 1}, o.Shape().Sizes) + assert.Equal(t, []int{3, 2}, o.Shape().Sizes) o = Mul(a, tensor.Transpose(a)).(*tensor.Float64) assert.Equal(t, []float64{5, 11, 11, 25}, o.Values) diff --git a/tensor/matrix/ops.go b/tensor/matrix/ops.go index 6f709072b7..3a7e5dfce0 100644 --- a/tensor/matrix/ops.go +++ b/tensor/matrix/ops.go @@ -5,6 +5,8 @@ package matrix import ( + "slices" + "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" "gonum.org/v1/gonum/mat" @@ -56,12 +58,18 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { nb := b.NumDims() ea := a eb := b + collapse := false + colDim := 0 if na == 1 { ea = tensor.Reshape(a, 1, a.DimSize(0)) + collapse = true + colDim = -2 na = 2 } if nb == 1 { eb = tensor.Reshape(b, b.DimSize(0), 1) + collapse = true + colDim = -1 nb = 2 } if na > 2 { @@ -122,6 +130,15 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { default: return errors.New("matrix.Mul: input dimensions do not align") } + if collapse { + nd := out.NumDims() + sz := slices.Clone(out.Shape().Sizes) + if colDim == -1 { + out.SetShapeSizes(sz[:nd-1]...) + } else { + out.SetShapeSizes(append(sz[:nd-2], sz[nd-1])...) + } + } return nil } From b7427a3d8dbdfc55517dd59db01bf2c7a013e273 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 16:49:29 -0700 Subject: [PATCH 180/311] matrix: annoying Tri indexing functions all working -- hard to get the N equation right for all the factors but it seems to be now.. --- tensor/create_test.go | 43 ++++++ tensor/funcs_test.go | 21 --- tensor/matrix/indices.go | 241 ++++++++++++++++++++++++++++++++++ tensor/matrix/indices_test.go | 80 +++++++++++ tensor/stats/metric/matrix.go | 39 ------ 5 files changed, 364 insertions(+), 60 deletions(-) create mode 100644 tensor/create_test.go create mode 100644 tensor/matrix/indices.go create mode 100644 tensor/matrix/indices_test.go diff --git a/tensor/create_test.go b/tensor/create_test.go new file mode 100644 index 0000000000..be7927a543 --- /dev/null +++ b/tensor/create_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tensor + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreate(t *testing.T) { + assert.Equal(t, 5.5, NewFloat64Scalar(5.5).Float1D(0)) + assert.Equal(t, 5, NewIntScalar(5).Int1D(0)) + assert.Equal(t, "test", NewStringScalar("test").String1D(0)) + + assert.Equal(t, []float64{5.5, 1.5}, NewFloat64FromValues(5.5, 1.5).Values) + assert.Equal(t, []int{5, 1}, NewIntFromValues(5, 1).Values) + assert.Equal(t, []string{"test1", "test2"}, NewStringFromValues("test1", "test2").Values) + + assert.Equal(t, []float64{5.5, 5.5, 5.5, 5.5}, NewFloat64Full(5.5, 2, 2).Values) + assert.Equal(t, []float64{1, 1, 1, 1}, NewFloat64Ones(2, 2).Values) + + ar := NewIntRange(5) + assert.Equal(t, []int{0, 1, 2, 3, 4}, AsIntSlice(ar)) + + ar = NewIntRange(2, 5) + assert.Equal(t, []int{2, 3, 4}, AsIntSlice(ar)) + + ar = NewIntRange(0, 5, 2) + assert.Equal(t, []int{0, 2, 4}, AsIntSlice(ar)) + + lr := NewFloat64SpacedLinear(NewFloat64Scalar(0), NewFloat64Scalar(5), 6, true) + assert.Equal(t, []float64{0, 1, 2, 3, 4, 5}, AsFloat64Slice(lr)) + + lr = NewFloat64SpacedLinear(NewFloat64Scalar(0), NewFloat64Scalar(5), 5, false) + assert.Equal(t, []float64{0, 1, 2, 3, 4}, AsFloat64Slice(lr)) + + lr2 := NewFloat64SpacedLinear(NewFloat64FromValues(0, 2), NewFloat64FromValues(5, 7), 5, false) + // fmt.Println(lr2) + assert.Equal(t, []float64{0, 2, 1, 3, 2, 4, 3, 5, 4, 6}, AsFloat64Slice(lr2)) +} diff --git a/tensor/funcs_test.go b/tensor/funcs_test.go index 358c7e4dd0..e2465ab132 100644 --- a/tensor/funcs_test.go +++ b/tensor/funcs_test.go @@ -69,24 +69,3 @@ func TestAlign(t *testing.T) { assert.Equal(t, []int{3, 1, 4}, as.Sizes) assert.Equal(t, []int{1, 3, 4}, bs.Sizes) } - -func TestCreate(t *testing.T) { - ar := NewIntRange(5) - assert.Equal(t, []int{0, 1, 2, 3, 4}, AsIntSlice(ar)) - - ar = NewIntRange(2, 5) - assert.Equal(t, []int{2, 3, 4}, AsIntSlice(ar)) - - ar = NewIntRange(0, 5, 2) - assert.Equal(t, []int{0, 2, 4}, AsIntSlice(ar)) - - lr := NewFloat64SpacedLinear(NewFloat64Scalar(0), NewFloat64Scalar(5), 6, true) - assert.Equal(t, []float64{0, 1, 2, 3, 4, 5}, AsFloat64Slice(lr)) - - lr = NewFloat64SpacedLinear(NewFloat64Scalar(0), NewFloat64Scalar(5), 5, false) - assert.Equal(t, []float64{0, 1, 2, 3, 4}, AsFloat64Slice(lr)) - - lr2 := NewFloat64SpacedLinear(NewFloat64FromValues(0, 2), NewFloat64FromValues(5, 7), 5, false) - // fmt.Println(lr2) - assert.Equal(t, []float64{0, 2, 1, 3, 2, 4, 3, 5, 4, 6}, AsFloat64Slice(lr2)) -} diff --git a/tensor/matrix/indices.go b/tensor/matrix/indices.go new file mode 100644 index 0000000000..b70e7d6200 --- /dev/null +++ b/tensor/matrix/indices.go @@ -0,0 +1,241 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" +) + +// offCols is a helper function to process the optional offset_cols args +func offCols(size int, offset_cols ...int) (off, cols int) { + off = 0 + cols = size + if len(offset_cols) >= 1 { + off = offset_cols[0] + } + if len(offset_cols) == 2 { + cols = offset_cols[1] + } + return +} + +// Eye returns a new 2D Float64 tensor with 1s along the diagonal and +// 0s elsewhere, with the given row and column size. +// - If one additional parameter is passed, it is the offset, +// to set values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func Eye(size int, offset_cols ...int) *tensor.Float64 { + off, cols := offCols(size, offset_cols...) + tsr := tensor.NewFloat64(size, cols) + for r := range size { + c := r + off + if c < 0 || c >= cols { + continue + } + tsr.SetFloat(1, r, c) + } + return tsr +} + +// Tri returns a new 2D Float64 tensor with 1s along the diagonal and +// below it, and 0s elsewhere (i.e., a filled lower triangle). +// - If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func Tri(size int, offset_cols ...int) *tensor.Float64 { + off, cols := offCols(size, offset_cols...) + tsr := tensor.NewFloat64(size, cols) + for r := range size { + for c := range cols { + if c <= r+off { + tsr.SetFloat(1, r, c) + } + } + } + return tsr +} + +// TriUpper returns a new 2D Float64 tensor with 1s along the diagonal and +// above it, and 0s elsewhere (i.e., a filled upper triangle). +// - If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func TriUpper(size int, offset_cols ...int) *tensor.Float64 { + off, cols := offCols(size, offset_cols...) + tsr := tensor.NewFloat64(size, cols) + for r := range size { + for c := range cols { + if c >= r+off { + tsr.SetFloat(1, r, c) + } + } + } + return tsr +} + +// TriUN returns the number of elements in the upper triangular region +// of a 2D matrix of given row and column size, where the triangle includes the +// elements along the diagonal. +// - If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func TriUN(size int, offset_cols ...int) int { + off, cols := offCols(size, offset_cols...) + rows := size + if off > 0 { + if cols > rows { + return TriUN(rows, 0, cols-off) + } else { + return TriUN(rows-off, 0, cols-off) + } + } else if off < 0 { // invert + return cols*rows - TriUN(cols, -(off-1), rows) + } + if cols <= size { + return cols + (cols*(cols-1))/2 + } + return rows + (rows*(2*cols-rows-1))/2 +} + +// TriLN returns the number of elements in the lower triangular region +// of a 2D matrix of given row and column size, where the triangle includes the +// elements along the diagonal. +// - If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func TriLN(size int, offset_cols ...int) int { + off, cols := offCols(size, offset_cols...) + return TriUN(cols, -off, size) +} + +// TriLIndicies returns the list of r, c indexes for the lower triangular +// portion of a square matrix of size n, including the diagonal. +// The result is a 2D list of indices, where the outer (row) dimension +// is the number of indices, and the inner dimension is 2 for the r, c coords. +// - If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix. +func TriLIndicies(size int, offset_cols ...int) *tensor.Int { + off, cols := offCols(size, offset_cols...) + trin := TriLN(size, off, cols) + coords := tensor.NewInt(trin, 2) + i := 0 + for r := range size { + for c := range cols { + if c <= r+off { + coords.SetInt(r, i, 0) + coords.SetInt(c, i, 1) + i++ + } + } + } + return coords +} + +// TriUIndicies returns the list of r, c indexes for the upper triangular +// portion of a square matrix of size n, including the diagonal. +// If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// If a second additional parameter is passed, it is the number of columns +// for a non-square matrix. +// The result is a 2D list of indices, where the outer (row) dimension +// is the number of indices, and the inner dimension is 2 for the r, c coords. +func TriUIndicies(size int, offset_cols ...int) *tensor.Int { + off, cols := offCols(size, offset_cols...) + trin := TriUN(size, off, cols) + coords := tensor.NewInt(trin, 2) + i := 0 + for r := range size { + for c := range cols { + if c >= r+off { + coords.SetInt(r, i, 0) + coords.SetInt(c, i, 1) + i++ + } + } + } + return coords +} + +// TriLView returns an [Indexed] view of the given tensor for the lower triangular +// region of values, as a 1D list. An error is logged if the tensor is not 2D. +// Use the optional offset parameter to get values above (positive) or +// below (negative) the diagonal. +func TriLView(tsr tensor.Tensor, offset ...int) *tensor.Indexed { + if tsr.NumDims() != 2 { + errors.Log(errors.New("matrix.TriLView requires a 2D tensor")) + return nil + } + off := 0 + if len(offset) == 1 { + off = offset[0] + } + return tensor.NewIndexed(tsr, TriLIndicies(tsr.DimSize(0), off, tsr.DimSize(1))) +} + +// TriUView returns an [Indexed] view of the given tensor for the upper triangular +// region of values, as a 1D list. An error is logged if the tensor is not 2D. +// Use the optional offset parameter to get values above (positive) or +// below (negative) the diagonal. +func TriUView(tsr tensor.Tensor, offset ...int) *tensor.Indexed { + if tsr.NumDims() != 2 { + errors.Log(errors.New("matrix.TriUView requires a 2D tensor")) + return nil + } + off := 0 + if len(offset) == 1 { + off = offset[0] + } + return tensor.NewIndexed(tsr, TriUIndicies(tsr.DimSize(0), off, tsr.DimSize(1))) +} + +// TriL returns a copy of the given tensor containing the lower triangular +// region of values (including the diagonal), with the lower triangular region +// zeroed. An error is logged if the tensor is not 2D. +// Use the optional offset parameter to include values above (positive) or +// below (negative) the diagonal. +func TriL(tsr tensor.Tensor, offset ...int) tensor.Tensor { + if tsr.NumDims() != 2 { + errors.Log(errors.New("matrix.TriL requires a 2D tensor")) + return nil + } + off := 0 + if len(offset) == 1 { + off = offset[0] + } + off += 1 + tc := tensor.Clone(tsr) + tv := TriUView(tc, off) // opposite + tensor.SetAllFloat64(tv, 0) + return tc +} + +// TriU returns a copy of the given tensor containing the upper triangular +// region of values (including the diagonal), with the lower triangular region +// zeroed. An error is logged if the tensor is not 2D. +// Use the optional offset parameter to include values above (positive) or +// below (negative) the diagonal. +func TriU(tsr tensor.Tensor, offset ...int) tensor.Tensor { + if tsr.NumDims() != 2 { + errors.Log(errors.New("matrix.TriU requires a 2D tensor")) + return nil + } + off := 0 + if len(offset) == 1 { + off = offset[0] + } + off -= 1 + tc := tensor.Clone(tsr) + tv := TriLView(tc, off) // opposite + tensor.SetAllFloat64(tv, 0) + return tc +} diff --git a/tensor/matrix/indices_test.go b/tensor/matrix/indices_test.go new file mode 100644 index 0000000000..e02334d273 --- /dev/null +++ b/tensor/matrix/indices_test.go @@ -0,0 +1,80 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "testing" + + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/vector" + "github.com/stretchr/testify/assert" +) + +func TestIndices(t *testing.T) { + assert.Equal(t, []float64{1, 0, 0, 1}, Eye(2).Values) + assert.Equal(t, []float64{1, 0, 0, 0, 1, 0, 0, 0, 1}, Eye(3).Values) + assert.Equal(t, []float64{0, 0, 0, 1, 0, 0, 0, 1, 0}, Eye(3, -1).Values) + + assert.Equal(t, []float64{1, 0, 0, 1, 1, 0, 1, 1, 1}, Tri(3).Values) + assert.Equal(t, []float64{1, 1, 0, 1, 1, 1, 1, 1, 1}, Tri(3, 1).Values) + assert.Equal(t, []float64{0, 0, 0, 1, 0, 0, 1, 1, 0}, Tri(3, -1).Values) + + assert.Equal(t, []float64{0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0}, Tri(3, -1, 4).Values) + assert.Equal(t, []float64{0, 0, 1, 0, 1, 1}, Tri(3, -1, 2).Values) + + assert.Equal(t, []float64{1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1}, Tri(3, 1, 4).Values) + assert.Equal(t, []float64{1, 1, 1, 1, 1, 1}, Tri(3, 1, 2).Values) + + assert.Equal(t, int(vector.Sum(TriUpper(3)).Float1D(0)), TriUN(3)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 0, 4)).Float1D(0)), TriUN(3, 0, 4)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 0, 2)).Float1D(0)), TriUN(3, 0, 2)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 1)).Float1D(0)), TriUN(3, 1)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 1, 4)).Float1D(0)), TriUN(3, 1, 4)) + assert.Equal(t, int(vector.Sum(TriUpper(10, 4, 7)).Float1D(0)), TriUN(10, 4, 7)) + assert.Equal(t, int(vector.Sum(TriUpper(10, 4, 12)).Float1D(0)), TriUN(10, 4, 12)) + assert.Equal(t, int(vector.Sum(TriUpper(3, -1)).Float1D(0)), TriUN(3, -1)) + assert.Equal(t, int(vector.Sum(TriUpper(3, -1, 4)).Float1D(0)), TriUN(3, -1, 4)) + assert.Equal(t, int(vector.Sum(TriUpper(3, -1, 2)).Float1D(0)), TriUN(3, -1, 2)) + assert.Equal(t, int(vector.Sum(TriUpper(10, -4, 7)).Float1D(0)), TriUN(10, -4, 7)) + assert.Equal(t, int(vector.Sum(TriUpper(10, -4, 12)).Float1D(0)), TriUN(10, -4, 12)) + + assert.Equal(t, int(vector.Sum(Tri(3)).Float1D(0)), TriLN(3)) + assert.Equal(t, int(vector.Sum(Tri(3, 0, 4)).Float1D(0)), TriLN(3, 0, 4)) + assert.Equal(t, int(vector.Sum(Tri(3, 0, 2)).Float1D(0)), TriLN(3, 0, 2)) + assert.Equal(t, int(vector.Sum(Tri(3, 1)).Float1D(0)), TriLN(3, 1)) + assert.Equal(t, int(vector.Sum(Tri(3, 1, 4)).Float1D(0)), TriLN(3, 1, 4)) + assert.Equal(t, int(vector.Sum(Tri(10, 4, 7)).Float1D(0)), TriLN(10, 4, 7)) + assert.Equal(t, int(vector.Sum(Tri(10, 4, 12)).Float1D(0)), TriLN(10, 4, 12)) + assert.Equal(t, int(vector.Sum(Tri(3, -1)).Float1D(0)), TriLN(3, -1)) + assert.Equal(t, int(vector.Sum(Tri(3, -1, 4)).Float1D(0)), TriLN(3, -1, 4)) + assert.Equal(t, int(vector.Sum(Tri(3, -1, 2)).Float1D(0)), TriLN(3, -1, 2)) + assert.Equal(t, int(vector.Sum(Tri(10, -4, 7)).Float1D(0)), TriLN(10, -4, 7)) + assert.Equal(t, int(vector.Sum(Tri(10, -4, 12)).Float1D(0)), TriLN(10, -4, 12)) + + tli := TriLIndicies(3) + assert.Equal(t, []int{0, 0, 1, 0, 1, 1, 2, 0, 2, 1, 2, 2}, tli.Values) + + tli = TriLIndicies(3, -1) + assert.Equal(t, []int{1, 0, 2, 0, 2, 1}, tli.Values) + + tli = TriLIndicies(3, 1) + assert.Equal(t, []int{0, 0, 0, 1, 1, 0, 1, 1, 1, 2, 2, 0, 2, 1, 2, 2}, tli.Values) + + tli = TriUIndicies(3, 1) + assert.Equal(t, []int{0, 1, 0, 2, 1, 2}, tli.Values) + + tli = TriUIndicies(3, -1) + assert.Equal(t, []int{0, 0, 0, 1, 0, 2, 1, 0, 1, 1, 1, 2, 2, 1, 2, 2}, tli.Values) + + tf := tensor.NewFloat64Ones(3, 4) + + assert.Equal(t, Tri(3, -1, 4).Values, TriL(tf, -1).(*tensor.Float64).Values) + assert.Equal(t, Tri(3, 0, 4).Values, TriL(tf).(*tensor.Float64).Values) + assert.Equal(t, Tri(3, 1, 4).Values, TriL(tf, 1).(*tensor.Float64).Values) + + assert.Equal(t, TriUpper(3, -1, 4).Values, TriU(tf, -1).(*tensor.Float64).Values) + assert.Equal(t, TriUpper(3, 0, 4).Values, TriU(tf).(*tensor.Float64).Values) + assert.Equal(t, TriUpper(3, 1, 4).Values, TriU(tf, 1).(*tensor.Float64).Values) +} diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index c3f8718c0b..519a5f9f8a 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -296,42 +296,3 @@ func ProjectOnMatrixColumn(mtx, vec, colindex tensor.Tensor) tensor.Values { errors.Log(ProjectOnMatrixColumnOut(mtx, vec, colindex, out)) return out } - -//////////////////////////////////////////// -// Triangular square matrix functions - -// TODO: move these somewhere more appropriate - -// note: this answer gives an index into the upper triangular -// https://math.stackexchange.com/questions/2134011/conversion-of-upper-triangle-linear-index-from-index-on-symmetrical-array -// return TriangularN(n) - ((n-c)*(n-c-1))/2 + r - c - 1 <- this works for lower excluding diag -// return (n * (n - 1) / 2) - ((n-r)*(n-r-1))/2 + c <- this works for upper including diag -// but I wasn't able to get an equation for r, c back from index, for this "including diagonal" -// https://stackoverflow.com/questions/27086195/linear-index-upper-triangular-matrix?rq=3 -// python just iterates manually and returns a list -// https://github.com/numpy/numpy/blob/v2.1.0/numpy/lib/_twodim_base_impl.py#L902-L985 - -// TriangularN returns the number of elements in the triangular region -// of a square matrix of given size, where the triangle includes the -// n elements along the diagonal. -func TriangularN(n int) int { - return n + (n*(n-1))/2 -} - -// TriangularLIndicies returns the list of r, c indexes (as X, Y coordinates) -// for the lower triangular portion of a square matrix of size n, -// including the diagonal. -func TriangularLIndicies(n int) []vecint.Vector2i { - trin := TriangularN(n) - coords := make([]vecint.Vector2i, trin) - i := 0 - for r := range n { - for c := range n { - if c <= r { - coords[i] = vecint.Vector2i{X: r, Y: c} - i++ - } - } - } - return coords -} From 1862a6c82a22d7b7b751493823841948f7b2b529 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 1 Oct 2024 22:02:27 -0700 Subject: [PATCH 181/311] all the triangle, diagonal, trace functions in matrix --- goal/README.md | 10 +- goal/transpile/math.go | 25 ++--- goal/transpile/transpile_test.go | 2 +- tensor/matrix/indices.go | 105 ++++++++++++++++-- tensor/matrix/indices_test.go | 81 +++++++++----- tensor/stats/metric/matrix.go | 50 +++++---- .../cogentcore_org-core-tensor-matrix.go | 35 ++++-- ...cogentcore_org-core-tensor-stats-metric.go | 2 - 8 files changed, 219 insertions(+), 91 deletions(-) diff --git a/goal/README.md b/goal/README.md index 8061119a04..2ca0f9b783 100644 --- a/goal/README.md +++ b/goal/README.md @@ -364,12 +364,16 @@ In Goal and NumPy, the standard `+, -, *, /` operators perform _element-wise_ op | `tensor` Go | Goal | NumPy | MATLAB | Notes | | ------------ | ----------- | ------ | ------ | ---------------- | | `matrix.Mul(a,b)` | same: |`a @ b` | `a * b` | matrix multiply | -| `tensor.Transpose(a)` | same: |`a.transpose() or a.T` | `a.'` | transpose of `a` | +| `tensor.Transpose(a)` | <- or `a.T` |`a.transpose()` or `a.T` | `a.'` | transpose of `a` | | TODO: | |`a.conj().transpose() or a.conj().T` | `a'` | conjugate transpose of `a` | | `matrix.Det(a)` | `matrix.Det(a)` | `np.linalg.det(a)` | ? | determinant of `a` | -| | |`np.eye(3)` | `eye(3)` | 3x3 identity matrix | -| | |`np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D tensor, `a` | +| `matrix.Identity(3)` | <- |`np.eye(3)` | `eye(3)` | 3x3 identity matrix | +| `matrix.Diagonal(a)` | <- |`np.diag(a)` | `diag(a)` | returns a vector of the diagonal elements of 2D tensor, `a`. Goal returns a read / write view. | | | |`np.diag(v, 0)` | `diag(v,0)` | returns a square diagonal matrix whose nonzero values are the elements of vector, v | +| `matrix.Trace(a)` | <- |`np.trace(a)` | `trace(a)` | returns the sum of the elements along the diagonal of `a`. | +| `matrix.Tri()` | <- |`np.tri()` | `tri()` | returns a new 2D Float64 matrix with 1s in the lower triangular region (including the diagonal) and the remaining upper triangular elements zero | +| `matrix.TriL(a)` | <- |`np.tril(a)` | `tril(a)` | returns a copy of `a` with the lower triangular elements (including the diagonal) from `a` and the remaining upper triangular elements zeroed out | +| `matrix.TriU(a)` | <- |`np.triu(a)` | `triu(a)` | returns a copy of `a` with the upper triangular elements (including the diagonal) from `a` and the remaining lower triangular elements zeroed out | | | |`linalg.inv(a)` | `inv(a)` | inverse of square 2D tensor a | | | |`linalg.pinv(a)` | `pinv(a)` | pseudo-inverse of 2D tensor a | | | |`np.linalg.matrix_rank(a)` | `rank(a)` | matrix rank of a 2D tensor a | diff --git a/goal/transpile/math.go b/goal/transpile/math.go index c909bb71ba..095920c522 100644 --- a/goal/transpile/math.go +++ b/goal/transpile/math.go @@ -846,19 +846,18 @@ func (mp *mathParse) arrayShape(ex []ast.Expr, sh *[]int) { // nofun = do not accept a function version, just a method var numpyFuncs = map[string]funWrap{ // "array": {"tensor.NewFloatFromValues", ""}, // todo: probably not right, maybe don't have? - "zeros": {"tensor.NewFloat64", ""}, - "full": {"tensor.NewFloat64Full", ""}, - "ones": {"tensor.NewFloat64Ones", ""}, - "rand": {"tensor.NewFloat64Rand", ""}, - "arange": {"tensor.NewIntRange", ""}, - "linspace": {"tensor.NewFloat64SpacedLinear", ""}, - "transpose": {"tensor.Transpose", ""}, - "reshape": {"tensor.Reshape", ""}, - "copy": {"tensor.Clone", ""}, - "get": {"datafs.Get", ""}, - "set": {"datafs.Set", ""}, - "flatten": {"tensor.Flatten", "nofun"}, - "squeeze": {"tensor.Squeeze", "nofun"}, + "zeros": {"tensor.NewFloat64", ""}, + "full": {"tensor.NewFloat64Full", ""}, + "ones": {"tensor.NewFloat64Ones", ""}, + "rand": {"tensor.NewFloat64Rand", ""}, + "arange": {"tensor.NewIntRange", ""}, + "linspace": {"tensor.NewFloat64SpacedLinear", ""}, + "reshape": {"tensor.Reshape", ""}, + "copy": {"tensor.Clone", ""}, + "get": {"datafs.Get", ""}, + "set": {"datafs.Set", ""}, + "flatten": {"tensor.Flatten", "nofun"}, + "squeeze": {"tensor.Squeeze", "nofun"}, } func (mp *mathParse) callExpr(ex *ast.CallExpr) { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index eca158db40..99bad2a1eb 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -204,7 +204,6 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# a := [1,2,3,4]", `a := tensor.Tensor(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...))`}, {"# a := [[1,2], [3,4]]", `a := tensor.Tensor(tensor.Reshape(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...), 2, 2))`}, } st := NewState() @@ -228,6 +227,7 @@ func TestMath(t *testing.T) { {"# x += 1", `tmath.AddAssign(x, tensor.NewIntScalar(1))`}, {"# a := [1,2,3,4]", `a := tensor.Tensor(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...))`}, {"# a := [1.,2,3,4]", `a := tensor.Tensor(tensor.NewFloat64FromValues([]float64 { 1., 2, 3, 4 } ...))`}, + {"# a := [[1,2], [3,4]]", `a := tensor.Tensor(tensor.Reshape(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...), 2, 2))`}, {"# a.ndim", `tensor.NewIntScalar(a.NumDims())`}, {"# ndim(a)", `tensor.NewIntScalar(a.NumDims())`}, {"# x.shape", `tensor.NewIntFromValues(x.Shape().Sizes ...)`}, diff --git a/tensor/matrix/indices.go b/tensor/matrix/indices.go index b70e7d6200..40f0e01c62 100644 --- a/tensor/matrix/indices.go +++ b/tensor/matrix/indices.go @@ -6,7 +6,9 @@ package matrix import ( "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/num" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/vector" ) // offCols is a helper function to process the optional offset_cols args @@ -22,13 +24,13 @@ func offCols(size int, offset_cols ...int) (off, cols int) { return } -// Eye returns a new 2D Float64 tensor with 1s along the diagonal and +// Identity returns a new 2D Float64 tensor with 1s along the diagonal and // 0s elsewhere, with the given row and column size. // - If one additional parameter is passed, it is the offset, // to set values above (positive) or below (negative) the diagonal. // - If a second additional parameter is passed, it is the number of columns // for a non-square matrix (first size parameter = number of rows). -func Eye(size int, offset_cols ...int) *tensor.Float64 { +func Identity(size int, offset_cols ...int) *tensor.Float64 { off, cols := offCols(size, offset_cols...) tsr := tensor.NewFloat64(size, cols) for r := range size { @@ -41,6 +43,85 @@ func Eye(size int, offset_cols ...int) *tensor.Float64 { return tsr } +// DiagonalN returns the number of elements in the along the diagonal +// of a 2D matrix of given row and column size. +// - If one additional parameter is passed, it is the offset, +// to include values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func DiagonalN(size int, offset_cols ...int) int { + off, cols := offCols(size, offset_cols...) + rows := size + if num.Abs(off) > 0 { + oa := num.Abs(off) + if off > 0 { + if cols > rows { + return DiagonalN(rows, 0, cols-oa) + } else { + return DiagonalN(rows-oa, 0, cols-oa) + } + } else { + if rows > cols { + return DiagonalN(rows-oa, 0, cols) + } else { + return DiagonalN(rows-oa, 0, cols-oa) + } + } + } + n := min(rows, cols) + return n +} + +// DiagonalIndices returns a list of indices for the diagonal elements of +// a 2D matrix of given row and column size. +// The result is a 2D list of indices, where the outer (row) dimension +// is the number of indices, and the inner dimension is 2 for the r, c coords. +// - If one additional parameter is passed, it is the offset, +// to set values above (positive) or below (negative) the diagonal. +// - If a second additional parameter is passed, it is the number of columns +// for a non-square matrix (first size parameter = number of rows). +func DiagonalIndices(size int, offset_cols ...int) *tensor.Int { + off, cols := offCols(size, offset_cols...) + dn := DiagonalN(size, off, cols) + tsr := tensor.NewInt(dn, 2) + idx := 0 + for r := range size { + c := r + off + if c < 0 || c >= cols { + continue + } + tsr.SetInt(r, idx, 0) + tsr.SetInt(c, idx, 1) + idx++ + } + return tsr +} + +// Diagonal returns an [Indexed] view of the given tensor for the diagonal +// values, as a 1D list. An error is logged if the tensor is not 2D. +// Use the optional offset parameter to get values above (positive) or +// below (negative) the diagonal. +func Diagonal(tsr tensor.Tensor, offset ...int) *tensor.Indexed { + if tsr.NumDims() != 2 { + errors.Log(errors.New("matrix.TriLView requires a 2D tensor")) + return nil + } + off := 0 + if len(offset) == 1 { + off = offset[0] + } + return tensor.NewIndexed(tsr, DiagonalIndices(tsr.DimSize(0), off, tsr.DimSize(1))) +} + +// Trace returns the sum of the [Diagonal] elements of the given +// tensor, as a tensor scalar. +// An error is logged if the tensor is not 2D. +// Use the optional offset parameter to get values above (positive) or +// below (negative) the diagonal. +func Trace(tsr tensor.Tensor, offset ...int) tensor.Values { + return vector.Sum(Diagonal(tsr, offset...)) +} + // Tri returns a new 2D Float64 tensor with 1s along the diagonal and // below it, and 0s elsewhere (i.e., a filled lower triangle). // - If one additional parameter is passed, it is the offset, @@ -79,24 +160,24 @@ func TriUpper(size int, offset_cols ...int) *tensor.Float64 { return tsr } -// TriUN returns the number of elements in the upper triangular region +// TriUNum returns the number of elements in the upper triangular region // of a 2D matrix of given row and column size, where the triangle includes the // elements along the diagonal. // - If one additional parameter is passed, it is the offset, // to include values above (positive) or below (negative) the diagonal. // - If a second additional parameter is passed, it is the number of columns // for a non-square matrix (first size parameter = number of rows). -func TriUN(size int, offset_cols ...int) int { +func TriUNum(size int, offset_cols ...int) int { off, cols := offCols(size, offset_cols...) rows := size if off > 0 { if cols > rows { - return TriUN(rows, 0, cols-off) + return TriUNum(rows, 0, cols-off) } else { - return TriUN(rows-off, 0, cols-off) + return TriUNum(rows-off, 0, cols-off) } } else if off < 0 { // invert - return cols*rows - TriUN(cols, -(off-1), rows) + return cols*rows - TriUNum(cols, -(off-1), rows) } if cols <= size { return cols + (cols*(cols-1))/2 @@ -104,16 +185,16 @@ func TriUN(size int, offset_cols ...int) int { return rows + (rows*(2*cols-rows-1))/2 } -// TriLN returns the number of elements in the lower triangular region +// TriLNum returns the number of elements in the lower triangular region // of a 2D matrix of given row and column size, where the triangle includes the // elements along the diagonal. // - If one additional parameter is passed, it is the offset, // to include values above (positive) or below (negative) the diagonal. // - If a second additional parameter is passed, it is the number of columns // for a non-square matrix (first size parameter = number of rows). -func TriLN(size int, offset_cols ...int) int { +func TriLNum(size int, offset_cols ...int) int { off, cols := offCols(size, offset_cols...) - return TriUN(cols, -off, size) + return TriUNum(cols, -off, size) } // TriLIndicies returns the list of r, c indexes for the lower triangular @@ -126,7 +207,7 @@ func TriLN(size int, offset_cols ...int) int { // for a non-square matrix. func TriLIndicies(size int, offset_cols ...int) *tensor.Int { off, cols := offCols(size, offset_cols...) - trin := TriLN(size, off, cols) + trin := TriLNum(size, off, cols) coords := tensor.NewInt(trin, 2) i := 0 for r := range size { @@ -151,7 +232,7 @@ func TriLIndicies(size int, offset_cols ...int) *tensor.Int { // is the number of indices, and the inner dimension is 2 for the r, c coords. func TriUIndicies(size int, offset_cols ...int) *tensor.Int { off, cols := offCols(size, offset_cols...) - trin := TriUN(size, off, cols) + trin := TriUNum(size, off, cols) coords := tensor.NewInt(trin, 2) i := 0 for r := range size { diff --git a/tensor/matrix/indices_test.go b/tensor/matrix/indices_test.go index e02334d273..df40fd0771 100644 --- a/tensor/matrix/indices_test.go +++ b/tensor/matrix/indices_test.go @@ -13,9 +13,34 @@ import ( ) func TestIndices(t *testing.T) { - assert.Equal(t, []float64{1, 0, 0, 1}, Eye(2).Values) - assert.Equal(t, []float64{1, 0, 0, 0, 1, 0, 0, 0, 1}, Eye(3).Values) - assert.Equal(t, []float64{0, 0, 0, 1, 0, 0, 0, 1, 0}, Eye(3, -1).Values) + assert.Equal(t, []float64{1, 0, 0, 1}, Identity(2).Values) + assert.Equal(t, []float64{1, 0, 0, 0, 1, 0, 0, 0, 1}, Identity(3).Values) + assert.Equal(t, []float64{0, 0, 0, 1, 0, 0, 0, 1, 0}, Identity(3, -1).Values) + + assert.Equal(t, int(vector.Sum(Identity(3)).Float1D(0)), DiagonalN(3)) + assert.Equal(t, int(vector.Sum(Identity(3, 0, 4)).Float1D(0)), DiagonalN(3, 0, 4)) + assert.Equal(t, int(vector.Sum(Identity(3, 0, 2)).Float1D(0)), DiagonalN(3, 0, 2)) + assert.Equal(t, int(vector.Sum(Identity(3, 1)).Float1D(0)), DiagonalN(3, 1)) + assert.Equal(t, int(vector.Sum(Identity(10, 4, 7)).Float1D(0)), DiagonalN(10, 4, 7)) + assert.Equal(t, int(vector.Sum(Identity(10, 4, 12)).Float1D(0)), DiagonalN(10, 4, 12)) + assert.Equal(t, int(vector.Sum(Identity(3, -1)).Float1D(0)), DiagonalN(3, -1)) + assert.Equal(t, int(vector.Sum(Identity(10, -4, 7)).Float1D(0)), DiagonalN(10, -4, 7)) + assert.Equal(t, int(vector.Sum(Identity(10, -4, 12)).Float1D(0)), DiagonalN(10, -4, 12)) + + assert.Equal(t, []int{0, 0, 1, 1, 2, 2}, DiagonalIndices(3).Values) + assert.Equal(t, []int{0, 1, 1, 2}, DiagonalIndices(3, 1).Values) + assert.Equal(t, []int{1, 0, 2, 1}, DiagonalIndices(3, -1).Values) + assert.Equal(t, []int{1, 0, 2, 1}, DiagonalIndices(3, -1, 4).Values) + assert.Equal(t, []int{0, 1, 1, 2, 2, 3}, DiagonalIndices(3, 1, 4).Values) + + a := tensor.AsFloat64(tensor.Reshape(tensor.NewIntRange(1, 10), 3, 3)) + assert.Equal(t, []float64{1, 5, 9}, tensor.Flatten(Diagonal(a)).(*tensor.Float64).Values) + assert.Equal(t, []float64{4, 8}, tensor.Flatten(Diagonal(a, -1)).(*tensor.Float64).Values) + assert.Equal(t, []float64{2, 6}, tensor.Flatten(Diagonal(a, 1)).(*tensor.Float64).Values) + + assert.Equal(t, 15.0, Trace(a).Float1D(0)) + assert.Equal(t, 12.0, Trace(a, -1).Float1D(0)) + assert.Equal(t, 8.0, Trace(a, 1).Float1D(0)) assert.Equal(t, []float64{1, 0, 0, 1, 1, 0, 1, 1, 1}, Tri(3).Values) assert.Equal(t, []float64{1, 1, 0, 1, 1, 1, 1, 1, 1}, Tri(3, 1).Values) @@ -27,31 +52,31 @@ func TestIndices(t *testing.T) { assert.Equal(t, []float64{1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1}, Tri(3, 1, 4).Values) assert.Equal(t, []float64{1, 1, 1, 1, 1, 1}, Tri(3, 1, 2).Values) - assert.Equal(t, int(vector.Sum(TriUpper(3)).Float1D(0)), TriUN(3)) - assert.Equal(t, int(vector.Sum(TriUpper(3, 0, 4)).Float1D(0)), TriUN(3, 0, 4)) - assert.Equal(t, int(vector.Sum(TriUpper(3, 0, 2)).Float1D(0)), TriUN(3, 0, 2)) - assert.Equal(t, int(vector.Sum(TriUpper(3, 1)).Float1D(0)), TriUN(3, 1)) - assert.Equal(t, int(vector.Sum(TriUpper(3, 1, 4)).Float1D(0)), TriUN(3, 1, 4)) - assert.Equal(t, int(vector.Sum(TriUpper(10, 4, 7)).Float1D(0)), TriUN(10, 4, 7)) - assert.Equal(t, int(vector.Sum(TriUpper(10, 4, 12)).Float1D(0)), TriUN(10, 4, 12)) - assert.Equal(t, int(vector.Sum(TriUpper(3, -1)).Float1D(0)), TriUN(3, -1)) - assert.Equal(t, int(vector.Sum(TriUpper(3, -1, 4)).Float1D(0)), TriUN(3, -1, 4)) - assert.Equal(t, int(vector.Sum(TriUpper(3, -1, 2)).Float1D(0)), TriUN(3, -1, 2)) - assert.Equal(t, int(vector.Sum(TriUpper(10, -4, 7)).Float1D(0)), TriUN(10, -4, 7)) - assert.Equal(t, int(vector.Sum(TriUpper(10, -4, 12)).Float1D(0)), TriUN(10, -4, 12)) - - assert.Equal(t, int(vector.Sum(Tri(3)).Float1D(0)), TriLN(3)) - assert.Equal(t, int(vector.Sum(Tri(3, 0, 4)).Float1D(0)), TriLN(3, 0, 4)) - assert.Equal(t, int(vector.Sum(Tri(3, 0, 2)).Float1D(0)), TriLN(3, 0, 2)) - assert.Equal(t, int(vector.Sum(Tri(3, 1)).Float1D(0)), TriLN(3, 1)) - assert.Equal(t, int(vector.Sum(Tri(3, 1, 4)).Float1D(0)), TriLN(3, 1, 4)) - assert.Equal(t, int(vector.Sum(Tri(10, 4, 7)).Float1D(0)), TriLN(10, 4, 7)) - assert.Equal(t, int(vector.Sum(Tri(10, 4, 12)).Float1D(0)), TriLN(10, 4, 12)) - assert.Equal(t, int(vector.Sum(Tri(3, -1)).Float1D(0)), TriLN(3, -1)) - assert.Equal(t, int(vector.Sum(Tri(3, -1, 4)).Float1D(0)), TriLN(3, -1, 4)) - assert.Equal(t, int(vector.Sum(Tri(3, -1, 2)).Float1D(0)), TriLN(3, -1, 2)) - assert.Equal(t, int(vector.Sum(Tri(10, -4, 7)).Float1D(0)), TriLN(10, -4, 7)) - assert.Equal(t, int(vector.Sum(Tri(10, -4, 12)).Float1D(0)), TriLN(10, -4, 12)) + assert.Equal(t, int(vector.Sum(TriUpper(3)).Float1D(0)), TriUNum(3)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 0, 4)).Float1D(0)), TriUNum(3, 0, 4)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 0, 2)).Float1D(0)), TriUNum(3, 0, 2)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 1)).Float1D(0)), TriUNum(3, 1)) + assert.Equal(t, int(vector.Sum(TriUpper(3, 1, 4)).Float1D(0)), TriUNum(3, 1, 4)) + assert.Equal(t, int(vector.Sum(TriUpper(10, 4, 7)).Float1D(0)), TriUNum(10, 4, 7)) + assert.Equal(t, int(vector.Sum(TriUpper(10, 4, 12)).Float1D(0)), TriUNum(10, 4, 12)) + assert.Equal(t, int(vector.Sum(TriUpper(3, -1)).Float1D(0)), TriUNum(3, -1)) + assert.Equal(t, int(vector.Sum(TriUpper(3, -1, 4)).Float1D(0)), TriUNum(3, -1, 4)) + assert.Equal(t, int(vector.Sum(TriUpper(3, -1, 2)).Float1D(0)), TriUNum(3, -1, 2)) + assert.Equal(t, int(vector.Sum(TriUpper(10, -4, 7)).Float1D(0)), TriUNum(10, -4, 7)) + assert.Equal(t, int(vector.Sum(TriUpper(10, -4, 12)).Float1D(0)), TriUNum(10, -4, 12)) + + assert.Equal(t, int(vector.Sum(Tri(3)).Float1D(0)), TriLNum(3)) + assert.Equal(t, int(vector.Sum(Tri(3, 0, 4)).Float1D(0)), TriLNum(3, 0, 4)) + assert.Equal(t, int(vector.Sum(Tri(3, 0, 2)).Float1D(0)), TriLNum(3, 0, 2)) + assert.Equal(t, int(vector.Sum(Tri(3, 1)).Float1D(0)), TriLNum(3, 1)) + assert.Equal(t, int(vector.Sum(Tri(3, 1, 4)).Float1D(0)), TriLNum(3, 1, 4)) + assert.Equal(t, int(vector.Sum(Tri(10, 4, 7)).Float1D(0)), TriLNum(10, 4, 7)) + assert.Equal(t, int(vector.Sum(Tri(10, 4, 12)).Float1D(0)), TriLNum(10, 4, 12)) + assert.Equal(t, int(vector.Sum(Tri(3, -1)).Float1D(0)), TriLNum(3, -1)) + assert.Equal(t, int(vector.Sum(Tri(3, -1, 4)).Float1D(0)), TriLNum(3, -1, 4)) + assert.Equal(t, int(vector.Sum(Tri(3, -1, 2)).Float1D(0)), TriLNum(3, -1, 2)) + assert.Equal(t, int(vector.Sum(Tri(10, -4, 7)).Float1D(0)), TriLNum(10, -4, 7)) + assert.Equal(t, int(vector.Sum(Tri(10, -4, 12)).Float1D(0)), TriLNum(10, -4, 12)) tli := TriLIndicies(3) assert.Equal(t, []int{0, 0, 1, 0, 1, 1, 2, 0, 2, 1, 2, 2}, tli.Values) diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 519a5f9f8a..6a8e9c9683 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -34,22 +34,25 @@ func MatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { return nil } out.SetShapeSizes(rows, rows) - coords := TriangularLIndicies(rows) - nc := len(coords) + coords := matrix.TriLIndicies(rows) + nc := coords.DimSize(0) // note: flops estimating 3 per item on average -- different for different metrics. tensor.VectorizeThreaded(cells*3, func(tsr ...tensor.Tensor) int { return nc }, func(idx int, tsr ...tensor.Tensor) { - c := coords[idx] - sa := tensor.Cells1D(tsr[0], c.X) - sb := tensor.Cells1D(tsr[0], c.Y) + cx := coords.Int(idx, 0) + cy := coords.Int(idx, 1) + sa := tensor.Cells1D(tsr[0], cx) + sb := tensor.Cells1D(tsr[0], cy) mout := mfun(sa, sb) - tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) + tsr[1].SetFloat(mout.Float1D(0), cx, cy) }, in, out) - for _, c := range coords { // copy to upper - if c.X == c.Y { // exclude diag + for idx := range nc { // copy to upper + cx := coords.Int(idx, 0) + cy := coords.Int(idx, 1) + if cx == cy { // exclude diag continue } - out.SetFloat(out.Float(c.X, c.Y), c.Y, c.X) + out.SetFloat(out.Float(cx, cy), cy, cx) } return nil } @@ -147,28 +150,31 @@ func CovarianceMatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { var av, bv tensor.Tensor curCoords := vecint.Vector2i{-1, -1} - coords := TriangularLIndicies(cells) - nc := len(coords) + coords := matrix.TriLIndicies(cells) + nc := coords.DimSize(0) // note: flops estimating 3 per item on average -- different for different metrics. tensor.VectorizeThreaded(rows*3, func(tsr ...tensor.Tensor) int { return nc }, func(idx int, tsr ...tensor.Tensor) { - c := coords[idx] - if c.X != curCoords.X { - av = tensor.Reslice(tsr[0], tensor.FullAxis, c.X) - curCoords.X = c.X + cx := coords.Int(idx, 0) + cy := coords.Int(idx, 1) + if cx != curCoords.X { + av = tensor.Reslice(tsr[0], tensor.FullAxis, cx) + curCoords.X = cx } - if c.Y != curCoords.Y { - bv = tensor.Reslice(tsr[0], tensor.FullAxis, c.Y) - curCoords.Y = c.Y + if cy != curCoords.Y { + bv = tensor.Reslice(tsr[0], tensor.FullAxis, cy) + curCoords.Y = cy } mout := mfun(av, bv) - tsr[1].SetFloat(mout.Float1D(0), c.X, c.Y) + tsr[1].SetFloat(mout.Float1D(0), cx, cy) }, flatvw, out) - for _, c := range coords { // copy to upper - if c.X == c.Y { // exclude diag + for idx := range nc { // copy to upper + cx := coords.Int(idx, 0) + cy := coords.Int(idx, 1) + if cx == cy { // exclude diag continue } - out.SetFloat(out.Float(c.X, c.Y), c.Y, c.X) + out.SetFloat(out.Float(cx, cy), cy, cx) } return nil } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go b/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go index 7124f52e29..ce4a4583b6 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go @@ -10,16 +10,31 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/matrix/matrix"] = map[string]reflect.Value{ // function, constant and variable definitions - "CallOut2": reflect.ValueOf(matrix.CallOut2), - "CopyFromDense": reflect.ValueOf(matrix.CopyFromDense), - "Det": reflect.ValueOf(matrix.Det), - "LogDet": reflect.ValueOf(matrix.LogDet), - "Mul": reflect.ValueOf(matrix.Mul), - "MulOut": reflect.ValueOf(matrix.MulOut), - "NewDense": reflect.ValueOf(matrix.NewDense), - "NewMatrix": reflect.ValueOf(matrix.NewMatrix), - "NewSymmetric": reflect.ValueOf(matrix.NewSymmetric), - "StringCheck": reflect.ValueOf(matrix.StringCheck), + "CallOut2": reflect.ValueOf(matrix.CallOut2), + "CopyFromDense": reflect.ValueOf(matrix.CopyFromDense), + "Det": reflect.ValueOf(matrix.Det), + "Diagonal": reflect.ValueOf(matrix.Diagonal), + "DiagonalIndices": reflect.ValueOf(matrix.DiagonalIndices), + "DiagonalN": reflect.ValueOf(matrix.DiagonalN), + "Identity": reflect.ValueOf(matrix.Identity), + "LogDet": reflect.ValueOf(matrix.LogDet), + "Mul": reflect.ValueOf(matrix.Mul), + "MulOut": reflect.ValueOf(matrix.MulOut), + "NewDense": reflect.ValueOf(matrix.NewDense), + "NewMatrix": reflect.ValueOf(matrix.NewMatrix), + "NewSymmetric": reflect.ValueOf(matrix.NewSymmetric), + "StringCheck": reflect.ValueOf(matrix.StringCheck), + "Trace": reflect.ValueOf(matrix.Trace), + "Tri": reflect.ValueOf(matrix.Tri), + "TriL": reflect.ValueOf(matrix.TriL), + "TriLIndicies": reflect.ValueOf(matrix.TriLIndicies), + "TriLNum": reflect.ValueOf(matrix.TriLNum), + "TriLView": reflect.ValueOf(matrix.TriLView), + "TriU": reflect.ValueOf(matrix.TriU), + "TriUIndicies": reflect.ValueOf(matrix.TriUIndicies), + "TriUNum": reflect.ValueOf(matrix.TriUNum), + "TriUView": reflect.ValueOf(matrix.TriUView), + "TriUpper": reflect.ValueOf(matrix.TriUpper), // type definitions "Matrix": reflect.ValueOf((*matrix.Matrix)(nil)), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go index 6a72da3926..1e926da0de 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go @@ -70,8 +70,6 @@ func init() { "SumSquaresOut": reflect.ValueOf(metric.SumSquaresOut), "SumSquaresOut64": reflect.ValueOf(metric.SumSquaresOut64), "SumSquaresScaleOut64": reflect.ValueOf(metric.SumSquaresScaleOut64), - "TriangularLIndicies": reflect.ValueOf(metric.TriangularLIndicies), - "TriangularN": reflect.ValueOf(metric.TriangularN), "Vec2in3outFunc": reflect.ValueOf(metric.Vec2in3outFunc), "Vec2inFunc": reflect.ValueOf(metric.Vec2inFunc), "Vec3outFunc": reflect.ValueOf(metric.Vec3outFunc), From fea4f0072226c22df8aefa6e61ae24f643ebe2c2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 2 Oct 2024 16:33:30 -0700 Subject: [PATCH 182/311] move gosl into goal, and initial plans for GPU mode -- seems pretty good --- goal/GPU.md | 159 ++++++++++++++++++ goal/README.md | 2 + {gpu => goal}/gosl/README.md | 16 +- {gpu => goal}/gosl/alignsl/README.md | 0 {gpu => goal}/gosl/alignsl/alignsl.go | 0 {gpu => goal}/gosl/doc.go | 0 {gpu => goal}/gosl/examples/basic/README.md | 0 {gpu => goal}/gosl/examples/basic/compute.go | 0 {gpu => goal}/gosl/examples/basic/main.go | 0 .../gosl/examples/basic/shaders/basic.wgsl | 0 .../gosl/examples/basic/shaders/fastexp.wgsl | 0 {gpu => goal}/gosl/examples/rand/README.md | 0 {gpu => goal}/gosl/examples/rand/main.go | 0 {gpu => goal}/gosl/examples/rand/rand.go | 0 {gpu => goal}/gosl/examples/rand/rand.wgsl | 0 .../gosl/examples/rand/shaders/rand.wgsl | 0 .../gosl/examples/rand/shaders/slrand.wgsl | 0 .../gosl/examples/rand/shaders/sltype.wgsl | 0 {gpu => goal}/gosl/extract.go | 0 {gpu => goal}/gosl/files.go | 0 {gpu => goal}/gosl/gosl.go | 0 {gpu => goal}/gosl/gosl_test.go | 0 {gpu => goal}/gosl/process.go | 0 {gpu => goal}/gosl/slbool/README.md | 0 {gpu => goal}/gosl/slbool/slbool.go | 0 .../gosl/slbool/slboolcore/slboolcore.go | 0 {gpu => goal}/gosl/sledits.go | 0 {gpu => goal}/gosl/slprint/comment.go | 0 {gpu => goal}/gosl/slprint/gobuild.go | 0 {gpu => goal}/gosl/slprint/nodes.go | 0 {gpu => goal}/gosl/slprint/printer.go | 0 {gpu => goal}/gosl/slrand/README.md | 0 {gpu => goal}/gosl/slrand/slrand.go | 0 {gpu => goal}/gosl/slrand/slrand.wgsl | 0 {gpu => goal}/gosl/slrand/slrand_test.go | 0 {gpu => goal}/gosl/sltype/README.md | 0 {gpu => goal}/gosl/sltype/float.go | 0 {gpu => goal}/gosl/sltype/int.go | 0 {gpu => goal}/gosl/sltype/sltype.go | 0 {gpu => goal}/gosl/sltype/sltype.wgsl | 0 {gpu => goal}/gosl/sltype/sltype_test.go | 0 {gpu => goal}/gosl/testdata/basic.go | 0 {gpu => goal}/gosl/testdata/basic.golden | 0 {gpu => goal}/gosl/threading/threading.go | 0 44 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 goal/GPU.md rename {gpu => goal}/gosl/README.md (88%) rename {gpu => goal}/gosl/alignsl/README.md (100%) rename {gpu => goal}/gosl/alignsl/alignsl.go (100%) rename {gpu => goal}/gosl/doc.go (100%) rename {gpu => goal}/gosl/examples/basic/README.md (100%) rename {gpu => goal}/gosl/examples/basic/compute.go (100%) rename {gpu => goal}/gosl/examples/basic/main.go (100%) rename {gpu => goal}/gosl/examples/basic/shaders/basic.wgsl (100%) rename {gpu => goal}/gosl/examples/basic/shaders/fastexp.wgsl (100%) rename {gpu => goal}/gosl/examples/rand/README.md (100%) rename {gpu => goal}/gosl/examples/rand/main.go (100%) rename {gpu => goal}/gosl/examples/rand/rand.go (100%) rename {gpu => goal}/gosl/examples/rand/rand.wgsl (100%) rename {gpu => goal}/gosl/examples/rand/shaders/rand.wgsl (100%) rename {gpu => goal}/gosl/examples/rand/shaders/slrand.wgsl (100%) rename {gpu => goal}/gosl/examples/rand/shaders/sltype.wgsl (100%) rename {gpu => goal}/gosl/extract.go (100%) rename {gpu => goal}/gosl/files.go (100%) rename {gpu => goal}/gosl/gosl.go (100%) rename {gpu => goal}/gosl/gosl_test.go (100%) rename {gpu => goal}/gosl/process.go (100%) rename {gpu => goal}/gosl/slbool/README.md (100%) rename {gpu => goal}/gosl/slbool/slbool.go (100%) rename {gpu => goal}/gosl/slbool/slboolcore/slboolcore.go (100%) rename {gpu => goal}/gosl/sledits.go (100%) rename {gpu => goal}/gosl/slprint/comment.go (100%) rename {gpu => goal}/gosl/slprint/gobuild.go (100%) rename {gpu => goal}/gosl/slprint/nodes.go (100%) rename {gpu => goal}/gosl/slprint/printer.go (100%) rename {gpu => goal}/gosl/slrand/README.md (100%) rename {gpu => goal}/gosl/slrand/slrand.go (100%) rename {gpu => goal}/gosl/slrand/slrand.wgsl (100%) rename {gpu => goal}/gosl/slrand/slrand_test.go (100%) rename {gpu => goal}/gosl/sltype/README.md (100%) rename {gpu => goal}/gosl/sltype/float.go (100%) rename {gpu => goal}/gosl/sltype/int.go (100%) rename {gpu => goal}/gosl/sltype/sltype.go (100%) rename {gpu => goal}/gosl/sltype/sltype.wgsl (100%) rename {gpu => goal}/gosl/sltype/sltype_test.go (100%) rename {gpu => goal}/gosl/testdata/basic.go (100%) rename {gpu => goal}/gosl/testdata/basic.golden (100%) rename {gpu => goal}/gosl/threading/threading.go (100%) diff --git a/goal/GPU.md b/goal/GPU.md new file mode 100644 index 0000000000..d2c04af409 --- /dev/null +++ b/goal/GPU.md @@ -0,0 +1,159 @@ +# Goal GPU support + +The use of massively parallel _Graphical Processsing Unit_ (_GPU_) hardware has revolutionized machine learning and other fields, producing many factors of speedup relative to traditional _CPU_ (_Central Processing Unit_) computation. However, there are numerous challenges for supporting GPU-based computation, relative to the more flexible CPU coding. + +Goal provides a solution to these challenges that enables the same Go-based code to work efficiently and reasonably naturally on both the GPU and CPU (i.e., standard Go execution), for maximum portability. The ability to run the same code on both types of hardware is also critical for debugging the otherwise difficult to debug GPU version, and avoiding bugs in the first place by catching them first on the CPU, while providing known correct comparison results. + +The two most important challenges are: + +* The GPU _has its own separate memory space_ that needs to be synchronized explicitly and bidirectionally with the standard CPU memory (this is true programmatically even if at a hardware level there is shared memory). + +* Computation must be organized into discrete chunks that can be computed efficiently in parallel, and each such chunk of computation lives in its own separate _kernel_ (_compute shader_) in the GPU, as an entirely separate, self-contained program, operating on _global variables_ that define the entire memory space of the computation. + +To be maximally efficient, both of these factors must be optimized, such that: + +* The bidirectional syncing of memory between CPU and GPU should be minimized, because such transfers incur a significant overhead. + +* The overall computation should be broken down into the _largest possible chunks_ to minimize the number of discrete kernel runs, each of which incurs significant overhead. + +Thus, it is unfortunately _highly inefficient_ to implement GPU-based computation by running each elemental vectorizable tensor operation (add, multiply, etc) as a separate GPU kernel, with its own separate bidirectional memory sync, even though that is a conceptually attractive and simple way to organize GPU computation, with minimal disruption relative to the CPU model. + +The [JAX](https://github.com/jax-ml/jax) framework in Python provides one solution to this situation, optimized for neural network machine learning uses, by imposing strict _functional programming_ constraints on the code you write (i.e., all functions must be _read-only_), and leveraging those to automatically combine elemental computations into larger parallelizable chunks, using a "just in time" (_jit_) compiler. + +We take a different approach, which is much simpler implementationally but requires a bit more work from the developer, which is to provide tools that allow _you_ to organize your computation into kernel-sized chunks according to your knowledge of the problem, and transparently turn that code into the final CPU and GPU programs. + +In many cases, a human programmer can most likely out-perform the automatic compilation process, by knowing the full scope of what needs to be computed, and figuring out how to package it most efficiently per the above constraints. In the end, you get maximum efficiency and complete transparency about exactly what is being computed, perhaps with fewer "gotcha" bugs arising from all the magic happening under the hood, but again it may take a bit more work to get there. + +The role of Goal is to allow you to express the full computation in the clear, simple, Go language, using intuitive data structures that minimize the need for additional boilerplate to run efficiently on CPU and GPU. This ability to write a single codebase that runs efficiently on CPU and GPU is similar to the [SYCL](https://en.wikipedia.org/wiki/SYCL) framework (and several others discussed on that wikipedia page), which builds on [OpenCL](https://en.wikipedia.org/wiki/OpenCL), both of which are based on the C / C++ programming language. + +In addition to the critical differences between Go and C++ as languages, Goal targets only one hardware platform: WebGPU (via our [gpu](../gpu) package), so it is more specifically optimized for this use-case. Furthermore, SYCL and other approaches require you to write GPU-like code that can also run on the CPU (with lots of explicit fine-grained memory and compute management), whereas Goal provides a more natural CPU-like programming model, while imposing some stronger constraints that encourage more efficient implementations. + +The [gosl](gosl) (_Go shader language_) package within Goal does the heavy lifting of translating Go code into WGSL shader language code that can run on the WebGPU, and generally manages most of the gpu-specific functionality. It has various important functions defined in the `gosl.` package space, and a number of `//gosl:` comment directives described below, that make everything work. + +Meanwhile, the `goal build` command provides an outer-loop of orchestration and math code transpiling prior to handing off to gosl to run on the relevant files. + +For example, `gosl.UseCPU()` causes subsequent execution to use CPU code, while `gosl.UseGPU()` causes it to use GPU kernels. Other `gosl` calls configure and activate the GPU during an initialization step. + +## Overall Code Organization + +First, we assume the scope is a single Go package that implements a set of computations on some number of associated data representations. The package will likely contain a lot of CPU-only Go code that manages all the surrounding infrastructure for the computations, in terms of creating and configuring the data in memory, visualization, i/o, etc. + +The GPU-specific computation is organized into some (hopefully small) number of **kernel** functions, that are called using a special `parallel` version of a `for range` loop: + +```Go +for i := range parallel(data) { + MyCompute(i) +} +``` + +Where the `parallel` function is a special Goal keyword that triggers GPU vs. CPU execution, depending on prior configuration setup, and `MyCompute` is a kernel function. The `i` index effectively iterates over the range of the values of the `tensor` variable `data` (using specific dimension(s) with optional parameter(s)), with the GPU version launching the kernel on the GPU. + +We assume that multiple kernels will in general be required, and that there is likely to be a significant amount of shared code infrastructure across these kernels. + +> We support multiple CPU-based kernels within a single Go package directory. + +Even though the GPU kernels must each be compiled separately into a single distinct WGSL _shader_ file that is run under WebGPU, they can `import` a shared codebase of files, and thus replicate the same overall shared code structure as the CPU versions. + +The GPU code can only handle a highly restricted _subset_ of Go code, with data structures having strict alignment requirements, and no `string` or other composite variable-length data structures (maps, slices etc). Thus, the [gosl](gosl) package recognizes `//gosl:start` and `//gosl:end` comment directives surrounding the GPU-safe (and relevant) portions of the overall code. Any `.go` or `.goal` file can contribute GPU relevant code, including in other packages, and the gosl system automatically builds a shadow package-based set of `.wgsl` files accordingly. + +> Each kernel must be written in a separate `.goal` file, marked with the `//gosl:kernel` directive at the top. There must be a "main" function entry point for each kernel, marked with the //gosl:main directive, which takes the index argument as shown in the code example above. + +For CPU (regular Go) mode, the parallel `for range` loop as shown above translates into a `tensor.VectorizeThreaded` call of the named Go function. For GPU mode, it launches the kernel on the GPU. + +## Memory Organization + +Perhaps the strongest constraints for GPU programming stem from the need to organize and synchronize all the memory buffers holding the data that the GPU kernel operates on. Furthermore, within a GPU kernel, the variables representing this data are _global variables_, which is sensible given the standalone nature of each kernel. + +> To provide a common programming environment, all GPU-relevant variables must be Go global variables. + +Thus, names must be chosen appropriately for these variables, given their global scope within the Go package. The specific _values_ for these variables can be dynamically set in an easy way, but the variables themselves are global. + +Within the [gpu](../gpu) framework, each `ComputeSystem` defines a specific organization of such GPU buffer variables, and maximum efficiency is achieved by minimizing the number of such compute systems, and associated memory buffers. Each system also encapsulates the associated kernel shaders that operate on the associated memory data, so + +> Kernels and variables both must be defined within a specific system context. + +The following comment directive can be used in any kernel file to specify which system it uses, and there is an initial `default` system that is used if none is ever specified. +```Go +//gosl:system +``` + +To define the global variables for each system, use a standard Go `var` block declaration (with optional system name qualifier): +```Go +var ( //gosl:vars [system name] + + //gosl:group -uniform Params + + // Layer-level parameters + Layers []LayerParam // note: struct with appropriate memory alignment + + // Path-level parameters + Paths []PathParam + + //gosl:group Units + + // Unit state values + Units tensor.Float32 + + // Synapse weight state values + Weights tensor.Float32 +) +``` + +The `//gosl:vars` directive flags this block of variables as GPU-accessible variables, which will drive the automatic generation of [gpu](../gpu) code to define these variables for the current (named) system, and to declare them in each kernel associated with that system. + +The `//gosl:group` directives specify groups of variables, which generally should have similar memory syncing behavior, as documented in the [gpu](../gpu) system. + +### datafs mapping + +The grouped global variables can be mapped directly to a corresponding [datafs](../tensor/datafs) directory, which provides direct accessibility to this data within interactive Goal usage. Further, different sets of variable values can be easily managed by saving and loading different such directories. + +```Go + gosl.ToDataFS("path/to/dir" [, system]) // set datafs items in given path to current global vars + + gosl.FromDataFS("path/to/dir" [,system]) // set global vars from given datafs path +``` + +These and all such `gosl` functions use the current system if none is explicitly specified, which is settable using the `gosl.SetSystem` call. Any given variable can use the `get` or `set` Goal math mode functions directly. + +## Memory syncing + +It is up to the programmer to manage the syncing of memory between the CPU and the GPU, using simple `gosl` wrapper functions that manage all the details: + +```Go + gosl.ToGPU(varName...) // sync current contents of given GPU variable(s) from CPU to GPU + + gosl.FromGPU(varName...) // sync back from GPU to CPU +``` + +These are no-ops if operating in CPU mode. + +## Memory access + +In general, all global GPU variables will be arrays (slices) or tensors, which are exposed to the GPU as an array of floats. + +The tensor-based indexing syntax in Goal math mode transparently works across CPU and GPU modes, and is thus the preferred way to access tensor data. + +It is critical to appreciate that none of the other convenient math-mode operations will work as you expect on the GPU, because: + +> There is only one outer-loop, kernel-level parallel looping operation allowed at a time. + +You cannot nest multiple such loops within each other. A kernel cannot launch another kernel. Therefore, as noted above, you must directly organize your computation to maximize the amount of parallel computation happening wthin each such kernel call. + +> Therefore, tensor indexing on the GPU only supports direct index values, not ranges. + +Furthermore: + +> Pointer-based access of global variables is not supported in GPU mode. + +You have to use _indexes_ into arrays exclusively. Thus, some of the data structures you may need to copy up to the GPU include index variables that determine how to access other variables. TODO: do we need helpers for any of this? + +## Optimization + +can run naga on wgsl code to get wgsl code out, but it doesn't seem to do much dead code elimination: https://github.com/gfx-rs/wgpu/tree/trunk/naga + +``` +naga --compact gpu_applyext.wgsl tmp.wgsl +``` + +https://github.com/LucentFlux/wgsl-minifier does radical minification but the result is unreadable so we don't know if it is doing dead code elimination. in theory it is just calling naga --compact for that. + diff --git a/goal/README.md b/goal/README.md index 2ca0f9b783..def641ee54 100644 --- a/goal/README.md +++ b/goal/README.md @@ -8,6 +8,8 @@ Goal transpiles directly into Go, so it automatically leverages all the great fe * Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and need extended indexing and slicing expressions. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. +* GPU-based parallel computation, which can greatly speed up some types of parallelizable computations significantly by effectively running many instances of the same code in parallel across a large array of data. Goal (and [gosl](gosl)) allow you to run the same Go-based code on a GPU or CPU (using parallel goroutines). See the [GPU](GPU.md) docs for full info. + The main goal of Goal is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where the Go language has been a barrier. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, Goal retains in its Go foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. For the shell scripting aspect of Goal, the simple idea is that each line of code is either Go or shell commands, determined in a fairly intuitive way mostly by the content at the start of the line (formal rules below). If a line starts off with something like `ls -la...` then it is clear that it is not valid Go code, and it is therefore processed as a shell command. diff --git a/gpu/gosl/README.md b/goal/gosl/README.md similarity index 88% rename from gpu/gosl/README.md rename to goal/gosl/README.md index 754697f848..415053fddd 100644 --- a/gpu/gosl/README.md +++ b/goal/gosl/README.md @@ -1,13 +1,13 @@ -# gosl +# gosl: Go as a shader language `gosl` implements _Go as a shader language_ for GPU compute shaders (using [WebGPU](https://www.w3.org/TR/webgpu/)), **enabling standard Go code to run on the GPU**. +`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go langauge. See the [GPU](../GPU.md) documentation for an overview. + The relevant subsets of Go code are specifically marked using `//gosl:` comment directives, and this code must only use basic expressions and concrete types that will compile correctly in a shader (see [Restrictions](#restrictions) below). Method functions and pass-by-reference pointer arguments to `struct` types are supported and incur no additional compute cost due to inlining (see notes below for more detail). A large and complex biologically-based neural network simulation framework called [axon](https://github.com/emer/axon) has been implemented using `gosl`, allowing 1000's of lines of equations and data structures to run through standard Go on the CPU, and accelerated significantly on the GPU. This allows efficient debugging and unit testing of the code in Go, whereas debugging on the GPU is notoriously difficult. -`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader. - See [examples/basic](examples/basic) and [rand](examples/rand) for examples, using the [gpu](../../gpu) GPU compute shader system. It is also possible in principle to use gosl to generate shader files for any other WebGPU application, but this has not been tested. You must also install `goimports` which is used on the extracted subset of Go code, to get the imports right: @@ -22,14 +22,18 @@ $ go install cogentcore.org/core/vgpu/gosl@latest In your Go code, use these comment directives: ``` -//gosl:start +//gosl:start < Go code to be translated > -//gosl:end +//gosl:end ``` -to bracket code to be processed. The resulting converted code is copied into a `shaders` subdirectory created under the current directory where the `gosl` command is run, using the filenames specified in the comment directives. Each such filename should correspond to a complete shader program (i.e., a "kernel"), or a file that can be included into other shader programs. Code is appended to the target file names in the order of the source .go files on the command line, so multiple .go files can be combined into one resulting WGSL file. +to bracket code to be processed for GPU. The resulting converted code is copied into a `shaders` subdirectory created under the current directory where the `gosl` command is run, using the filenames specified in the comment directives, or the name of the current file if not specified. + +Use the `//gosl:import package` directive to include files from other packages, similar to the standard Go import directive. It is assumed that many other Go imports are not GPU relevant, so this separate directive is required. + +Each such filename should correspond to a complete shader program (i.e., a "kernel"), or a file that can be included into other shader programs. Code is appended to the target file names in the order of the source .go files on the command line, so multiple .go files can be combined into one resulting WGSL file. WGSL specific code, e.g., for the `main` compute function or to specify `#include` files, can be included either by specifying files with a `.wgsl` extension as arguments to the `gosl` command, or by using a `//gosl:wgsl` comment directive as follows: ``` diff --git a/gpu/gosl/alignsl/README.md b/goal/gosl/alignsl/README.md similarity index 100% rename from gpu/gosl/alignsl/README.md rename to goal/gosl/alignsl/README.md diff --git a/gpu/gosl/alignsl/alignsl.go b/goal/gosl/alignsl/alignsl.go similarity index 100% rename from gpu/gosl/alignsl/alignsl.go rename to goal/gosl/alignsl/alignsl.go diff --git a/gpu/gosl/doc.go b/goal/gosl/doc.go similarity index 100% rename from gpu/gosl/doc.go rename to goal/gosl/doc.go diff --git a/gpu/gosl/examples/basic/README.md b/goal/gosl/examples/basic/README.md similarity index 100% rename from gpu/gosl/examples/basic/README.md rename to goal/gosl/examples/basic/README.md diff --git a/gpu/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go similarity index 100% rename from gpu/gosl/examples/basic/compute.go rename to goal/gosl/examples/basic/compute.go diff --git a/gpu/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go similarity index 100% rename from gpu/gosl/examples/basic/main.go rename to goal/gosl/examples/basic/main.go diff --git a/gpu/gosl/examples/basic/shaders/basic.wgsl b/goal/gosl/examples/basic/shaders/basic.wgsl similarity index 100% rename from gpu/gosl/examples/basic/shaders/basic.wgsl rename to goal/gosl/examples/basic/shaders/basic.wgsl diff --git a/gpu/gosl/examples/basic/shaders/fastexp.wgsl b/goal/gosl/examples/basic/shaders/fastexp.wgsl similarity index 100% rename from gpu/gosl/examples/basic/shaders/fastexp.wgsl rename to goal/gosl/examples/basic/shaders/fastexp.wgsl diff --git a/gpu/gosl/examples/rand/README.md b/goal/gosl/examples/rand/README.md similarity index 100% rename from gpu/gosl/examples/rand/README.md rename to goal/gosl/examples/rand/README.md diff --git a/gpu/gosl/examples/rand/main.go b/goal/gosl/examples/rand/main.go similarity index 100% rename from gpu/gosl/examples/rand/main.go rename to goal/gosl/examples/rand/main.go diff --git a/gpu/gosl/examples/rand/rand.go b/goal/gosl/examples/rand/rand.go similarity index 100% rename from gpu/gosl/examples/rand/rand.go rename to goal/gosl/examples/rand/rand.go diff --git a/gpu/gosl/examples/rand/rand.wgsl b/goal/gosl/examples/rand/rand.wgsl similarity index 100% rename from gpu/gosl/examples/rand/rand.wgsl rename to goal/gosl/examples/rand/rand.wgsl diff --git a/gpu/gosl/examples/rand/shaders/rand.wgsl b/goal/gosl/examples/rand/shaders/rand.wgsl similarity index 100% rename from gpu/gosl/examples/rand/shaders/rand.wgsl rename to goal/gosl/examples/rand/shaders/rand.wgsl diff --git a/gpu/gosl/examples/rand/shaders/slrand.wgsl b/goal/gosl/examples/rand/shaders/slrand.wgsl similarity index 100% rename from gpu/gosl/examples/rand/shaders/slrand.wgsl rename to goal/gosl/examples/rand/shaders/slrand.wgsl diff --git a/gpu/gosl/examples/rand/shaders/sltype.wgsl b/goal/gosl/examples/rand/shaders/sltype.wgsl similarity index 100% rename from gpu/gosl/examples/rand/shaders/sltype.wgsl rename to goal/gosl/examples/rand/shaders/sltype.wgsl diff --git a/gpu/gosl/extract.go b/goal/gosl/extract.go similarity index 100% rename from gpu/gosl/extract.go rename to goal/gosl/extract.go diff --git a/gpu/gosl/files.go b/goal/gosl/files.go similarity index 100% rename from gpu/gosl/files.go rename to goal/gosl/files.go diff --git a/gpu/gosl/gosl.go b/goal/gosl/gosl.go similarity index 100% rename from gpu/gosl/gosl.go rename to goal/gosl/gosl.go diff --git a/gpu/gosl/gosl_test.go b/goal/gosl/gosl_test.go similarity index 100% rename from gpu/gosl/gosl_test.go rename to goal/gosl/gosl_test.go diff --git a/gpu/gosl/process.go b/goal/gosl/process.go similarity index 100% rename from gpu/gosl/process.go rename to goal/gosl/process.go diff --git a/gpu/gosl/slbool/README.md b/goal/gosl/slbool/README.md similarity index 100% rename from gpu/gosl/slbool/README.md rename to goal/gosl/slbool/README.md diff --git a/gpu/gosl/slbool/slbool.go b/goal/gosl/slbool/slbool.go similarity index 100% rename from gpu/gosl/slbool/slbool.go rename to goal/gosl/slbool/slbool.go diff --git a/gpu/gosl/slbool/slboolcore/slboolcore.go b/goal/gosl/slbool/slboolcore/slboolcore.go similarity index 100% rename from gpu/gosl/slbool/slboolcore/slboolcore.go rename to goal/gosl/slbool/slboolcore/slboolcore.go diff --git a/gpu/gosl/sledits.go b/goal/gosl/sledits.go similarity index 100% rename from gpu/gosl/sledits.go rename to goal/gosl/sledits.go diff --git a/gpu/gosl/slprint/comment.go b/goal/gosl/slprint/comment.go similarity index 100% rename from gpu/gosl/slprint/comment.go rename to goal/gosl/slprint/comment.go diff --git a/gpu/gosl/slprint/gobuild.go b/goal/gosl/slprint/gobuild.go similarity index 100% rename from gpu/gosl/slprint/gobuild.go rename to goal/gosl/slprint/gobuild.go diff --git a/gpu/gosl/slprint/nodes.go b/goal/gosl/slprint/nodes.go similarity index 100% rename from gpu/gosl/slprint/nodes.go rename to goal/gosl/slprint/nodes.go diff --git a/gpu/gosl/slprint/printer.go b/goal/gosl/slprint/printer.go similarity index 100% rename from gpu/gosl/slprint/printer.go rename to goal/gosl/slprint/printer.go diff --git a/gpu/gosl/slrand/README.md b/goal/gosl/slrand/README.md similarity index 100% rename from gpu/gosl/slrand/README.md rename to goal/gosl/slrand/README.md diff --git a/gpu/gosl/slrand/slrand.go b/goal/gosl/slrand/slrand.go similarity index 100% rename from gpu/gosl/slrand/slrand.go rename to goal/gosl/slrand/slrand.go diff --git a/gpu/gosl/slrand/slrand.wgsl b/goal/gosl/slrand/slrand.wgsl similarity index 100% rename from gpu/gosl/slrand/slrand.wgsl rename to goal/gosl/slrand/slrand.wgsl diff --git a/gpu/gosl/slrand/slrand_test.go b/goal/gosl/slrand/slrand_test.go similarity index 100% rename from gpu/gosl/slrand/slrand_test.go rename to goal/gosl/slrand/slrand_test.go diff --git a/gpu/gosl/sltype/README.md b/goal/gosl/sltype/README.md similarity index 100% rename from gpu/gosl/sltype/README.md rename to goal/gosl/sltype/README.md diff --git a/gpu/gosl/sltype/float.go b/goal/gosl/sltype/float.go similarity index 100% rename from gpu/gosl/sltype/float.go rename to goal/gosl/sltype/float.go diff --git a/gpu/gosl/sltype/int.go b/goal/gosl/sltype/int.go similarity index 100% rename from gpu/gosl/sltype/int.go rename to goal/gosl/sltype/int.go diff --git a/gpu/gosl/sltype/sltype.go b/goal/gosl/sltype/sltype.go similarity index 100% rename from gpu/gosl/sltype/sltype.go rename to goal/gosl/sltype/sltype.go diff --git a/gpu/gosl/sltype/sltype.wgsl b/goal/gosl/sltype/sltype.wgsl similarity index 100% rename from gpu/gosl/sltype/sltype.wgsl rename to goal/gosl/sltype/sltype.wgsl diff --git a/gpu/gosl/sltype/sltype_test.go b/goal/gosl/sltype/sltype_test.go similarity index 100% rename from gpu/gosl/sltype/sltype_test.go rename to goal/gosl/sltype/sltype_test.go diff --git a/gpu/gosl/testdata/basic.go b/goal/gosl/testdata/basic.go similarity index 100% rename from gpu/gosl/testdata/basic.go rename to goal/gosl/testdata/basic.go diff --git a/gpu/gosl/testdata/basic.golden b/goal/gosl/testdata/basic.golden similarity index 100% rename from gpu/gosl/testdata/basic.golden rename to goal/gosl/testdata/basic.golden diff --git a/gpu/gosl/threading/threading.go b/goal/gosl/threading/threading.go similarity index 100% rename from gpu/gosl/threading/threading.go rename to goal/gosl/threading/threading.go From 2ea13751f1339bdafef33a6634e0ab426101ba37 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 2 Oct 2024 21:54:12 -0700 Subject: [PATCH 183/311] reorganize gosl code with everything in parse, using cli config, leaving top-level gosl package with gosl helper functions. allows goal to directly run all of gosl. --- goal/GPU.md | 12 +- goal/gosl/README.md | 6 +- goal/gosl/cmd/gosl/gosl.go | 16 +++ goal/gosl/examples/basic/shaders/fastexp.wgsl | 14 -- goal/gosl/gosl.go | 86 ------------- goal/gosl/{slprint => parse}/comment.go | 9 +- goal/gosl/parse/config.go | 76 +++++++++++ goal/gosl/{ => parse}/extract.go | 12 +- goal/gosl/{ => parse}/files.go | 20 +-- goal/gosl/{slprint => parse}/gobuild.go | 9 +- goal/gosl/{ => parse}/gosl_test.go | 2 +- goal/gosl/{slprint => parse}/nodes.go | 13 +- goal/gosl/{slprint => parse}/printer.go | 47 ++++--- goal/gosl/{ => parse}/process.go | 42 +++--- goal/gosl/{ => parse}/sledits.go | 2 +- goal/gosl/{ => parse}/testdata/basic.go | 0 goal/gosl/{ => parse}/testdata/basic.golden | 0 goal/gosl/parse/typegen.go | 121 ++++++++++++++++++ 18 files changed, 318 insertions(+), 169 deletions(-) create mode 100644 goal/gosl/cmd/gosl/gosl.go delete mode 100644 goal/gosl/examples/basic/shaders/fastexp.wgsl delete mode 100644 goal/gosl/gosl.go rename goal/gosl/{slprint => parse}/comment.go (93%) create mode 100644 goal/gosl/parse/config.go rename goal/gosl/{ => parse}/extract.go (94%) rename goal/gosl/{ => parse}/files.go (89%) rename goal/gosl/{slprint => parse}/gobuild.go (94%) rename goal/gosl/{ => parse}/gosl_test.go (99%) rename goal/gosl/{slprint => parse}/nodes.go (99%) rename goal/gosl/{slprint => parse}/printer.go (97%) rename goal/gosl/{ => parse}/process.go (81%) rename goal/gosl/{ => parse}/sledits.go (99%) rename goal/gosl/{ => parse}/testdata/basic.go (100%) rename goal/gosl/{ => parse}/testdata/basic.golden (100%) create mode 100644 goal/gosl/parse/typegen.go diff --git a/goal/GPU.md b/goal/GPU.md index d2c04af409..8e50bb3853 100644 --- a/goal/GPU.md +++ b/goal/GPU.md @@ -2,7 +2,7 @@ The use of massively parallel _Graphical Processsing Unit_ (_GPU_) hardware has revolutionized machine learning and other fields, producing many factors of speedup relative to traditional _CPU_ (_Central Processing Unit_) computation. However, there are numerous challenges for supporting GPU-based computation, relative to the more flexible CPU coding. -Goal provides a solution to these challenges that enables the same Go-based code to work efficiently and reasonably naturally on both the GPU and CPU (i.e., standard Go execution), for maximum portability. The ability to run the same code on both types of hardware is also critical for debugging the otherwise difficult to debug GPU version, and avoiding bugs in the first place by catching them first on the CPU, while providing known correct comparison results. +Goal provides a solution to these challenges that enables the same Go-based code to work efficiently and reasonably naturally on both the GPU and CPU (i.e., standard Go execution), for maximum portability. The ability to run the same code on both types of hardware is also critical for debugging the otherwise difficult to debug GPU version, and for avoiding bugs in the first place by catching them first on the CPU, while providing known correct comparison results. The two most important challenges are: @@ -28,6 +28,10 @@ The role of Goal is to allow you to express the full computation in the clear, s In addition to the critical differences between Go and C++ as languages, Goal targets only one hardware platform: WebGPU (via our [gpu](../gpu) package), so it is more specifically optimized for this use-case. Furthermore, SYCL and other approaches require you to write GPU-like code that can also run on the CPU (with lots of explicit fine-grained memory and compute management), whereas Goal provides a more natural CPU-like programming model, while imposing some stronger constraints that encourage more efficient implementations. +The bottom line is that the fantasy of being able to write CPU-native code and have it magically "just work" on the GPU with high levels of efficiency is just that: a fantasy. The reality is that code must be specifically structured and organized to work efficiently on the GPU. Goal just makes this process relatively clean and efficient and easy to read, with a minimum of extra boilerplate. The resulting code should be easily understood by anyone familiar with the Go language, even if that isn't the way you would have written it in the first place. The reward is that you can get highly efficient results with significant GPU-accelerated speedups that works on _any platform_, including the web and mobile phones, all with a single easy-to-read codebase. + +## Gosl: go shader language + The [gosl](gosl) (_Go shader language_) package within Goal does the heavy lifting of translating Go code into WGSL shader language code that can run on the WebGPU, and generally manages most of the gpu-specific functionality. It has various important functions defined in the `gosl.` package space, and a number of `//gosl:` comment directives described below, that make everything work. Meanwhile, the `goal build` command provides an outer-loop of orchestration and math code transpiling prior to handing off to gosl to run on the relevant files. @@ -147,6 +151,12 @@ Furthermore: You have to use _indexes_ into arrays exclusively. Thus, some of the data structures you may need to copy up to the GPU include index variables that determine how to access other variables. TODO: do we need helpers for any of this? +# Examples + +A large and complex biologically-based neural network simulation framework called [axon](https://github.com/emer/axon) has been implemented using `gosl`, allowing 1000's of lines of equations and data structures to run through standard Go on the CPU, and accelerated significantly on the GPU. This allows efficient debugging and unit testing of the code in Go, whereas debugging on the GPU is notoriously difficult. + +# TODO + ## Optimization can run naga on wgsl code to get wgsl code out, but it doesn't seem to do much dead code elimination: https://github.com/gfx-rs/wgpu/tree/trunk/naga diff --git a/goal/gosl/README.md b/goal/gosl/README.md index 415053fddd..3dc0b3695b 100644 --- a/goal/gosl/README.md +++ b/goal/gosl/README.md @@ -2,12 +2,10 @@ `gosl` implements _Go as a shader language_ for GPU compute shaders (using [WebGPU](https://www.w3.org/TR/webgpu/)), **enabling standard Go code to run on the GPU**. -`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go langauge. See the [GPU](../GPU.md) documentation for an overview. +`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go langauge. See the [GPU](../GPU.md) documentation for an overview. The `goal` command processes more compact math-mode expressions that The relevant subsets of Go code are specifically marked using `//gosl:` comment directives, and this code must only use basic expressions and concrete types that will compile correctly in a shader (see [Restrictions](#restrictions) below). Method functions and pass-by-reference pointer arguments to `struct` types are supported and incur no additional compute cost due to inlining (see notes below for more detail). -A large and complex biologically-based neural network simulation framework called [axon](https://github.com/emer/axon) has been implemented using `gosl`, allowing 1000's of lines of equations and data structures to run through standard Go on the CPU, and accelerated significantly on the GPU. This allows efficient debugging and unit testing of the code in Go, whereas debugging on the GPU is notoriously difficult. - See [examples/basic](examples/basic) and [rand](examples/rand) for examples, using the [gpu](../../gpu) GPU compute shader system. It is also possible in principle to use gosl to generate shader files for any other WebGPU application, but this has not been tested. You must also install `goimports` which is used on the extracted subset of Go code, to get the imports right: @@ -17,7 +15,7 @@ $ go install golang.org/x/tools/cmd/goimports@latest To install the `gosl` command, do: ```bash -$ go install cogentcore.org/core/vgpu/gosl@latest +$ go install cogentcore.org/core/gpu/gosl@latest ``` In your Go code, use these comment directives: diff --git a/goal/gosl/cmd/gosl/gosl.go b/goal/gosl/cmd/gosl/gosl.go new file mode 100644 index 0000000000..9d71930175 --- /dev/null +++ b/goal/gosl/cmd/gosl/gosl.go @@ -0,0 +1,16 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "cogentcore.org/core/cli" + "cogentcore.org/core/goal/gosl/parse" +) + +func main() { //types:skip + opts := cli.DefaultOptions("gosl", "Go as a shader language converts Go code to WGSL WebGPU shader code, which can be run on the GPU through WebGPU.") + cfg := &parse.Config{} + cli.Run(opts, cfg, parse.Run) +} diff --git a/goal/gosl/examples/basic/shaders/fastexp.wgsl b/goal/gosl/examples/basic/shaders/fastexp.wgsl deleted file mode 100644 index c5f278921b..0000000000 --- a/goal/gosl/examples/basic/shaders/fastexp.wgsl +++ /dev/null @@ -1,14 +0,0 @@ - -// FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph -// It does not have any of the sanity checking of a standard method -- returns -// nonsense when arg is out of range. Runs in 2.23ns vs. 6.3ns for 64bit which is faster -// than exp actually. -fn FastExp(x: f32) -> f32 { - if (x <= -88.02969) { // this doesn't add anything and -exp is main use-case anyway - return f32(0.0); - } - var i = i32(12102203*x) + i32(127)*(i32(1)<<23); - var m = i >> 7 & 0xFFFF; // copy mantissa - i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11); - return bitcast(u32(i)); -} diff --git a/goal/gosl/gosl.go b/goal/gosl/gosl.go deleted file mode 100644 index e7638857d2..0000000000 --- a/goal/gosl/gosl.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2022, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// copied and heavily edited from go src/cmd/gofmt/gofmt.go: - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package main - -import ( - "flag" - "fmt" - "os" - "strings" - - "cogentcore.org/core/gpu/gosl/slprint" -) - -// flags -var ( - outDir = flag.String("out", "shaders", "output directory for shader code, relative to where gosl is invoked; must not be an empty string") - excludeFunctions = flag.String("exclude", "Update,Defaults", "comma-separated list of names of functions to exclude from exporting to WGSL") - keepTmp = flag.Bool("keep", false, "keep temporary converted versions of the source files, for debugging") - debug = flag.Bool("debug", false, "enable debugging messages while running") - excludeFunctionMap = map[string]bool{} -) - -// Keep these in sync with go/format/format.go. -const ( - tabWidth = 8 - printerMode = slprint.UseSpaces | slprint.TabIndent | printerNormalizeNumbers - - // printerNormalizeNumbers means to canonicalize number literal prefixes - // and exponents while printing. See https://golang.org/doc/go1.13#gosl. - // - // This value is defined in go/printer specifically for go/format and cmd/gosl. - printerNormalizeNumbers = 1 << 30 -) - -func usage() { - fmt.Fprintf(os.Stderr, "usage: gosl [flags] [path ...]\n") - flag.PrintDefaults() -} - -func main() { - flag.Usage = usage - flag.Parse() - goslMain() -} - -func GoslArgs() { - exs := *excludeFunctions - ex := strings.Split(exs, ",") - for _, fn := range ex { - excludeFunctionMap[fn] = true - } -} - -func goslMain() { - if *outDir == "" { - fmt.Println("Must have an output directory (default shaders), specified in -out arg") - os.Exit(1) - return - } - - if gomod := os.Getenv("GO111MODULE"); gomod == "off" { - fmt.Println("gosl only works in go modules mode, but GO111MODULE=off") - os.Exit(1) - return - } - - os.MkdirAll(*outDir, 0755) - RemoveGenFiles(*outDir) - - args := flag.Args() - if len(args) == 0 { - fmt.Printf("at least one file name must be passed\n") - return - } - - GoslArgs() - ProcessFiles(args) -} diff --git a/goal/gosl/slprint/comment.go b/goal/gosl/parse/comment.go similarity index 93% rename from goal/gosl/slprint/comment.go rename to goal/gosl/parse/comment.go index f97a9a2084..345dc31373 100644 --- a/goal/gosl/slprint/comment.go +++ b/goal/gosl/parse/comment.go @@ -1,8 +1,15 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is largely copied from the Go source, +// src/go/printer/comment.go: + // Copyright 2022 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package slprint +package parse import ( "go/ast" diff --git a/goal/gosl/parse/config.go b/goal/gosl/parse/config.go new file mode 100644 index 0000000000..4a9ef1d98f --- /dev/null +++ b/goal/gosl/parse/config.go @@ -0,0 +1,76 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parse + +import ( + "errors" + "os" + "strings" +) + +//go:generate core generate -add-types -add-funcs + +// Keep these in sync with go/format/format.go. +const ( + tabWidth = 8 + printerMode = UseSpaces | TabIndent | printerNormalizeNumbers + + // printerNormalizeNumbers means to canonicalize number literal prefixes + // and exponents while printing. See https://golang.org/doc/go1.13#gosl. + // + // This value is defined in go/printer specifically for go/format and cmd/gosl. + printerNormalizeNumbers = 1 << 30 +) + +// Config has the configuration info for the gosl system +type Config struct { + + // Inputs are the input files to run/compile. + Inputs []string `posarg:"all" required:"+"` + + // Output is the output directory for shader code, + // relative to where gosl is invoked; must not be an empty string. + Output string `flag:"out" default:"shaders"` + + // Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL. + Exclude string `default:"Update,Defaults"` + + // Keep keeps temporary converted versions of the source files, for debugging. + Keep bool + + // Debug enables debugging messages while running. + Debug bool + + // ExcludeMap is the compiled map of functions to exclude. + ExcludeMap map[string]bool +} + +func (cfg *Config) Update() { + cfg.ExcludeMap = make(map[string]bool) + ex := strings.Split(cfg.Exclude, ",") + for _, fn := range ex { + cfg.ExcludeMap[fn] = true + } +} + +//cli:cmd -root +func Run(cfg *Config) error { //types:add + if cfg.Output == "" { + err := errors.New("Must have an output directory (default 'shaders'), specified in -out arg") + return err + } + + if gomod := os.Getenv("GO111MODULE"); gomod == "off" { + err := errors.New("gosl only works in go modules mode, but GO111MODULE=off") + return err + } + + os.MkdirAll(cfg.Output, 0755) + RemoveGenFiles(cfg.Output) + + cfg.Update() + _, err := cfg.ProcessFiles() + return err +} diff --git a/goal/gosl/extract.go b/goal/gosl/parse/extract.go similarity index 94% rename from goal/gosl/extract.go rename to goal/gosl/parse/extract.go index 5a41aa1649..6ac1ba955f 100644 --- a/goal/gosl/extract.go +++ b/goal/gosl/parse/extract.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package parse import ( "bytes" @@ -15,7 +15,7 @@ import ( "slices" ) -func ReadFileLines(fn string) ([][]byte, error) { +func (cfg *Config) ReadFileLines(fn string) ([][]byte, error) { nl := []byte("\n") buf, err := os.ReadFile(fn) if err != nil { @@ -27,7 +27,7 @@ func ReadFileLines(fn string) ([][]byte, error) { } // Extracts comment-directive tagged regions from .go files -func ExtractGoFiles(files []string) map[string][]byte { +func (cfg *Config) ExtractGoFiles(files []string) map[string][]byte { sls := map[string][][]byte{} key := []byte("//gosl:") start := []byte("start") @@ -41,7 +41,7 @@ func ExtractGoFiles(files []string) map[string][]byte { if !strings.HasSuffix(fn, ".go") { continue } - lines, err := ReadFileLines(fn) + lines, err := cfg.ReadFileLines(fn) if err != nil { continue } @@ -97,7 +97,7 @@ func ExtractGoFiles(files []string) map[string][]byte { rsls := make(map[string][]byte) for fn, lns := range sls { - outfn := filepath.Join(*outDir, fn+".go") + outfn := filepath.Join(cfg.Output, fn+".go") olns := [][]byte{} olns = append(olns, []byte("package main")) olns = append(olns, []byte(`import ( @@ -128,7 +128,7 @@ func ExtractGoFiles(files []string) map[string][]byte { // ExtractWGSL extracts the WGSL code embedded within .Go files. // Returns true if WGSL contains a void main( function. -func ExtractWGSL(buf []byte) ([]byte, bool) { +func (cfg *Config) ExtractWGSL(buf []byte) ([]byte, bool) { key := []byte("//gosl:") wgsl := []byte("wgsl") nowgsl := []byte("nowgsl") diff --git a/goal/gosl/files.go b/goal/gosl/parse/files.go similarity index 89% rename from goal/gosl/files.go rename to goal/gosl/parse/files.go index c37feaf0e6..b6e6c18692 100644 --- a/goal/gosl/files.go +++ b/goal/gosl/parse/files.go @@ -1,8 +1,8 @@ -// Copyright (c) 2022, Cogent Core. All rights reserved. +// Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package parse import ( "fmt" @@ -35,7 +35,7 @@ func IsSPVFile(f fs.DirEntry) bool { return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".spv") && !f.IsDir() } -func AddFile(fn string, fls []string, procd map[string]bool) []string { +func (cfg *Config) AddFile(fn string, fls []string, procd map[string]bool) []string { if _, has := procd[fn]; has { return fls } @@ -60,7 +60,7 @@ func AddFile(fn string, fls []string, procd map[string]bool) []string { // FilesFromPaths processes all paths and returns a full unique list of files // for subsequent processing. -func FilesFromPaths(paths []string) []string { +func (cfg *Config) FilesFromPaths(paths []string) []string { fls := make([]string, 0, len(paths)) procd := make(map[string]bool) for _, path := range paths { @@ -86,20 +86,20 @@ func FilesFromPaths(paths []string) []string { if fl != "" { for _, gf := range gofls { if strings.HasSuffix(gf, fl) { - fls = AddFile(gf, fls, procd) + fls = cfg.AddFile(gf, fls, procd) // fmt.Printf("added file: %s from package: %s\n", gf, path) break } } } else { for _, gf := range gofls { - fls = AddFile(gf, fls, procd) + fls = cfg.AddFile(gf, fls, procd) // fmt.Printf("added file: %s from package: %s\n", gf, path) } } case !info.IsDir(): path := path - fls = AddFile(path, fls, procd) + fls = cfg.AddFile(path, fls, procd) default: // Directories are walked, ignoring non-Go, non-WGSL files. err := filepath.WalkDir(path, func(path string, f fs.DirEntry, err error) error { @@ -110,7 +110,7 @@ func FilesFromPaths(paths []string) []string { if err != nil { return nil } - fls = AddFile(path, fls, procd) + fls = cfg.AddFile(path, fls, procd) return nil }) if err != nil { @@ -139,8 +139,8 @@ func CopyFile(src, dst string) error { // CopyPackageFile copies given file name from given package path // into the current output directory. // e.g., "slrand.wgsl", "cogentcore.org/core/gpu/gosl/slrand" -func CopyPackageFile(fnm, packagePath string) error { - tofn := filepath.Join(*outDir, fnm) +func (cfg *Config) CopyPackageFile(fnm, packagePath string) error { + tofn := filepath.Join(cfg.Output, fnm) pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles}, packagePath) if err != nil { fmt.Println(err) diff --git a/goal/gosl/slprint/gobuild.go b/goal/gosl/parse/gobuild.go similarity index 94% rename from goal/gosl/slprint/gobuild.go rename to goal/gosl/parse/gobuild.go index fd0c4b4002..e5ad8bc847 100644 --- a/goal/gosl/slprint/gobuild.go +++ b/goal/gosl/parse/gobuild.go @@ -1,8 +1,15 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is largely copied from the Go source, +// src/go/printer/gobuild.go: + // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package slprint +package parse import ( "go/build/constraint" diff --git a/goal/gosl/gosl_test.go b/goal/gosl/parse/gosl_test.go similarity index 99% rename from goal/gosl/gosl_test.go rename to goal/gosl/parse/gosl_test.go index 8a8ec76b5a..ea07965ca9 100644 --- a/goal/gosl/gosl_test.go +++ b/goal/gosl/parse/gosl_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package parse import ( "bytes" diff --git a/goal/gosl/slprint/nodes.go b/goal/gosl/parse/nodes.go similarity index 99% rename from goal/gosl/slprint/nodes.go rename to goal/gosl/parse/nodes.go index 13ace493a0..2dad0f9991 100644 --- a/goal/gosl/slprint/nodes.go +++ b/goal/gosl/parse/nodes.go @@ -1,3 +1,10 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is largely copied from the Go source, +// src/go/printer/nodes.go: + // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,7 +13,7 @@ // expressions, statements, declarations, and files. It uses // the print functionality implemented in printer.go. -package slprint +package parse import ( "fmt" @@ -1102,7 +1109,7 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { } case *ast.BasicLit: - if p.Config.Mode&normalizeNumbers != 0 { + if p.PrintConfig.Mode&normalizeNumbers != 0 { x = normalizedNumber(x) } p.print(x) @@ -2322,7 +2329,7 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) { // nodeSize computation must be independent of particular // style so that we always get the same decision; print // in RawFormat - cfg := Config{Mode: RawFormat} + cfg := PrintConfig{Mode: RawFormat} var counter sizeCounter if err := cfg.fprint(&counter, p.pkg, n, p.nodeSizes); err != nil { return diff --git a/goal/gosl/slprint/printer.go b/goal/gosl/parse/printer.go similarity index 97% rename from goal/gosl/slprint/printer.go rename to goal/gosl/parse/printer.go index 0fa383be44..3cefd1ef7e 100644 --- a/goal/gosl/slprint/printer.go +++ b/goal/gosl/parse/printer.go @@ -1,8 +1,15 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is largely copied from the Go source, +// src/go/printer/printer.go: + // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package slprint +package parse import ( "fmt" @@ -54,7 +61,7 @@ type commentInfo struct { type printer struct { // Configuration (does not change after initialization) - Config + PrintConfig fset *token.FileSet pkg *packages.Package // gosl: extra @@ -225,7 +232,7 @@ func (p *printer) writeLineDirective(pos token.Position) { func (p *printer) writeIndent() { // use "hard" htabs - indentation columns // must not be discarded by the tabwriter - n := p.Config.Indent + p.indent // include base indentation + n := p.PrintConfig.Indent + p.indent // include base indentation for i := 0; i < n; i++ { p.output = append(p.output, '\t') } @@ -287,7 +294,7 @@ func (p *printer) writeByte(ch byte, n int) { // printer benchmark by up to 10%. func (p *printer) writeString(pos token.Position, s string, isLit bool) { if p.out.Column == 1 { - if p.Config.Mode&SourcePos != 0 { + if p.PrintConfig.Mode&SourcePos != 0 { p.writeLineDirective(pos) } p.writeIndent() @@ -1321,8 +1328,8 @@ const ( normalizeNumbers Mode = 1 << 30 ) -// A Config node controls the output of Fprint. -type Config struct { +// A PrintConfig node controls the output of Fprint. +type PrintConfig struct { Mode Mode // default: 0 Tabwidth int // default: 8 Indent int // default: 0 (all code is indented at least by this much) @@ -1342,18 +1349,18 @@ var printerPool = sync.Pool{ }, } -func newPrinter(cfg *Config, pkg *packages.Package, nodeSizes map[ast.Node]int) *printer { +func newPrinter(cfg *PrintConfig, pkg *packages.Package, nodeSizes map[ast.Node]int) *printer { p := printerPool.Get().(*printer) *p = printer{ - Config: *cfg, - pkg: pkg, - fset: pkg.Fset, - pos: token.Position{Line: 1, Column: 1}, - out: token.Position{Line: 1, Column: 1}, - wsbuf: p.wsbuf[:0], - nodeSizes: nodeSizes, - cachedPos: -1, - output: p.output[:0], + PrintConfig: *cfg, + pkg: pkg, + fset: pkg.Fset, + pos: token.Position{Line: 1, Column: 1}, + out: token.Position{Line: 1, Column: 1}, + wsbuf: p.wsbuf[:0], + nodeSizes: nodeSizes, + cachedPos: -1, + output: p.output[:0], } return p } @@ -1368,7 +1375,7 @@ func (p *printer) free() { } // fprint implements Fprint and takes a nodesSizes map for setting up the printer state. -func (cfg *Config) fprint(output io.Writer, pkg *packages.Package, node any, nodeSizes map[ast.Node]int) (err error) { +func (cfg *PrintConfig) fprint(output io.Writer, pkg *packages.Package, node any, nodeSizes map[ast.Node]int) (err error) { // print node p := newPrinter(cfg, pkg, nodeSizes) defer p.free() @@ -1431,14 +1438,14 @@ type CommentedNode struct { // Position information is interpreted relative to the file set fset. // The node type must be *[ast.File], *[CommentedNode], [][ast.Decl], [][ast.Stmt], // or assignment-compatible to [ast.Expr], [ast.Decl], [ast.Spec], or [ast.Stmt]. -func (cfg *Config) Fprint(output io.Writer, pkg *packages.Package, node any) error { +func (cfg *PrintConfig) Fprint(output io.Writer, pkg *packages.Package, node any) error { return cfg.fprint(output, pkg, node, make(map[ast.Node]int)) } // Fprint "pretty-prints" an AST node to output. -// It calls [Config.Fprint] with default settings. +// It calls [PrintConfig.Fprint] with default settings. // Note that gofmt uses tabs for indentation but spaces for alignment; // use format.Node (package go/format) for output that matches gofmt. func Fprint(output io.Writer, pkg *packages.Package, node any) error { - return (&Config{Tabwidth: 8}).Fprint(output, pkg, node) + return (&PrintConfig{Tabwidth: 8}).Fprint(output, pkg, node) } diff --git a/goal/gosl/process.go b/goal/gosl/parse/process.go similarity index 81% rename from goal/gosl/process.go rename to goal/gosl/parse/process.go index de818b4903..ad702cb2ff 100644 --- a/goal/gosl/process.go +++ b/goal/gosl/parse/process.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package parse import ( "bytes" @@ -18,16 +18,16 @@ import ( "strings" "cogentcore.org/core/base/errors" + "cogentcore.org/core/goal/gosl/alignsl" "cogentcore.org/core/gpu" - "cogentcore.org/core/gpu/gosl/alignsl" - "cogentcore.org/core/gpu/gosl/slprint" "golang.org/x/tools/go/packages" ) // does all the file processing -func ProcessFiles(paths []string) (map[string][]byte, error) { - fls := FilesFromPaths(paths) - gosls := ExtractGoFiles(fls) // extract Go files to shader/*.go +func (cfg *Config) ProcessFiles() (map[string][]byte, error) { + paths := cfg.Inputs + fls := cfg.FilesFromPaths(paths) + gosls := cfg.ExtractGoFiles(fls) // extract Go files to shader/*.go wgslFiles := []string{} for _, fn := range fls { @@ -36,7 +36,7 @@ func ProcessFiles(paths []string) (map[string][]byte, error) { } } - pf := "./" + *outDir + pf := "./" + cfg.Output pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesSizes | packages.NeedTypesInfo}, pf) if err != nil { log.Println(err) @@ -69,7 +69,7 @@ func ProcessFiles(paths []string) (map[string][]byte, error) { sltypeCopied := false for fn := range gosls { gofn := fn + ".go" - if *debug { + if cfg.Debug { fmt.Printf("###################################\nProcessing Go file: %s\n", gofn) } @@ -90,32 +90,32 @@ func ProcessFiles(paths []string) (map[string][]byte, error) { } var buf bytes.Buffer - cfg := slprint.Config{Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: excludeFunctionMap} - cfg.Fprint(&buf, pkg, afile) + pcfg := PrintConfig{Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: cfg.ExcludeMap} + pcfg.Fprint(&buf, pkg, afile) // ioutil.WriteFile(filepath.Join(*outDir, fn+".tmp"), buf.Bytes(), 0644) slfix, hasSltype, hasSlrand := SlEdits(buf.Bytes()) if hasSlrand && !slrandCopied { hasSltype = true - if *debug { + if cfg.Debug { fmt.Printf("\tcopying slrand.wgsl to shaders\n") } - CopyPackageFile("slrand.wgsl", "cogentcore.org/core/gpu/gosl/slrand") + cfg.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/gpu/gosl/slrand") slrandCopied = true } if hasSltype && !sltypeCopied { - if *debug { + if cfg.Debug { fmt.Printf("\tcopying sltype.wgsl to shaders\n") } - CopyPackageFile("sltype.wgsl", "cogentcore.org/core/gpu/gosl/sltype") + cfg.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/gpu/gosl/sltype") sltypeCopied = true } - exsl, hasMain := ExtractWGSL(slfix) + exsl, hasMain := cfg.ExtractWGSL(slfix) gosls[fn] = exsl if hasMain { needsCompile[fn] = true } - if !*keepTmp { + if !cfg.Keep { os.Remove(fpos.Filename) } @@ -136,7 +136,7 @@ func ProcessFiles(paths []string) (map[string][]byte, error) { break } - slfn := filepath.Join(*outDir, fn+".wgsl") + slfn := filepath.Join(cfg.Output, fn+".wgsl") ioutil.WriteFile(slfn, exsl, 0644) } @@ -153,20 +153,20 @@ func ProcessFiles(paths []string) (map[string][]byte, error) { continue } _, slfno := filepath.Split(slfn) // could be in a subdir - tofn := filepath.Join(*outDir, slfno) + tofn := filepath.Join(cfg.Output, slfno) CopyFile(slfn, tofn) fn := strings.TrimSuffix(slfno, ".wgsl") needsCompile[fn] = true // assume any standalone wgsl is a main } for fn := range needsCompile { - CompileFile(fn + ".wgsl") + cfg.CompileFile(fn + ".wgsl") } return gosls, nil } -func CompileFile(fn string) error { - dir, _ := filepath.Abs(*outDir) +func (cfg *Config) CompileFile(fn string) error { + dir, _ := filepath.Abs(cfg.Output) fsys := os.DirFS(dir) b, err := fs.ReadFile(fsys, fn) if errors.Log(err) != nil { diff --git a/goal/gosl/sledits.go b/goal/gosl/parse/sledits.go similarity index 99% rename from goal/gosl/sledits.go rename to goal/gosl/parse/sledits.go index b9ea5e2318..e66b8758ac 100644 --- a/goal/gosl/sledits.go +++ b/goal/gosl/parse/sledits.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package main +package parse import ( "bytes" diff --git a/goal/gosl/testdata/basic.go b/goal/gosl/parse/testdata/basic.go similarity index 100% rename from goal/gosl/testdata/basic.go rename to goal/gosl/parse/testdata/basic.go diff --git a/goal/gosl/testdata/basic.golden b/goal/gosl/parse/testdata/basic.golden similarity index 100% rename from goal/gosl/testdata/basic.golden rename to goal/gosl/parse/testdata/basic.golden diff --git a/goal/gosl/parse/typegen.go b/goal/gosl/parse/typegen.go new file mode 100644 index 0000000000..54bf0288c4 --- /dev/null +++ b/goal/gosl/parse/typegen.go @@ -0,0 +1,121 @@ +// Code generated by "core generate -add-types -add-funcs"; DO NOT EDIT. + +package parse + +import ( + "cogentcore.org/core/types" +) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system", Fields: []types.Field{{Name: "Inputs", Doc: "Inputs are the input files to run/compile."}, {Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.exprListMode", IDName: "expr-list-mode"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.paramMode", IDName: "param-mode"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.sizeCounter", IDName: "size-counter", Doc: "sizeCounter is an io.Writer which counts the number of bytes written,\nas well as whether a newline character was seen.", Fields: []types.Field{{Name: "hasNewline"}, {Name: "size"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.whiteSpace", IDName: "white-space"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.pmode", IDName: "pmode", Doc: "A pmode value represents the current printer mode."}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.commentInfo", IDName: "comment-info", Fields: []types.Field{{Name: "cindex"}, {Name: "comment"}, {Name: "commentOffset"}, {Name: "commentNewline"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.printer", IDName: "printer", Embeds: []types.Field{{Name: "PrintConfig", Doc: "Configuration (does not change after initialization)"}, {Name: "commentInfo", Doc: "Information about p.comments[p.cindex]; set up by nextComment."}}, Fields: []types.Field{{Name: "fset"}, {Name: "pkg"}, {Name: "output", Doc: "Current state"}, {Name: "indent"}, {Name: "level"}, {Name: "mode"}, {Name: "endAlignment"}, {Name: "impliedSemi"}, {Name: "lastTok"}, {Name: "prevOpen"}, {Name: "wsbuf"}, {Name: "goBuild"}, {Name: "plusBuild"}, {Name: "pos", Doc: "Positions\nThe out position differs from the pos position when the result\nformatting differs from the source formatting (in the amount of\nwhite space). If there's a difference and SourcePos is set in\nConfigMode, //line directives are used in the output to restore\noriginal source positions for a reader."}, {Name: "out"}, {Name: "last"}, {Name: "linePtr"}, {Name: "sourcePosErr"}, {Name: "comments", Doc: "The list of all source comments, in order of appearance."}, {Name: "useNodeComments"}, {Name: "nodeSizes", Doc: "Cache of already computed node sizes."}, {Name: "cachedPos", Doc: "Cache of most recently computed line position."}, {Name: "cachedLine"}, {Name: "curPtrArgs", Doc: "current arguments to function that are pointers and thus need dereferencing\nwhen accessing fields"}, {Name: "curMethRecv"}, {Name: "curReturnType"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.trimmer", IDName: "trimmer", Doc: "A trimmer is an io.Writer filter for stripping tabwriter.Escape\ncharacters, trailing blanks and tabs, and for converting formfeed\nand vtab characters into newlines and htabs (in case no tabwriter\nis used). Text bracketed by tabwriter.Escape characters is passed\nthrough unchanged.", Fields: []types.Field{{Name: "output"}, {Name: "state"}, {Name: "space"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Mode", IDName: "mode", Doc: "A Mode value is a set of flags (or 0). They control printing."}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.PrintConfig", IDName: "print-config", Doc: "A PrintConfig node controls the output of Fprint.", Fields: []types.Field{{Name: "Mode"}, {Name: "Tabwidth"}, {Name: "Indent"}, {Name: "ExcludeFunctions"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.CommentedNode", IDName: "commented-node", Doc: "A CommentedNode bundles an AST node and corresponding comments.\nIt may be provided as argument to any of the [Fprint] functions.", Fields: []types.Field{{Name: "Node"}, {Name: "Comments"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Replace", IDName: "replace", Fields: []types.Field{{Name: "From"}, {Name: "To"}}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.formatDocComment", Doc: "formatDocComment reformats the doc comment list,\nreturning the canonical formatting.", Args: []string{"list"}, Returns: []string{"Comment"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isDirective", Doc: "isDirective reports whether c is a comment directive.\nSee go.dev/issue/37974.\nThis code is also in go/ast.", Args: []string{"c"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.allStars", Doc: "allStars reports whether text is the interior of an\nold-style /* */ comment with a star at the start of each line.", Args: []string{"text"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.Run", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsGoFile", Args: []string{"f"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsWGSLFile", Args: []string{"f"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsSPVFile", Args: []string{"f"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.CopyFile", Args: []string{"src", "dst"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.RemoveGenFiles", Doc: "RemoveGenFiles removes .go, .wgsl, .spv files in shader generated dir", Args: []string{"dir"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.appendLines", Doc: "appendLines is like append(x, y...)\nbut it avoids creating doubled blank lines,\nwhich would not be gofmt-standard output.\nIt assumes that only whole blocks of lines are being appended,\nnot line fragments.", Args: []string{"x", "y"}, Returns: []string{"[]byte"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isNL", Args: []string{"b"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.combinesWithName", Doc: "combinesWithName reports whether a name followed by the expression x\nsyntactically combines to another valid (value) expression. For instance\nusing *T for x, \"name *T\" syntactically appears as the expression x*T.\nOn the other hand, using P|Q or *P|~Q for x, \"name P|Q\" or name *P|~Q\"\ncannot be combined into a valid (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isTypeElem", Doc: "isTypeElem reports whether x is a (possibly parenthesized) type element expression.\nThe result is false if x could be a type element OR an ordinary (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.identListSize", Args: []string{"list", "maxSize"}, Returns: []string{"size"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.walkBinary", Args: []string{"e"}, Returns: []string{"has4", "has5", "maxProblem"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.cutoff", Args: []string{"e", "depth"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.diffPrec", Args: []string{"expr", "prec"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.reduceDepth", Args: []string{"depth"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isBinary", Args: []string{"expr"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.normalizedNumber", Doc: "normalizedNumber rewrites base prefixes and exponents\nof numbers to use lower-case letters (0X123 to 0x123 and 1.2E3 to 1.2e3),\nand removes leading 0's from integer imaginary literals (0765i to 765i).\nIt leaves hexadecimal digits alone.\n\nnormalizedNumber doesn't modify the ast.BasicLit value lit points to.\nIf lit is not a number or a number in canonical format already,\nlit is returned as is. Otherwise a new ast.BasicLit is created.", Args: []string{"lit"}, Returns: []string{"BasicLit"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.fieldByName", Args: []string{"st", "name"}, Returns: []string{"Var"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getLocalTypeName", Args: []string{"typ"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getStructType", Args: []string{"typ"}, Returns: []string{"Struct", "error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isTypeName", Args: []string{"x"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.stripParens", Args: []string{"x"}, Returns: []string{"Expr"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.stripParensAlways", Args: []string{"x"}, Returns: []string{"Expr"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.keepTypeColumn", Doc: "The keepTypeColumn function determines if the type column of a series of\nconsecutive const or var declarations must be kept, or if initialization\nvalues (V) can be placed in the type column (T) instead. The i'th entry\nin the result slice is true if the type column in spec[i] must be kept.\n\nFor example, the declaration:\n\n\t\tconst (\n\t\t\tfoobar int = 42 // comment\n\t\t\tx = 7 // comment\n\t\t\tfoo\n\t bar = 991\n\t\t)\n\nleads to the type/values matrix below. A run of value columns (V) can\nbe moved into the type column if there is no type for any of the values\nin that column (we only move entire columns so that they align properly).\n\n\t\tmatrix formatted result\n\t matrix\n\t\tT V -> T V -> true there is a T and so the type\n\t\t- V - V true column must be kept\n\t\t- - - - false\n\t\t- V V - false V is moved into T column", Args: []string{"specs"}, Returns: []string{"[]bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.sanitizeImportPath", Args: []string{"lit"}, Returns: []string{"BasicLit"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.declToken", Args: []string{"decl"}, Returns: []string{"tok"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isBlank", Doc: "Returns true if s contains only white space\n(only tabs and blanks can appear in the printer's context).", Args: []string{"s"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.commonPrefix", Doc: "commonPrefix returns the common prefix of a and b.", Args: []string{"a", "b"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.trimRight", Doc: "trimRight returns s with trailing whitespace removed.", Args: []string{"s"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.stripCommonPrefix", Doc: "stripCommonPrefix removes a common prefix from /*-style comment lines (unless no\ncomment line is indented, all but the first line have some form of space prefix).\nThe prefix is computed using heuristics such that is likely that the comment\ncontents are nicely laid out after re-printing each line using the printer's\ncurrent indentation.", Args: []string{"lines"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.nlimit", Doc: "nlimit limits n to maxNewlines.", Args: []string{"n"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.mayCombine", Args: []string{"prev", "next"}, Returns: []string{"b"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getDoc", Doc: "getDoc returns the ast.CommentGroup associated with n, if any.", Args: []string{"n"}, Returns: []string{"CommentGroup"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getLastComment", Args: []string{"n"}, Returns: []string{"CommentGroup"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.newPrinter", Args: []string{"cfg", "pkg", "nodeSizes"}, Returns: []string{"printer"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.Fprint", Doc: "Fprint \"pretty-prints\" an AST node to output.\nIt calls [PrintConfig.Fprint] with default settings.\nNote that gofmt uses tabs for indentation but spaces for alignment;\nuse format.Node (package go/format) for output that matches gofmt.", Args: []string{"output", "pkg", "node"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.MoveLines", Doc: "MoveLines moves the st,ed region to 'to' line", Args: []string{"lines", "to", "st", "ed"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlEdits", Doc: "SlEdits performs post-generation edits for wgsl\n* moves wgsl segments around, e.g., methods\ninto their proper classes\n* fixes printf, slice other common code\nreturns true if a slrand. or sltype. prefix was found,\ndriveing copying of those files.", Args: []string{"src"}, Returns: []string{"[]byte", "bool", "bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.MathReplaceAll", Args: []string{"mat", "ln"}, Returns: []string{"[]byte"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlEditsReplace", Doc: "SlEditsReplace replaces Go with equivalent WGSL code\nreturns true if has slrand. or sltype.\nto auto include that header file if so.", Args: []string{"lines"}, Returns: []string{"bool", "bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlBoolReplace", Doc: "SlBoolReplace replaces all the slbool methods with literal int32 expressions.", Args: []string{"lines"}}) From d801343106641913a13d89fafd5585dea567ffe1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 04:22:12 -0700 Subject: [PATCH 184/311] gosl major reorganization for package based processing --- goal/GPU.md | 11 +- goal/gosl/examples/basic/compute.go | 41 ++-- goal/gosl/examples/basic/main.go | 51 +++-- goal/gosl/examples/basic/shaders/basic.wgsl | 52 ----- goal/gosl/parse/config.go | 41 +--- goal/gosl/parse/extract.go | 171 +++++++-------- goal/gosl/parse/files.go | 221 +++++++++----------- goal/gosl/parse/process.go | 143 ++++++------- goal/gosl/parse/state.go | 105 ++++++++++ goal/gosl/parse/typegen.go | 24 ++- 10 files changed, 426 insertions(+), 434 deletions(-) delete mode 100644 goal/gosl/examples/basic/shaders/basic.wgsl create mode 100644 goal/gosl/parse/state.go diff --git a/goal/GPU.md b/goal/GPU.md index 8e50bb3853..800fa7afe8 100644 --- a/goal/GPU.md +++ b/goal/GPU.md @@ -46,7 +46,7 @@ The GPU-specific computation is organized into some (hopefully small) number of ```Go for i := range parallel(data) { - MyCompute(i) + Compute(i) } ``` @@ -60,7 +60,14 @@ Even though the GPU kernels must each be compiled separately into a single disti The GPU code can only handle a highly restricted _subset_ of Go code, with data structures having strict alignment requirements, and no `string` or other composite variable-length data structures (maps, slices etc). Thus, the [gosl](gosl) package recognizes `//gosl:start` and `//gosl:end` comment directives surrounding the GPU-safe (and relevant) portions of the overall code. Any `.go` or `.goal` file can contribute GPU relevant code, including in other packages, and the gosl system automatically builds a shadow package-based set of `.wgsl` files accordingly. -> Each kernel must be written in a separate `.goal` file, marked with the `//gosl:kernel` directive at the top. There must be a "main" function entry point for each kernel, marked with the //gosl:main directive, which takes the index argument as shown in the code example above. +> Each kernel function is marked with a `//gosl:kernel` directive, and the name of the function is used to create the name of the GPU shader file. + +```Go +// Compute does the main computation. +func Compute(i int32) { //gosl:kernel + Params[0].IntegFromRaw(&Data[i]) +} +``` For CPU (regular Go) mode, the parallel `for range` loop as shown above translates into a `tensor.VectorizeThreaded` call of the named Go function. For GPU mode, it launches the kernel on the GPU. diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 27b43a7c27..e6d7984ddf 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022, Cogent Core. All rights reserved. +// Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,11 +6,16 @@ package main import "cogentcore.org/core/math32" -//gosl:wgsl basic -// #include "fastexp.wgsl" -//gosl:end basic +//gosl:start +//gosl:import "cogentcore.org/core/math32" -//gosl:start basic +var ( //gosl:vars + // Params are the parameters for the computation + Params []ParamStruct + + // Data is the data on which the computation operates + Data []DataStruct +) // DataStruct has the test data type DataStruct struct { @@ -47,7 +52,12 @@ func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) { ds.Exp = math32.FastExp(-ds.Integ) } -//gosl:end basic +// Compute does the main computation +func Compute(i int32) { //gosl:kernel + Params[0].IntegFromRaw(&Data[i]) +} + +//gosl:end // note: only core compute code needs to be in shader -- all init is done CPU-side @@ -59,22 +69,3 @@ func (ps *ParamStruct) Defaults() { func (ps *ParamStruct) Update() { ps.Dt = 1.0 / ps.Tau } - -//gosl:wgsl basic -/* -@group(0) @binding(0) -var Params: array; - -@group(0) @binding(1) -var Data: array; - -@compute -@workgroup_size(64) -fn main(@builtin(global_invocation_id) idx: vec3) { - var pars = Params[0]; - var data = Data[idx.x]; - ParamStruct_IntegFromRaw(&pars, &data); - Data[idx.x] = data; -} -*/ -//gosl:end basic diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index 87a6be2469..5bf40ca88d 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022, Cogent Core. All rights reserved. +// Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -17,7 +17,7 @@ import ( "cogentcore.org/core/gpu" ) -//go:generate ../../gosl cogentcore.org/core/math32/fastexp.go compute.go +//go:generate gosl . //go:embed shaders/basic.wgsl shaders/fastexp.wgsl var shaders embed.FS @@ -51,12 +51,12 @@ func main() { pvl := pv.Values.Values[0] dvl := dv.Values.Values[0] - pars := make([]ParamStruct, 1) - pars[0].Defaults() + Params = make([]ParamStruct, 1) + Params[0].Defaults() - cd := make([]DataStruct, n) - for i := range cd { - cd[i].Raw = rand.Float32() + Data = make([]DataStruct, n) + for i := range Data { + Data[i].Raw = rand.Float32() } sd := make([]DataStruct, n) @@ -66,32 +66,43 @@ func main() { cpuTmr := timer.Time{} cpuTmr.Start() - for i := range cd { - pars[0].IntegFromRaw(&cd[i]) + + gosl.UseCPU() + for i := range parallel(Data) { + Compute(i) } cpuTmr.Stop() + cd := Data + Data = sd + gpuFullTmr := timer.Time{} gpuFullTmr.Start() - gpu.SetValueFrom(pvl, pars) - gpu.SetValueFrom(dvl, sd) - - sgp.CreateReadBuffers() + gosl.ToGPU(Params, Data) + // gpu.SetValueFrom(pvl, pars) + // gpu.SetValueFrom(dvl, sd) + // sgp.CreateReadBuffers() gpuTmr := timer.Time{} gpuTmr.Start() - ce, _ := sy.BeginComputePass() - pl.Dispatch1D(ce, n, threads) - ce.End() - dvl.GPUToRead(sy.CommandEncoder) - sy.EndComputePass(ce) + gosl.UseGPU() + for i := range parallel(Data) { + Compute(i) + } + + // ce, _ := sy.BeginComputePass() + // pl.Dispatch1D(ce, n, threads) + // ce.End() + // dvl.GPUToRead(sy.CommandEncoder) + // sy.EndComputePass(ce) gpuTmr.Stop() - dvl.ReadSync() - gpu.ReadToBytes(dvl, sd) + gosl.FromCPU(Data) + // dvl.ReadSync() + // gpu.ReadToBytes(dvl, sd) gpuFullTmr.Stop() diff --git a/goal/gosl/examples/basic/shaders/basic.wgsl b/goal/gosl/examples/basic/shaders/basic.wgsl deleted file mode 100644 index 929012d598..0000000000 --- a/goal/gosl/examples/basic/shaders/basic.wgsl +++ /dev/null @@ -1,52 +0,0 @@ - -#include "fastexp.wgsl" - -// DataStruct has the test data -struct DataStruct { - - // raw value - Raw: f32, - - // integrated value - Integ: f32, - - // exp of integ - Exp: f32, - - // must pad to multiple of 4 floats for arrays - pad: f32, -} - -// ParamStruct has the test params -struct ParamStruct { - - // rate constant in msec - Tau: f32, - - // 1/Tau - Dt: f32, - - pad: f32, - pad1: f32, -} - -// IntegFromRaw computes integrated value from current raw value -fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr) { - (*ds).Integ += (*ps).Dt * ((*ds).Raw - (*ds).Integ); - (*ds).Exp = FastExp(-(*ds).Integ); -} - -@group(0) @binding(0) -var Params: array; - -@group(0) @binding(1) -var Data: array; - -@compute -@workgroup_size(64) -fn main(@builtin(global_invocation_id) idx: vec3) { - var pars = Params[0]; - var data = Data[idx.x]; - ParamStruct_IntegFromRaw(&pars, &data); - Data[idx.x] = data; -} diff --git a/goal/gosl/parse/config.go b/goal/gosl/parse/config.go index 4a9ef1d98f..f8a0068066 100644 --- a/goal/gosl/parse/config.go +++ b/goal/gosl/parse/config.go @@ -4,12 +4,6 @@ package parse -import ( - "errors" - "os" - "strings" -) - //go:generate core generate -add-types -add-funcs // Keep these in sync with go/format/format.go. @@ -24,12 +18,9 @@ const ( printerNormalizeNumbers = 1 << 30 ) -// Config has the configuration info for the gosl system +// Config has the configuration info for the gosl system. type Config struct { - // Inputs are the input files to run/compile. - Inputs []string `posarg:"all" required:"+"` - // Output is the output directory for shader code, // relative to where gosl is invoked; must not be an empty string. Output string `flag:"out" default:"shaders"` @@ -42,35 +33,11 @@ type Config struct { // Debug enables debugging messages while running. Debug bool - - // ExcludeMap is the compiled map of functions to exclude. - ExcludeMap map[string]bool -} - -func (cfg *Config) Update() { - cfg.ExcludeMap = make(map[string]bool) - ex := strings.Split(cfg.Exclude, ",") - for _, fn := range ex { - cfg.ExcludeMap[fn] = true - } } //cli:cmd -root func Run(cfg *Config) error { //types:add - if cfg.Output == "" { - err := errors.New("Must have an output directory (default 'shaders'), specified in -out arg") - return err - } - - if gomod := os.Getenv("GO111MODULE"); gomod == "off" { - err := errors.New("gosl only works in go modules mode, but GO111MODULE=off") - return err - } - - os.MkdirAll(cfg.Output, 0755) - RemoveGenFiles(cfg.Output) - - cfg.Update() - _, err := cfg.ProcessFiles() - return err + st := &State{} + st.Init(cfg) + return st.Run() } diff --git a/goal/gosl/parse/extract.go b/goal/gosl/parse/extract.go index 6ac1ba955f..037d10074a 100644 --- a/goal/gosl/parse/extract.go +++ b/goal/gosl/parse/extract.go @@ -6,134 +6,108 @@ package parse import ( "bytes" - "fmt" - "io/ioutil" - "os" "path/filepath" - "strings" "slices" ) -func (cfg *Config) ReadFileLines(fn string) ([][]byte, error) { - nl := []byte("\n") - buf, err := os.ReadFile(fn) - if err != nil { - fmt.Println(err) - return nil, err +// ExtractImports processes all the imported files and saves the corresponding +// .go files, and +func (st *State) ExtractImports() { + if len(st.Imports) == 0 { + return + } + st.ImportPackages = make(map[string]bool) + for impath := range st.Imports { + _, pkg := filepath.Split(impath) + st.ImportPackages[pkg] = true + } + for _, im := range st.Imports { + for fn, fl := range im { + fl.Lines = st.ExtractGosl(fl.Lines) + lns := st.AppendGoHeader(fl.Lines) + ifn := filepath.Join("imports", fn) + st.WriteFileLines(ifn, lns) + } } - lines := bytes.Split(buf, nl) - return lines, nil + pdir := "./" + filepath.Join(st.Config.Output, "imports") + st.ProcessDir(pdir) } -// Extracts comment-directive tagged regions from .go files -func (cfg *Config) ExtractGoFiles(files []string) map[string][]byte { - sls := map[string][][]byte{} +// ExtractGosl gosl comment-directive tagged regions from given file. +func (st *State) ExtractGosl(lines [][]byte) [][]byte { key := []byte("//gosl:") start := []byte("start") wgsl := []byte("wgsl") nowgsl := []byte("nowgsl") end := []byte("end") - nl := []byte("\n") - include := []byte("#include") + imp := []byte("import") - for _, fn := range files { - if !strings.HasSuffix(fn, ".go") { - continue - } - lines, err := cfg.ReadFileLines(fn) - if err != nil { - continue + inReg := false + inHlsl := false + inNoHlsl := false + var outLns [][]byte + for _, ln := range lines { + tln := bytes.TrimSpace(ln) + isKey := bytes.HasPrefix(tln, key) + var keyStr []byte + if isKey { + keyStr = tln[len(key):] + // fmt.Printf("key: %s\n", string(keyStr)) } - - inReg := false - inHlsl := false - inNoHlsl := false - var outLns [][]byte - slFn := "" - for _, ln := range lines { - tln := bytes.TrimSpace(ln) - isKey := bytes.HasPrefix(tln, key) - var keyStr []byte - if isKey { - keyStr = tln[len(key):] - // fmt.Printf("key: %s\n", string(keyStr)) + switch { + case inReg && isKey && bytes.HasPrefix(keyStr, end): + if inHlsl || inNoHlsl { + outLns = append(outLns, ln) } - switch { - case inReg && isKey && bytes.HasPrefix(keyStr, end): - if inHlsl || inNoHlsl { - outLns = append(outLns, ln) - } - sls[slFn] = outLns - inReg = false - inHlsl = false - inNoHlsl = false - case inReg: - for pkg := range LoadedPackageNames { // remove package prefixes - if !bytes.Contains(ln, include) { - ln = bytes.ReplaceAll(ln, []byte(pkg+"."), []byte{}) - } + inReg = false + inHlsl = false + inNoHlsl = false + case inReg: + for pkg := range st.ImportPackages { // remove package prefixes + if !bytes.Contains(ln, imp) { + ln = bytes.ReplaceAll(ln, []byte(pkg+"."), []byte{}) } - outLns = append(outLns, ln) - case isKey && bytes.HasPrefix(keyStr, start): - inReg = true - slFn = string(keyStr[len(start)+1:]) - outLns = sls[slFn] - case isKey && bytes.HasPrefix(keyStr, nowgsl): - inReg = true - inNoHlsl = true - slFn = string(keyStr[len(nowgsl)+1:]) - outLns = sls[slFn] - outLns = append(outLns, ln) // key to include self here - case isKey && bytes.HasPrefix(keyStr, wgsl): - inReg = true - inHlsl = true - slFn = string(keyStr[len(wgsl)+1:]) - outLns = sls[slFn] - outLns = append(outLns, ln) } + outLns = append(outLns, ln) + case isKey && bytes.HasPrefix(keyStr, start): + inReg = true + case isKey && bytes.HasPrefix(keyStr, nowgsl): + inReg = true + inNoHlsl = true + outLns = append(outLns, ln) // key to include self here + case isKey && bytes.HasPrefix(keyStr, wgsl): + inReg = true + inHlsl = true + outLns = append(outLns, ln) } } + return outLns +} - rsls := make(map[string][]byte) - for fn, lns := range sls { - outfn := filepath.Join(cfg.Output, fn+".go") - olns := [][]byte{} - olns = append(olns, []byte("package main")) - olns = append(olns, []byte(`import ( +// AppendGoHeader appends Go header +func (st *State) AppendGoHeader(lines [][]byte) [][]byte { + olns := make([][]byte, 0, len(lines)+10) + olns = append(olns, []byte("package main")) + olns = append(olns, []byte(`import ( "math" - "cogentcore.org/core/gpu/gosl/slbool" - "cogentcore.org/core/gpu/gosl/slrand" - "cogentcore.org/core/gpu/gosl/sltype" + "cogentcore.org/core/goal/gosl/slbool" + "cogentcore.org/core/goal/gosl/slrand" + "cogentcore.org/core/goal/gosl/sltype" ) `)) - olns = append(olns, lns...) - SlBoolReplace(olns) - res := bytes.Join(olns, nl) - ioutil.WriteFile(outfn, res, 0644) - // not necessary and super slow: - // cmd := exec.Command("goimports", "-w", fn+".go") // get imports - // cmd.Dir, _ = filepath.Abs(*outDir) - // out, err := cmd.CombinedOutput() - // _ = out - // // fmt.Printf("\n################\ngoimports output for: %s\n%s\n", outfn, out) - // if err != nil { - // log.Println(err) - // } - rsls[fn] = bytes.Join(lns, nl) - } - - return rsls + olns = append(olns, lines...) + SlBoolReplace(olns) + return olns } // ExtractWGSL extracts the WGSL code embedded within .Go files. // Returns true if WGSL contains a void main( function. -func (cfg *Config) ExtractWGSL(buf []byte) ([]byte, bool) { +func (st *State) ExtractWGSL(b []byte) ([][]byte, bool) { key := []byte("//gosl:") wgsl := []byte("wgsl") nowgsl := []byte("nowgsl") end := []byte("end") - nl := []byte("\n") stComment := []byte("/*") edComment := []byte("*/") comment := []byte("// ") @@ -142,8 +116,9 @@ func (cfg *Config) ExtractWGSL(buf []byte) ([]byte, bool) { main := []byte("void main(") lparen := []byte("(") rparen := []byte(")") + nl := []byte("\n") - lines := bytes.Split(buf, nl) + lines := bytes.Split(b, nl) mx := min(10, len(lines)) stln := 0 @@ -211,5 +186,5 @@ func (cfg *Config) ExtractWGSL(buf []byte) ([]byte, bool) { noHlslStart = li } } - return bytes.Join(lines, nl), hasMain + return lines, hasMain } diff --git a/goal/gosl/parse/files.go b/goal/gosl/parse/files.go index b6e6c18692..cec1b9ac21 100644 --- a/goal/gosl/parse/files.go +++ b/goal/gosl/parse/files.go @@ -5,17 +5,59 @@ package parse import ( + "bytes" "fmt" - "io" "io/fs" "log" "os" "path/filepath" "strings" + "cogentcore.org/core/base/fsx" "golang.org/x/tools/go/packages" ) +// wgslFile returns the file with a ".wgsl" extension +func wgslFile(fn string) string { + f, _ := fsx.ExtSplit(fn) + return f + ".wgsl" +} + +// bareFile returns the file with no extention +func bareFile(fn string) string { + f, _ := fsx.ExtSplit(fn) + return f +} + +func ReadFileLines(fn string) ([][]byte, error) { + nl := []byte("\n") + buf, err := os.ReadFile(fn) + if err != nil { + fmt.Println(err) + return nil, err + } + lines := bytes.Split(buf, nl) + return lines, nil +} + +func (st *State) WriteFileLines(fn string, lines [][]byte) error { + outfn := filepath.Join(st.Config.Output, fn) + res := bytes.Join(lines, []byte("\n")) + return os.WriteFile(outfn, res, 0644) +} + +// HasGoslTag returns true if given file has a //gosl: tag +func HasGoslTag(lines [][]byte) bool { + key := []byte("//gosl:") + for _, ln := range lines { + tln := bytes.TrimSpace(ln) + if bytes.HasPrefix(tln, key) { + return true + } + } + return false +} + // LoadedPackageNames are single prefix names of packages that were // loaded in the list of files to process var LoadedPackageNames = map[string]bool{} @@ -35,137 +77,74 @@ func IsSPVFile(f fs.DirEntry) bool { return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".spv") && !f.IsDir() } -func (cfg *Config) AddFile(fn string, fls []string, procd map[string]bool) []string { - if _, has := procd[fn]; has { - return fls - } - fls = append(fls, fn) - procd[fn] = true - dir, _ := filepath.Split(fn) - if dir != "" { - dir = dir[:len(dir)-1] - pd, sd := filepath.Split(dir) - if pd != "" { - dir = sd +// ProjectFiles gets the files in the current directory. +func (st *State) ProjectFiles() { + fls := fsx.Filenames(".", ".go") + st.Files = make(map[string]*File) + for _, fn := range fls { + fl := &File{Name: fn} + var err error + fl.Lines, err = ReadFileLines(fn) + if err != nil { + continue } - if !(dir == "math32") { - if _, has := LoadedPackageNames[dir]; !has { - LoadedPackageNames[dir] = true - // fmt.Printf("package: %s\n", dir) - } + if !HasGoslTag(fl.Lines) { + continue } + st.Files[fn] = fl + st.ImportFiles(fl.Lines) } - return fls } -// FilesFromPaths processes all paths and returns a full unique list of files -// for subsequent processing. -func (cfg *Config) FilesFromPaths(paths []string) []string { - fls := make([]string, 0, len(paths)) - procd := make(map[string]bool) - for _, path := range paths { - switch info, err := os.Stat(path); { - case err != nil: - var pkgs []*packages.Package - dir, fl := filepath.Split(path) - if dir != "" && fl != "" && strings.HasSuffix(fl, ".go") { - pkgs, err = packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles}, dir) - } else { - fl = "" - pkgs, err = packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles}, path) - } +// ImportFiles checks the given content for //gosl:import tags +// and imports the package if so. +func (st *State) ImportFiles(lines [][]byte) { + key := []byte("//gosl:import ") + for _, ln := range lines { + tln := bytes.TrimSpace(ln) + if !bytes.HasPrefix(tln, key) { + continue + } + impath := strings.TrimSpace(string(tln[len(key):])) + if impath[0] == '"' { + impath = impath[1:] + } + if impath[len(impath)-1] == '"' { + impath = impath[:len(impath)-1] + } + _, ok := st.Imports[impath] + if ok { + continue + } + var pkgs []*packages.Package + var err error + pkgs, err = packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles}, impath) + if err != nil { + fmt.Println(err) + continue + } + pfls := make(map[string]*File) + st.Imports[impath] = pfls + pkg := pkgs[0] + gofls := pkg.GoFiles + if len(gofls) == 0 { + fmt.Printf("WARNING: no go files found in path: %s\n", impath) + } + for _, gf := range gofls { + lns, err := ReadFileLines(gf) if err != nil { - fmt.Println(err) continue } - pkg := pkgs[0] - gofls := pkg.GoFiles - if len(gofls) == 0 { - fmt.Printf("WARNING: no go files found in path: %s\n", path) - } - if fl != "" { - for _, gf := range gofls { - if strings.HasSuffix(gf, fl) { - fls = cfg.AddFile(gf, fls, procd) - // fmt.Printf("added file: %s from package: %s\n", gf, path) - break - } - } - } else { - for _, gf := range gofls { - fls = cfg.AddFile(gf, fls, procd) - // fmt.Printf("added file: %s from package: %s\n", gf, path) - } - } - case !info.IsDir(): - path := path - fls = cfg.AddFile(path, fls, procd) - default: - // Directories are walked, ignoring non-Go, non-WGSL files. - err := filepath.WalkDir(path, func(path string, f fs.DirEntry, err error) error { - if err != nil || !(IsGoFile(f) || IsWGSLFile(f)) { - return err - } - _, err = f.Info() - if err != nil { - return nil - } - fls = cfg.AddFile(path, fls, procd) - return nil - }) - if err != nil { - log.Println(err) + if !HasGoslTag(lns) { + continue } + _, fo := filepath.Split(gf) + pfls[fo] = &File{Name: fo, Lines: lns} + st.ImportFiles(lns) + // fmt.Printf("added file: %s from package: %s\n", gf, impath) } + st.Imports[impath] = pfls } - return fls -} - -func CopyFile(src, dst string) error { - in, err := os.Open(src) - if err != nil { - return err - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return err - } - defer out.Close() - _, err = io.Copy(out, in) - return err -} - -// CopyPackageFile copies given file name from given package path -// into the current output directory. -// e.g., "slrand.wgsl", "cogentcore.org/core/gpu/gosl/slrand" -func (cfg *Config) CopyPackageFile(fnm, packagePath string) error { - tofn := filepath.Join(cfg.Output, fnm) - pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles}, packagePath) - if err != nil { - fmt.Println(err) - return err - } - if len(pkgs) != 1 { - err = fmt.Errorf("%s package not found", packagePath) - fmt.Println(err) - return err - } - pkg := pkgs[0] - var fn string - if len(pkg.GoFiles) > 0 { - fn = pkg.GoFiles[0] - } else if len(pkg.OtherFiles) > 0 { - fn = pkg.GoFiles[0] - } else { - err = fmt.Errorf("No files found in package: %s", packagePath) - fmt.Println(err) - return err - } - dir, _ := filepath.Split(fn) - fmfn := filepath.Join(dir, fnm) - CopyFile(fmfn, tofn) - return nil } // RemoveGenFiles removes .go, .wgsl, .spv files in shader generated dir diff --git a/goal/gosl/parse/process.go b/goal/gosl/parse/process.go index ad702cb2ff..dc0404fe4c 100644 --- a/goal/gosl/parse/process.go +++ b/goal/gosl/parse/process.go @@ -15,7 +15,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" "cogentcore.org/core/base/errors" "cogentcore.org/core/goal/gosl/alignsl" @@ -23,53 +22,42 @@ import ( "golang.org/x/tools/go/packages" ) -// does all the file processing -func (cfg *Config) ProcessFiles() (map[string][]byte, error) { - paths := cfg.Inputs - fls := cfg.FilesFromPaths(paths) - gosls := cfg.ExtractGoFiles(fls) // extract Go files to shader/*.go - - wgslFiles := []string{} - for _, fn := range fls { - if strings.HasSuffix(fn, ".wgsl") { - wgslFiles = append(wgslFiles, fn) - } - } - - pf := "./" + cfg.Output +// ProcessDir process files in given directory. +func (st *State) ProcessDir(pf string) error { + nl := []byte("\n") pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesSizes | packages.NeedTypesInfo}, pf) if err != nil { - log.Println(err) - return nil, err + return errors.Log(err) } if len(pkgs) != 1 { err := fmt.Errorf("More than one package for path: %v", pf) - log.Println(err) - return nil, err + return errors.Log(err) } pkg := pkgs[0] - if len(pkg.GoFiles) == 0 { err := fmt.Errorf("No Go files found in package: %+v", pkg) - log.Println(err) - return nil, err + return errors.Log(err) } // fmt.Printf("go files: %+v", pkg.GoFiles) // return nil, err + files := pkg.GoFiles // map of files with a main function that needs to be compiled needsCompile := map[string]bool{} serr := alignsl.CheckPackage(pkg) + if serr != nil { fmt.Println(serr) } slrandCopied := false sltypeCopied := false - for fn := range gosls { - gofn := fn + ".go" - if cfg.Debug { + + for _, gofp := range files { + _, gofn := filepath.Split(gofp) + wgfn := wgslFile(gofn) + if st.Config.Debug { fmt.Printf("###################################\nProcessing Go file: %s\n", gofn) } @@ -90,83 +78,86 @@ func (cfg *Config) ProcessFiles() (map[string][]byte, error) { } var buf bytes.Buffer - pcfg := PrintConfig{Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: cfg.ExcludeMap} + pcfg := PrintConfig{Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: st.ExcludeMap} pcfg.Fprint(&buf, pkg, afile) // ioutil.WriteFile(filepath.Join(*outDir, fn+".tmp"), buf.Bytes(), 0644) slfix, hasSltype, hasSlrand := SlEdits(buf.Bytes()) + _ = slfix if hasSlrand && !slrandCopied { hasSltype = true - if cfg.Debug { + if st.Config.Debug { fmt.Printf("\tcopying slrand.wgsl to shaders\n") } - cfg.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/gpu/gosl/slrand") + // st.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/goal/gosl/slrand") slrandCopied = true } if hasSltype && !sltypeCopied { - if cfg.Debug { + if st.Config.Debug { fmt.Printf("\tcopying sltype.wgsl to shaders\n") } - cfg.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/gpu/gosl/sltype") + // st.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/goal/gosl/sltype") sltypeCopied = true } - exsl, hasMain := cfg.ExtractWGSL(slfix) - gosls[fn] = exsl - - if hasMain { - needsCompile[fn] = true - } - if !cfg.Keep { + exsl, hasMain := st.ExtractWGSL(slfix) + _ = hasMain + // gosls[fn] = exsl + + // if hasMain { + // needsCompile[fn] = true + // } + if !st.Config.Keep { os.Remove(fpos.Filename) } // add wgsl code - for _, slfn := range wgslFiles { - if fn+".wgsl" != slfn { - continue - } - buf, err := os.ReadFile(slfn) - if err != nil { - fmt.Println(err) - continue - } - exsl = append(exsl, []byte(fmt.Sprintf("\n// from file: %s\n", slfn))...) - exsl = append(exsl, buf...) - gosls[fn] = exsl - needsCompile[fn] = true // assume any standalone has main - break - } - - slfn := filepath.Join(cfg.Output, fn+".wgsl") - ioutil.WriteFile(slfn, exsl, 0644) + // for _, slfn := range wgslFiles { + // if fn+".wgsl" != slfn { + // continue + // } + // buf, err := os.ReadFile(slfn) + // if err != nil { + // fmt.Println(err) + // continue + // } + // exsl = append(exsl, []byte(fmt.Sprintf("\n// from file: %s\n", slfn))...) + // exsl = append(exsl, buf...) + // gosls[fn] = exsl + // needsCompile[fn] = true // assume any standalone has main + // break + // } + + slfn := filepath.Join(st.Config.Output, wgfn) + ioutil.WriteFile(slfn, bytes.Join(exsl, nl), 0644) } // check for wgsl files that had no go equivalent - for _, slfn := range wgslFiles { - hasGo := false - for fn := range gosls { - if fn+".wgsl" == slfn { - hasGo = true - break - } - } - if hasGo { - continue - } - _, slfno := filepath.Split(slfn) // could be in a subdir - tofn := filepath.Join(cfg.Output, slfno) - CopyFile(slfn, tofn) - fn := strings.TrimSuffix(slfno, ".wgsl") - needsCompile[fn] = true // assume any standalone wgsl is a main - } + + // for _, slfn := range wgslFiles { + // hasGo := false + // for fn := range gosls { + // if fn+".wgsl" == slfn { + // hasGo = true + // break + // } + // } + // if hasGo { + // continue + // } + // _, slfno := filepath.Split(slfn) // could be in a subdir + // tofn := filepath.Join(st.Config.Output, slfno) + // CopyFile(slfn, tofn) + // fn := strings.TrimSuffix(slfno, ".wgsl") + // needsCompile[fn] = true // assume any standalone wgsl is a main + // } for fn := range needsCompile { - cfg.CompileFile(fn + ".wgsl") + st.CompileFile(fn + ".wgsl") } - return gosls, nil + return nil } -func (cfg *Config) CompileFile(fn string) error { - dir, _ := filepath.Abs(cfg.Output) +func (st *State) CompileFile(fn string) error { + dir, _ := filepath.Abs(st.Config.Output) fsys := os.DirFS(dir) b, err := fs.ReadFile(fsys, fn) if errors.Log(err) != nil { diff --git a/goal/gosl/parse/state.go b/goal/gosl/parse/state.go new file mode 100644 index 0000000000..29debc7338 --- /dev/null +++ b/goal/gosl/parse/state.go @@ -0,0 +1,105 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package parse + +import ( + "errors" + "os" + "path/filepath" + "strings" +) + +// Var represents one variable +type Var struct { + Name string + + Type string +} + +// Group represents one variable group. +type Group struct { + Vars []*Var +} + +// System represents a ComputeSystem, and its variables. +type System struct { + Name string + + Groups []*Group +} + +// Kernel represents a kernel function, which is the basis for +// each wgsl generated code file. +type Kernel struct { + Name string + + // accumulating lines of code for the wgsl file. + FileLines [][]byte +} + +// File has info for a file being processed +type File struct { + Name string + + Lines [][]byte +} + +// State holds the current processing state +type State struct { + // Config options + Config *Config + + // files with gosl content in current directory + Files map[string]*File + + // has all the imports + Imports map[string]map[string]*File + + // short package names + ImportPackages map[string]bool + + Kernels map[string]*Kernel + + Systems map[string]*System + + // ExcludeMap is the compiled map of functions to exclude. + ExcludeMap map[string]bool +} + +func (st *State) Init(cfg *Config) { + st.Config = cfg + st.Imports = make(map[string]map[string]*File) + st.Kernels = make(map[string]*Kernel) + st.Systems = make(map[string]*System) + st.ExcludeMap = make(map[string]bool) + ex := strings.Split(cfg.Exclude, ",") + for _, fn := range ex { + st.ExcludeMap[fn] = true + } + + sy := &System{Name: "Default"} + sy.Groups = append(sy.Groups, &Group{}) + st.Systems["Default"] = sy +} + +func (st *State) Run() error { + if gomod := os.Getenv("GO111MODULE"); gomod == "off" { + err := errors.New("gosl only works in go modules mode, but GO111MODULE=off") + return err + } + if st.Config.Output == "" { + st.Config.Output = "shaders" + } + imps := filepath.Join(st.Config.Output, "imports") + os.MkdirAll(st.Config.Output, 0755) + os.MkdirAll(imps, 0755) + RemoveGenFiles(st.Config.Output) + RemoveGenFiles(imps) + + st.ProjectFiles() // recursively gets imports etc. + st.ExtractImports() // save all the import files + + return nil +} diff --git a/goal/gosl/parse/typegen.go b/goal/gosl/parse/typegen.go index 54bf0288c4..43d11ba2f5 100644 --- a/goal/gosl/parse/typegen.go +++ b/goal/gosl/parse/typegen.go @@ -6,7 +6,7 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system", Fields: []types.Field{{Name: "Inputs", Doc: "Inputs are the input files to run/compile."}, {Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system.", Fields: []types.Field{{Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.exprListMode", IDName: "expr-list-mode"}) @@ -32,6 +32,18 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Com var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Replace", IDName: "replace", Fields: []types.Field{{Name: "From"}, {Name: "To"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Var", IDName: "var", Doc: "Var represents one variable", Fields: []types.Field{{Name: "Name"}, {Name: "Type"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Group", IDName: "group", Doc: "Group represents one variable group.", Fields: []types.Field{{Name: "Vars"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.System", IDName: "system", Doc: "System represents a ComputeSystem, and its variables.", Fields: []types.Field{{Name: "Name"}, {Name: "Groups"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Kernel", IDName: "kernel", Doc: "Kernel represents a kernel function, which is the basis for\neach wgsl generated code file.", Fields: []types.Field{{Name: "Name"}, {Name: "FileLines", Doc: "accumulating lines of code for the wgsl file."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.File", IDName: "file", Doc: "File has info for a file being processed", Fields: []types.Field{{Name: "Name"}, {Name: "Lines"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.State", IDName: "state", Doc: "State holds the current processing state", Fields: []types.Field{{Name: "Config", Doc: "Config options"}, {Name: "Files", Doc: "files with gosl content in current directory"}, {Name: "Imports"}, {Name: "Kernels"}, {Name: "Systems"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude."}}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.formatDocComment", Doc: "formatDocComment reformats the doc comment list,\nreturning the canonical formatting.", Args: []string{"list"}, Returns: []string{"Comment"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isDirective", Doc: "isDirective reports whether c is a comment directive.\nSee go.dev/issue/37974.\nThis code is also in go/ast.", Args: []string{"c"}, Returns: []string{"bool"}}) @@ -40,14 +52,16 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.all var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.Run", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.ReadFileLines", Args: []string{"fn"}, Returns: []string{"[][]byte", "error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.HasGoslTag", Doc: "HasGoslTag returns true if given file has a //gosl: tag", Args: []string{"lines"}, Returns: []string{"bool"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsGoFile", Args: []string{"f"}, Returns: []string{"bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsWGSLFile", Args: []string{"f"}, Returns: []string{"bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsSPVFile", Args: []string{"f"}, Returns: []string{"bool"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.CopyFile", Args: []string{"src", "dst"}, Returns: []string{"error"}}) - var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.RemoveGenFiles", Doc: "RemoveGenFiles removes .go, .wgsl, .spv files in shader generated dir", Args: []string{"dir"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.appendLines", Doc: "appendLines is like append(x, y...)\nbut it avoids creating doubled blank lines,\nwhich would not be gofmt-standard output.\nIt assumes that only whole blocks of lines are being appended,\nnot line fragments.", Args: []string{"x", "y"}, Returns: []string{"[]byte"}}) @@ -110,6 +124,10 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.new var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.Fprint", Doc: "Fprint \"pretty-prints\" an AST node to output.\nIt calls [PrintConfig.Fprint] with default settings.\nNote that gofmt uses tabs for indentation but spaces for alignment;\nuse format.Node (package go/format) for output that matches gofmt.", Args: []string{"output", "pkg", "node"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.wgslFile", Doc: "wgslFile returns the file with a \".wgsl\" extension", Args: []string{"fn"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.bareFile", Doc: "bareFile returns the file with no extention", Args: []string{"fn"}, Returns: []string{"string"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.MoveLines", Doc: "MoveLines moves the st,ed region to 'to' line", Args: []string{"lines", "to", "st", "ed"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlEdits", Doc: "SlEdits performs post-generation edits for wgsl\n* moves wgsl segments around, e.g., methods\ninto their proper classes\n* fixes printf, slice other common code\nreturns true if a slrand. or sltype. prefix was found,\ndriveing copying of those files.", Args: []string{"src"}, Returns: []string{"[]byte", "bool", "bool"}}) From 372676190176fbdf6026688765d821afb3ab1d8f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 11:55:50 -0700 Subject: [PATCH 185/311] rename parse -> gotosl --- goal/gosl/cmd/gosl/gosl.go | 6 +- goal/gosl/{parse => gotosl}/comment.go | 2 +- goal/gosl/{parse => gotosl}/config.go | 2 +- goal/gosl/{parse => gotosl}/extract.go | 2 +- goal/gosl/{parse => gotosl}/files.go | 2 +- goal/gosl/{parse => gotosl}/gobuild.go | 2 +- goal/gosl/{parse => gotosl}/gosl_test.go | 2 +- goal/gosl/{parse => gotosl}/nodes.go | 10 +- goal/gosl/{parse => gotosl}/printer.go | 2 +- goal/gosl/{parse => gotosl}/process.go | 2 +- goal/gosl/{parse => gotosl}/sledits.go | 2 +- goal/gosl/{parse => gotosl}/state.go | 2 +- goal/gosl/{parse => gotosl}/testdata/basic.go | 0 .../{parse => gotosl}/testdata/basic.golden | 0 goal/gosl/gotosl/typegen.go | 139 ++++++++++++++++++ goal/gosl/parse/typegen.go | 139 ------------------ 16 files changed, 157 insertions(+), 157 deletions(-) rename goal/gosl/{parse => gotosl}/comment.go (99%) rename goal/gosl/{parse => gotosl}/config.go (98%) rename goal/gosl/{parse => gotosl}/extract.go (99%) rename goal/gosl/{parse => gotosl}/files.go (99%) rename goal/gosl/{parse => gotosl}/gobuild.go (99%) rename goal/gosl/{parse => gotosl}/gosl_test.go (99%) rename goal/gosl/{parse => gotosl}/nodes.go (99%) rename goal/gosl/{parse => gotosl}/printer.go (99%) rename goal/gosl/{parse => gotosl}/process.go (99%) rename goal/gosl/{parse => gotosl}/sledits.go (99%) rename goal/gosl/{parse => gotosl}/state.go (99%) rename goal/gosl/{parse => gotosl}/testdata/basic.go (100%) rename goal/gosl/{parse => gotosl}/testdata/basic.golden (100%) create mode 100644 goal/gosl/gotosl/typegen.go delete mode 100644 goal/gosl/parse/typegen.go diff --git a/goal/gosl/cmd/gosl/gosl.go b/goal/gosl/cmd/gosl/gosl.go index 9d71930175..c3343364df 100644 --- a/goal/gosl/cmd/gosl/gosl.go +++ b/goal/gosl/cmd/gosl/gosl.go @@ -6,11 +6,11 @@ package main import ( "cogentcore.org/core/cli" - "cogentcore.org/core/goal/gosl/parse" + "cogentcore.org/core/goal/gosl/gotosl" ) func main() { //types:skip opts := cli.DefaultOptions("gosl", "Go as a shader language converts Go code to WGSL WebGPU shader code, which can be run on the GPU through WebGPU.") - cfg := &parse.Config{} - cli.Run(opts, cfg, parse.Run) + cfg := &gotosl.Config{} + cli.Run(opts, cfg, gotosl.Run) } diff --git a/goal/gosl/parse/comment.go b/goal/gosl/gotosl/comment.go similarity index 99% rename from goal/gosl/parse/comment.go rename to goal/gosl/gotosl/comment.go index 345dc31373..ede45eeb98 100644 --- a/goal/gosl/parse/comment.go +++ b/goal/gosl/gotosl/comment.go @@ -9,7 +9,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "go/ast" diff --git a/goal/gosl/parse/config.go b/goal/gosl/gotosl/config.go similarity index 98% rename from goal/gosl/parse/config.go rename to goal/gosl/gotosl/config.go index f8a0068066..746edd7686 100644 --- a/goal/gosl/parse/config.go +++ b/goal/gosl/gotosl/config.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl //go:generate core generate -add-types -add-funcs diff --git a/goal/gosl/parse/extract.go b/goal/gosl/gotosl/extract.go similarity index 99% rename from goal/gosl/parse/extract.go rename to goal/gosl/gotosl/extract.go index 037d10074a..547de10cc8 100644 --- a/goal/gosl/parse/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "bytes" diff --git a/goal/gosl/parse/files.go b/goal/gosl/gotosl/files.go similarity index 99% rename from goal/gosl/parse/files.go rename to goal/gosl/gotosl/files.go index cec1b9ac21..5023873349 100644 --- a/goal/gosl/parse/files.go +++ b/goal/gosl/gotosl/files.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "bytes" diff --git a/goal/gosl/parse/gobuild.go b/goal/gosl/gotosl/gobuild.go similarity index 99% rename from goal/gosl/parse/gobuild.go rename to goal/gosl/gotosl/gobuild.go index e5ad8bc847..233196dde8 100644 --- a/goal/gosl/parse/gobuild.go +++ b/goal/gosl/gotosl/gobuild.go @@ -9,7 +9,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "go/build/constraint" diff --git a/goal/gosl/parse/gosl_test.go b/goal/gosl/gotosl/gosl_test.go similarity index 99% rename from goal/gosl/parse/gosl_test.go rename to goal/gosl/gotosl/gosl_test.go index ea07965ca9..8e23328ba5 100644 --- a/goal/gosl/parse/gosl_test.go +++ b/goal/gosl/gotosl/gosl_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "bytes" diff --git a/goal/gosl/parse/nodes.go b/goal/gosl/gotosl/nodes.go similarity index 99% rename from goal/gosl/parse/nodes.go rename to goal/gosl/gotosl/nodes.go index 2dad0f9991..49d1942c00 100644 --- a/goal/gosl/parse/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -13,7 +13,7 @@ // expressions, statements, declarations, and files. It uses // the print functionality implemented in printer.go. -package parse +package gotosl import ( "fmt" @@ -423,7 +423,7 @@ func (p *printer) parameters(fields *ast.FieldList, mode paramMode) { // A type parameter list [P T] where the name P and the type expression T syntactically // combine to another valid (value) expression requires a trailing comma, as in [P *T,] // (or an enclosing interface as in [P interface(*T)]), so that the type parameter list - // is not parsed as an array length [P*T]. + // is not gotosld as an array length [P*T]. p.print(token.COMMA) } @@ -1009,7 +1009,7 @@ func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int) { prec := x.Op.Precedence() if prec < prec1 { // parenthesis needed - // Note: The parser inserts an ast.ParenExpr node; thus this case + // Note: The gotoslr inserts an ast.ParenExpr node; thus this case // can only occur if the AST is created in a different way. p.print(token.LPAREN) p.expr0(x, reduceDepth(depth)) // parentheses undo one level of depth @@ -1895,7 +1895,7 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, nosemi bool) { default: // This can only happen with an incorrectly // constructed AST. Permit it but print so - // that it can be parsed without errors. + // that it can be gotosld without errors. p.print(token.LBRACE, indent, formfeed) p.stmt(s.Else, true, false) p.print(unindent, formfeed, token.RBRACE) @@ -2102,7 +2102,7 @@ func (p *printer) valueSpec(s *ast.ValueSpec, keepType bool, tok token.Token, fi } func sanitizeImportPath(lit *ast.BasicLit) *ast.BasicLit { - // Note: An unmodified AST generated by go/parser will already + // Note: An unmodified AST generated by go/gotoslr will already // contain a backward- or double-quoted path string that does // not contain any invalid characters, and most of the work // here is not needed. However, a modified or generated AST diff --git a/goal/gosl/parse/printer.go b/goal/gosl/gotosl/printer.go similarity index 99% rename from goal/gosl/parse/printer.go rename to goal/gosl/gotosl/printer.go index 3cefd1ef7e..cf341330e3 100644 --- a/goal/gosl/parse/printer.go +++ b/goal/gosl/gotosl/printer.go @@ -9,7 +9,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "fmt" diff --git a/goal/gosl/parse/process.go b/goal/gosl/gotosl/process.go similarity index 99% rename from goal/gosl/parse/process.go rename to goal/gosl/gotosl/process.go index dc0404fe4c..b9b66d45d6 100644 --- a/goal/gosl/parse/process.go +++ b/goal/gosl/gotosl/process.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "bytes" diff --git a/goal/gosl/parse/sledits.go b/goal/gosl/gotosl/sledits.go similarity index 99% rename from goal/gosl/parse/sledits.go rename to goal/gosl/gotosl/sledits.go index e66b8758ac..020aa4c6b4 100644 --- a/goal/gosl/parse/sledits.go +++ b/goal/gosl/gotosl/sledits.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "bytes" diff --git a/goal/gosl/parse/state.go b/goal/gosl/gotosl/state.go similarity index 99% rename from goal/gosl/parse/state.go rename to goal/gosl/gotosl/state.go index 29debc7338..ee4e04da19 100644 --- a/goal/gosl/parse/state.go +++ b/goal/gosl/gotosl/state.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package parse +package gotosl import ( "errors" diff --git a/goal/gosl/parse/testdata/basic.go b/goal/gosl/gotosl/testdata/basic.go similarity index 100% rename from goal/gosl/parse/testdata/basic.go rename to goal/gosl/gotosl/testdata/basic.go diff --git a/goal/gosl/parse/testdata/basic.golden b/goal/gosl/gotosl/testdata/basic.golden similarity index 100% rename from goal/gosl/parse/testdata/basic.golden rename to goal/gosl/gotosl/testdata/basic.golden diff --git a/goal/gosl/gotosl/typegen.go b/goal/gosl/gotosl/typegen.go new file mode 100644 index 0000000000..ea67c466fe --- /dev/null +++ b/goal/gosl/gotosl/typegen.go @@ -0,0 +1,139 @@ +// Code generated by "core generate -add-types -add-funcs"; DO NOT EDIT. + +package gotosl + +import ( + "cogentcore.org/core/types" +) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system.", Fields: []types.Field{{Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.exprListMode", IDName: "expr-list-mode"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.paramMode", IDName: "param-mode"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.sizeCounter", IDName: "size-counter", Doc: "sizeCounter is an io.Writer which counts the number of bytes written,\nas well as whether a newline character was seen.", Fields: []types.Field{{Name: "hasNewline"}, {Name: "size"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.whiteSpace", IDName: "white-space"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.pmode", IDName: "pmode", Doc: "A pmode value represents the current printer mode."}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.commentInfo", IDName: "comment-info", Fields: []types.Field{{Name: "cindex"}, {Name: "comment"}, {Name: "commentOffset"}, {Name: "commentNewline"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.printer", IDName: "printer", Embeds: []types.Field{{Name: "PrintConfig", Doc: "Configuration (does not change after initialization)"}, {Name: "commentInfo", Doc: "Information about p.comments[p.cindex]; set up by nextComment."}}, Fields: []types.Field{{Name: "fset"}, {Name: "pkg"}, {Name: "output", Doc: "Current state"}, {Name: "indent"}, {Name: "level"}, {Name: "mode"}, {Name: "endAlignment"}, {Name: "impliedSemi"}, {Name: "lastTok"}, {Name: "prevOpen"}, {Name: "wsbuf"}, {Name: "goBuild"}, {Name: "plusBuild"}, {Name: "pos", Doc: "Positions\nThe out position differs from the pos position when the result\nformatting differs from the source formatting (in the amount of\nwhite space). If there's a difference and SourcePos is set in\nConfigMode, //line directives are used in the output to restore\noriginal source positions for a reader."}, {Name: "out"}, {Name: "last"}, {Name: "linePtr"}, {Name: "sourcePosErr"}, {Name: "comments", Doc: "The list of all source comments, in order of appearance."}, {Name: "useNodeComments"}, {Name: "nodeSizes", Doc: "Cache of already computed node sizes."}, {Name: "cachedPos", Doc: "Cache of most recently computed line position."}, {Name: "cachedLine"}, {Name: "curPtrArgs", Doc: "current arguments to function that are pointers and thus need dereferencing\nwhen accessing fields"}, {Name: "curMethRecv"}, {Name: "curReturnType"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.trimmer", IDName: "trimmer", Doc: "A trimmer is an io.Writer filter for stripping tabwriter.Escape\ncharacters, trailing blanks and tabs, and for converting formfeed\nand vtab characters into newlines and htabs (in case no tabwriter\nis used). Text bracketed by tabwriter.Escape characters is passed\nthrough unchanged.", Fields: []types.Field{{Name: "output"}, {Name: "state"}, {Name: "space"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Mode", IDName: "mode", Doc: "A Mode value is a set of flags (or 0). They control printing."}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.PrintConfig", IDName: "print-config", Doc: "A PrintConfig node controls the output of Fprint.", Fields: []types.Field{{Name: "Mode"}, {Name: "Tabwidth"}, {Name: "Indent"}, {Name: "ExcludeFunctions"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.CommentedNode", IDName: "commented-node", Doc: "A CommentedNode bundles an AST node and corresponding comments.\nIt may be provided as argument to any of the [Fprint] functions.", Fields: []types.Field{{Name: "Node"}, {Name: "Comments"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Replace", IDName: "replace", Fields: []types.Field{{Name: "From"}, {Name: "To"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Var", IDName: "var", Doc: "Var represents one variable", Fields: []types.Field{{Name: "Name"}, {Name: "Type"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Group", IDName: "group", Doc: "Group represents one variable group.", Fields: []types.Field{{Name: "Vars"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.System", IDName: "system", Doc: "System represents a ComputeSystem, and its variables.", Fields: []types.Field{{Name: "Name"}, {Name: "Groups"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Kernel", IDName: "kernel", Doc: "Kernel represents a kernel function, which is the basis for\neach wgsl generated code file.", Fields: []types.Field{{Name: "Name"}, {Name: "FileLines", Doc: "accumulating lines of code for the wgsl file."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.File", IDName: "file", Doc: "File has info for a file being processed", Fields: []types.Field{{Name: "Name"}, {Name: "Lines"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current processing state", Fields: []types.Field{{Name: "Config", Doc: "Config options"}, {Name: "Files", Doc: "files with gosl content in current directory"}, {Name: "Imports"}, {Name: "Kernels"}, {Name: "Systems"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude."}}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.formatDocComment", Doc: "formatDocComment reformats the doc comment list,\nreturning the canonical formatting.", Args: []string{"list"}, Returns: []string{"Comment"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isDirective", Doc: "isDirective reports whether c is a comment directive.\nSee go.dev/issue/37974.\nThis code is also in go/ast.", Args: []string{"c"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.allStars", Doc: "allStars reports whether text is the interior of an\nold-style /* */ comment with a star at the start of each line.", Args: []string{"text"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.Run", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.ReadFileLines", Args: []string{"fn"}, Returns: []string{"[][]byte", "error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.HasGoslTag", Doc: "HasGoslTag returns true if given file has a //gosl: tag", Args: []string{"lines"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.IsGoFile", Args: []string{"f"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.IsWGSLFile", Args: []string{"f"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.IsSPVFile", Args: []string{"f"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.RemoveGenFiles", Doc: "RemoveGenFiles removes .go, .wgsl, .spv files in shader generated dir", Args: []string{"dir"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.appendLines", Doc: "appendLines is like append(x, y...)\nbut it avoids creating doubled blank lines,\nwhich would not be gofmt-standard output.\nIt assumes that only whole blocks of lines are being appended,\nnot line fragments.", Args: []string{"x", "y"}, Returns: []string{"[]byte"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isNL", Args: []string{"b"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.combinesWithName", Doc: "combinesWithName reports whether a name followed by the expression x\nsyntactically combines to another valid (value) expression. For instance\nusing *T for x, \"name *T\" syntactically appears as the expression x*T.\nOn the other hand, using P|Q or *P|~Q for x, \"name P|Q\" or name *P|~Q\"\ncannot be combined into a valid (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isTypeElem", Doc: "isTypeElem reports whether x is a (possibly parenthesized) type element expression.\nThe result is false if x could be a type element OR an ordinary (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.identListSize", Args: []string{"list", "maxSize"}, Returns: []string{"size"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.walkBinary", Args: []string{"e"}, Returns: []string{"has4", "has5", "maxProblem"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.cutoff", Args: []string{"e", "depth"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.diffPrec", Args: []string{"expr", "prec"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.reduceDepth", Args: []string{"depth"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isBinary", Args: []string{"expr"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.normalizedNumber", Doc: "normalizedNumber rewrites base prefixes and exponents\nof numbers to use lower-case letters (0X123 to 0x123 and 1.2E3 to 1.2e3),\nand removes leading 0's from integer imaginary literals (0765i to 765i).\nIt leaves hexadecimal digits alone.\n\nnormalizedNumber doesn't modify the ast.BasicLit value lit points to.\nIf lit is not a number or a number in canonical format already,\nlit is returned as is. Otherwise a new ast.BasicLit is created.", Args: []string{"lit"}, Returns: []string{"BasicLit"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.fieldByName", Args: []string{"st", "name"}, Returns: []string{"Var"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getLocalTypeName", Args: []string{"typ"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getStructType", Args: []string{"typ"}, Returns: []string{"Struct", "error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isTypeName", Args: []string{"x"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.stripParens", Args: []string{"x"}, Returns: []string{"Expr"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.stripParensAlways", Args: []string{"x"}, Returns: []string{"Expr"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.keepTypeColumn", Doc: "The keepTypeColumn function determines if the type column of a series of\nconsecutive const or var declarations must be kept, or if initialization\nvalues (V) can be placed in the type column (T) instead. The i'th entry\nin the result slice is true if the type column in spec[i] must be kept.\n\nFor example, the declaration:\n\n\t\tconst (\n\t\t\tfoobar int = 42 // comment\n\t\t\tx = 7 // comment\n\t\t\tfoo\n\t bar = 991\n\t\t)\n\nleads to the type/values matrix below. A run of value columns (V) can\nbe moved into the type column if there is no type for any of the values\nin that column (we only move entire columns so that they align properly).\n\n\t\tmatrix formatted result\n\t matrix\n\t\tT V -> T V -> true there is a T and so the type\n\t\t- V - V true column must be kept\n\t\t- - - - false\n\t\t- V V - false V is moved into T column", Args: []string{"specs"}, Returns: []string{"[]bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.sanitizeImportPath", Args: []string{"lit"}, Returns: []string{"BasicLit"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.declToken", Args: []string{"decl"}, Returns: []string{"tok"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isBlank", Doc: "Returns true if s contains only white space\n(only tabs and blanks can appear in the printer's context).", Args: []string{"s"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.commonPrefix", Doc: "commonPrefix returns the common prefix of a and b.", Args: []string{"a", "b"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.trimRight", Doc: "trimRight returns s with trailing whitespace removed.", Args: []string{"s"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.stripCommonPrefix", Doc: "stripCommonPrefix removes a common prefix from /*-style comment lines (unless no\ncomment line is indented, all but the first line have some form of space prefix).\nThe prefix is computed using heuristics such that is likely that the comment\ncontents are nicely laid out after re-printing each line using the printer's\ncurrent indentation.", Args: []string{"lines"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.nlimit", Doc: "nlimit limits n to maxNewlines.", Args: []string{"n"}, Returns: []string{"int"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.mayCombine", Args: []string{"prev", "next"}, Returns: []string{"b"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getDoc", Doc: "getDoc returns the ast.CommentGroup associated with n, if any.", Args: []string{"n"}, Returns: []string{"CommentGroup"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getLastComment", Args: []string{"n"}, Returns: []string{"CommentGroup"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.newPrinter", Args: []string{"cfg", "pkg", "nodeSizes"}, Returns: []string{"printer"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.Fprint", Doc: "Fprint \"pretty-prints\" an AST node to output.\nIt calls [PrintConfig.Fprint] with default settings.\nNote that gofmt uses tabs for indentation but spaces for alignment;\nuse format.Node (package go/format) for output that matches gofmt.", Args: []string{"output", "pkg", "node"}, Returns: []string{"error"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.wgslFile", Doc: "wgslFile returns the file with a \".wgsl\" extension", Args: []string{"fn"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.bareFile", Doc: "bareFile returns the file with no extention", Args: []string{"fn"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.MoveLines", Doc: "MoveLines moves the st,ed region to 'to' line", Args: []string{"lines", "to", "st", "ed"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlEdits", Doc: "SlEdits performs post-generation edits for wgsl\n* moves wgsl segments around, e.g., methods\ninto their proper classes\n* fixes printf, slice other common code\nreturns true if a slrand. or sltype. prefix was found,\ndriveing copying of those files.", Args: []string{"src"}, Returns: []string{"[]byte", "bool", "bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.MathReplaceAll", Args: []string{"mat", "ln"}, Returns: []string{"[]byte"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlEditsReplace", Doc: "SlEditsReplace replaces Go with equivalent WGSL code\nreturns true if has slrand. or sltype.\nto auto include that header file if so.", Args: []string{"lines"}, Returns: []string{"bool", "bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlBoolReplace", Doc: "SlBoolReplace replaces all the slbool methods with literal int32 expressions.", Args: []string{"lines"}}) diff --git a/goal/gosl/parse/typegen.go b/goal/gosl/parse/typegen.go deleted file mode 100644 index 43d11ba2f5..0000000000 --- a/goal/gosl/parse/typegen.go +++ /dev/null @@ -1,139 +0,0 @@ -// Code generated by "core generate -add-types -add-funcs"; DO NOT EDIT. - -package parse - -import ( - "cogentcore.org/core/types" -) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system.", Fields: []types.Field{{Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.exprListMode", IDName: "expr-list-mode"}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.paramMode", IDName: "param-mode"}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.sizeCounter", IDName: "size-counter", Doc: "sizeCounter is an io.Writer which counts the number of bytes written,\nas well as whether a newline character was seen.", Fields: []types.Field{{Name: "hasNewline"}, {Name: "size"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.whiteSpace", IDName: "white-space"}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.pmode", IDName: "pmode", Doc: "A pmode value represents the current printer mode."}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.commentInfo", IDName: "comment-info", Fields: []types.Field{{Name: "cindex"}, {Name: "comment"}, {Name: "commentOffset"}, {Name: "commentNewline"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.printer", IDName: "printer", Embeds: []types.Field{{Name: "PrintConfig", Doc: "Configuration (does not change after initialization)"}, {Name: "commentInfo", Doc: "Information about p.comments[p.cindex]; set up by nextComment."}}, Fields: []types.Field{{Name: "fset"}, {Name: "pkg"}, {Name: "output", Doc: "Current state"}, {Name: "indent"}, {Name: "level"}, {Name: "mode"}, {Name: "endAlignment"}, {Name: "impliedSemi"}, {Name: "lastTok"}, {Name: "prevOpen"}, {Name: "wsbuf"}, {Name: "goBuild"}, {Name: "plusBuild"}, {Name: "pos", Doc: "Positions\nThe out position differs from the pos position when the result\nformatting differs from the source formatting (in the amount of\nwhite space). If there's a difference and SourcePos is set in\nConfigMode, //line directives are used in the output to restore\noriginal source positions for a reader."}, {Name: "out"}, {Name: "last"}, {Name: "linePtr"}, {Name: "sourcePosErr"}, {Name: "comments", Doc: "The list of all source comments, in order of appearance."}, {Name: "useNodeComments"}, {Name: "nodeSizes", Doc: "Cache of already computed node sizes."}, {Name: "cachedPos", Doc: "Cache of most recently computed line position."}, {Name: "cachedLine"}, {Name: "curPtrArgs", Doc: "current arguments to function that are pointers and thus need dereferencing\nwhen accessing fields"}, {Name: "curMethRecv"}, {Name: "curReturnType"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.trimmer", IDName: "trimmer", Doc: "A trimmer is an io.Writer filter for stripping tabwriter.Escape\ncharacters, trailing blanks and tabs, and for converting formfeed\nand vtab characters into newlines and htabs (in case no tabwriter\nis used). Text bracketed by tabwriter.Escape characters is passed\nthrough unchanged.", Fields: []types.Field{{Name: "output"}, {Name: "state"}, {Name: "space"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Mode", IDName: "mode", Doc: "A Mode value is a set of flags (or 0). They control printing."}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.PrintConfig", IDName: "print-config", Doc: "A PrintConfig node controls the output of Fprint.", Fields: []types.Field{{Name: "Mode"}, {Name: "Tabwidth"}, {Name: "Indent"}, {Name: "ExcludeFunctions"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.CommentedNode", IDName: "commented-node", Doc: "A CommentedNode bundles an AST node and corresponding comments.\nIt may be provided as argument to any of the [Fprint] functions.", Fields: []types.Field{{Name: "Node"}, {Name: "Comments"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Replace", IDName: "replace", Fields: []types.Field{{Name: "From"}, {Name: "To"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Var", IDName: "var", Doc: "Var represents one variable", Fields: []types.Field{{Name: "Name"}, {Name: "Type"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Group", IDName: "group", Doc: "Group represents one variable group.", Fields: []types.Field{{Name: "Vars"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.System", IDName: "system", Doc: "System represents a ComputeSystem, and its variables.", Fields: []types.Field{{Name: "Name"}, {Name: "Groups"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.Kernel", IDName: "kernel", Doc: "Kernel represents a kernel function, which is the basis for\neach wgsl generated code file.", Fields: []types.Field{{Name: "Name"}, {Name: "FileLines", Doc: "accumulating lines of code for the wgsl file."}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.File", IDName: "file", Doc: "File has info for a file being processed", Fields: []types.Field{{Name: "Name"}, {Name: "Lines"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/parse.State", IDName: "state", Doc: "State holds the current processing state", Fields: []types.Field{{Name: "Config", Doc: "Config options"}, {Name: "Files", Doc: "files with gosl content in current directory"}, {Name: "Imports"}, {Name: "Kernels"}, {Name: "Systems"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude."}}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.formatDocComment", Doc: "formatDocComment reformats the doc comment list,\nreturning the canonical formatting.", Args: []string{"list"}, Returns: []string{"Comment"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isDirective", Doc: "isDirective reports whether c is a comment directive.\nSee go.dev/issue/37974.\nThis code is also in go/ast.", Args: []string{"c"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.allStars", Doc: "allStars reports whether text is the interior of an\nold-style /* */ comment with a star at the start of each line.", Args: []string{"text"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.Run", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.ReadFileLines", Args: []string{"fn"}, Returns: []string{"[][]byte", "error"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.HasGoslTag", Doc: "HasGoslTag returns true if given file has a //gosl: tag", Args: []string{"lines"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsGoFile", Args: []string{"f"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsWGSLFile", Args: []string{"f"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.IsSPVFile", Args: []string{"f"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.RemoveGenFiles", Doc: "RemoveGenFiles removes .go, .wgsl, .spv files in shader generated dir", Args: []string{"dir"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.appendLines", Doc: "appendLines is like append(x, y...)\nbut it avoids creating doubled blank lines,\nwhich would not be gofmt-standard output.\nIt assumes that only whole blocks of lines are being appended,\nnot line fragments.", Args: []string{"x", "y"}, Returns: []string{"[]byte"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isNL", Args: []string{"b"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.combinesWithName", Doc: "combinesWithName reports whether a name followed by the expression x\nsyntactically combines to another valid (value) expression. For instance\nusing *T for x, \"name *T\" syntactically appears as the expression x*T.\nOn the other hand, using P|Q or *P|~Q for x, \"name P|Q\" or name *P|~Q\"\ncannot be combined into a valid (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isTypeElem", Doc: "isTypeElem reports whether x is a (possibly parenthesized) type element expression.\nThe result is false if x could be a type element OR an ordinary (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.identListSize", Args: []string{"list", "maxSize"}, Returns: []string{"size"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.walkBinary", Args: []string{"e"}, Returns: []string{"has4", "has5", "maxProblem"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.cutoff", Args: []string{"e", "depth"}, Returns: []string{"int"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.diffPrec", Args: []string{"expr", "prec"}, Returns: []string{"int"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.reduceDepth", Args: []string{"depth"}, Returns: []string{"int"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isBinary", Args: []string{"expr"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.normalizedNumber", Doc: "normalizedNumber rewrites base prefixes and exponents\nof numbers to use lower-case letters (0X123 to 0x123 and 1.2E3 to 1.2e3),\nand removes leading 0's from integer imaginary literals (0765i to 765i).\nIt leaves hexadecimal digits alone.\n\nnormalizedNumber doesn't modify the ast.BasicLit value lit points to.\nIf lit is not a number or a number in canonical format already,\nlit is returned as is. Otherwise a new ast.BasicLit is created.", Args: []string{"lit"}, Returns: []string{"BasicLit"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.fieldByName", Args: []string{"st", "name"}, Returns: []string{"Var"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getLocalTypeName", Args: []string{"typ"}, Returns: []string{"string"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getStructType", Args: []string{"typ"}, Returns: []string{"Struct", "error"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isTypeName", Args: []string{"x"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.stripParens", Args: []string{"x"}, Returns: []string{"Expr"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.stripParensAlways", Args: []string{"x"}, Returns: []string{"Expr"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.keepTypeColumn", Doc: "The keepTypeColumn function determines if the type column of a series of\nconsecutive const or var declarations must be kept, or if initialization\nvalues (V) can be placed in the type column (T) instead. The i'th entry\nin the result slice is true if the type column in spec[i] must be kept.\n\nFor example, the declaration:\n\n\t\tconst (\n\t\t\tfoobar int = 42 // comment\n\t\t\tx = 7 // comment\n\t\t\tfoo\n\t bar = 991\n\t\t)\n\nleads to the type/values matrix below. A run of value columns (V) can\nbe moved into the type column if there is no type for any of the values\nin that column (we only move entire columns so that they align properly).\n\n\t\tmatrix formatted result\n\t matrix\n\t\tT V -> T V -> true there is a T and so the type\n\t\t- V - V true column must be kept\n\t\t- - - - false\n\t\t- V V - false V is moved into T column", Args: []string{"specs"}, Returns: []string{"[]bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.sanitizeImportPath", Args: []string{"lit"}, Returns: []string{"BasicLit"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.declToken", Args: []string{"decl"}, Returns: []string{"tok"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.isBlank", Doc: "Returns true if s contains only white space\n(only tabs and blanks can appear in the printer's context).", Args: []string{"s"}, Returns: []string{"bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.commonPrefix", Doc: "commonPrefix returns the common prefix of a and b.", Args: []string{"a", "b"}, Returns: []string{"string"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.trimRight", Doc: "trimRight returns s with trailing whitespace removed.", Args: []string{"s"}, Returns: []string{"string"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.stripCommonPrefix", Doc: "stripCommonPrefix removes a common prefix from /*-style comment lines (unless no\ncomment line is indented, all but the first line have some form of space prefix).\nThe prefix is computed using heuristics such that is likely that the comment\ncontents are nicely laid out after re-printing each line using the printer's\ncurrent indentation.", Args: []string{"lines"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.nlimit", Doc: "nlimit limits n to maxNewlines.", Args: []string{"n"}, Returns: []string{"int"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.mayCombine", Args: []string{"prev", "next"}, Returns: []string{"b"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getDoc", Doc: "getDoc returns the ast.CommentGroup associated with n, if any.", Args: []string{"n"}, Returns: []string{"CommentGroup"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.getLastComment", Args: []string{"n"}, Returns: []string{"CommentGroup"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.newPrinter", Args: []string{"cfg", "pkg", "nodeSizes"}, Returns: []string{"printer"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.Fprint", Doc: "Fprint \"pretty-prints\" an AST node to output.\nIt calls [PrintConfig.Fprint] with default settings.\nNote that gofmt uses tabs for indentation but spaces for alignment;\nuse format.Node (package go/format) for output that matches gofmt.", Args: []string{"output", "pkg", "node"}, Returns: []string{"error"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.wgslFile", Doc: "wgslFile returns the file with a \".wgsl\" extension", Args: []string{"fn"}, Returns: []string{"string"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.bareFile", Doc: "bareFile returns the file with no extention", Args: []string{"fn"}, Returns: []string{"string"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.MoveLines", Doc: "MoveLines moves the st,ed region to 'to' line", Args: []string{"lines", "to", "st", "ed"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlEdits", Doc: "SlEdits performs post-generation edits for wgsl\n* moves wgsl segments around, e.g., methods\ninto their proper classes\n* fixes printf, slice other common code\nreturns true if a slrand. or sltype. prefix was found,\ndriveing copying of those files.", Args: []string{"src"}, Returns: []string{"[]byte", "bool", "bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.MathReplaceAll", Args: []string{"mat", "ln"}, Returns: []string{"[]byte"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlEditsReplace", Doc: "SlEditsReplace replaces Go with equivalent WGSL code\nreturns true if has slrand. or sltype.\nto auto include that header file if so.", Args: []string{"lines"}, Returns: []string{"bool", "bool"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/parse.SlBoolReplace", Doc: "SlBoolReplace replaces all the slbool methods with literal int32 expressions.", Args: []string{"lines"}}) From d8a6f55628e7dab28c30cf4ec7e36acf39927ddb Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 13:22:34 -0700 Subject: [PATCH 186/311] document process, cleanup names -- basic translation phase good; now work on extracting kernels and vars, and then generating code. --- goal/gosl/gotosl/README.md | 23 ++++ goal/gosl/gotosl/extract.go | 43 +++---- goal/gosl/gotosl/files.go | 72 ++++++++--- goal/gosl/gotosl/gotosl.go | 121 ++++++++++++++++++ goal/gosl/gotosl/sledits.go | 16 +-- goal/gosl/gotosl/state.go | 105 --------------- goal/gosl/gotosl/{process.go => translate.go} | 64 ++------- 7 files changed, 231 insertions(+), 213 deletions(-) create mode 100644 goal/gosl/gotosl/README.md create mode 100644 goal/gosl/gotosl/gotosl.go delete mode 100644 goal/gosl/gotosl/state.go rename goal/gosl/gotosl/{process.go => translate.go} (63%) diff --git a/goal/gosl/gotosl/README.md b/goal/gosl/gotosl/README.md new file mode 100644 index 0000000000..af647c0218 --- /dev/null +++ b/goal/gosl/gotosl/README.md @@ -0,0 +1,23 @@ +# Implementational details of Go to SL translation process + +Overall, there are three main steps: + +1. Translate all the `.go` files in the current package, and all the files they `//gosl:import`, into corresponding `.wgsl` files, and put those in `shaders/imports`. All these files will be pasted into the generated primary kernel files, that go in `shaders`, and are saved to disk for reference. All the key kernel, system, variable info is extracted from the package .go file directives during this phase. + +2. Generate the `main` kernel `.wgsl` files, for each kernel function, which: a) declare the global buffer variables; b) include everything from imports; c) define the `main` function entry point. Each resulting file is pre-processed by `naga` to ensure it compiles, and to remove dead code not needed for this particular shader. + +3. Generate the `gosl.go` file in the package directory, which contains generated Go code for configuring the gpu compute systems according to the vars. + +## Go to SL translation + +1. `files.go`: Get a list of all the .go files in the current directory that have a `//gosl:` tag (`ProjectFiles`) and all the `//gosl:import` package files that those files import, recursively. + +2. `extract.go`: Extract the `//gosl:start` -> `end` regions from all the package and imported filees. + +3. Save all these files as new `.go` files in `shaders/imports`. We manually append a simple go "main" package header with basic gosl imports for each file, which allows the go compiler to process them properly. This is then removed in the next step. + +4. `translate.go:` Run `TranslateDir` on shaders/imports using the "golang.org/x/tools/go/packages" `Load` function, which gets `ast` and type information for all that code. Run the resulting `ast.File` for each file through the modified version of the Go stdlib `src/go/printer` code (`printer.go`, `nodes.go`, `gobuild.go`, `comment.go`), which prints out WGSL instead of Go code from the underlying `ast` representation of the Go files. This is what does the actual translation. + +5. `sledits.go:` Do various forms of post-processing text replacement cleanup on the generated WGSL files, in `SLEdits` function. + + diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index 547de10cc8..ba6cb1ab5d 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -11,27 +11,32 @@ import ( "slices" ) +// ExtractFiles processes all the package files and saves the corresponding +// .go files with simple go header. +func (st *State) ExtractFiles() { + for fn, fl := range st.GoFiles { + fl.Lines = st.ExtractGosl(fl.Lines) + st.WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) + } +} + // ExtractImports processes all the imported files and saves the corresponding -// .go files, and +// .go files with simple go header. func (st *State) ExtractImports() { - if len(st.Imports) == 0 { + if len(st.GoImports) == 0 { return } st.ImportPackages = make(map[string]bool) - for impath := range st.Imports { + for impath := range st.GoImports { _, pkg := filepath.Split(impath) st.ImportPackages[pkg] = true } - for _, im := range st.Imports { + for _, im := range st.GoImports { for fn, fl := range im { fl.Lines = st.ExtractGosl(fl.Lines) - lns := st.AppendGoHeader(fl.Lines) - ifn := filepath.Join("imports", fn) - st.WriteFileLines(ifn, lns) + st.WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) } } - pdir := "./" + filepath.Join(st.Config.Output, "imports") - st.ProcessDir(pdir) } // ExtractGosl gosl comment-directive tagged regions from given file. @@ -101,9 +106,9 @@ func (st *State) AppendGoHeader(lines [][]byte) [][]byte { return olns } -// ExtractWGSL extracts the WGSL code embedded within .Go files. -// Returns true if WGSL contains a void main( function. -func (st *State) ExtractWGSL(b []byte) ([][]byte, bool) { +// ExtractWGSL extracts the WGSL code embedded within .Go files, +// which is commented out in the Go code -- remove comments. +func (st *State) ExtractWGSL(lines [][]byte) [][]byte { key := []byte("//gosl:") wgsl := []byte("wgsl") nowgsl := []byte("nowgsl") @@ -113,12 +118,8 @@ func (st *State) ExtractWGSL(b []byte) ([][]byte, bool) { comment := []byte("// ") pack := []byte("package") imp := []byte("import") - main := []byte("void main(") lparen := []byte("(") rparen := []byte(")") - nl := []byte("\n") - - lines := bytes.Split(b, nl) mx := min(10, len(lines)) stln := 0 @@ -141,7 +142,6 @@ func (st *State) ExtractWGSL(b []byte) ([][]byte, bool) { lines = lines[stln:] // get rid of package, import - hasMain := false inHlsl := false inNoHlsl := false noHlslStart := 0 @@ -163,20 +163,13 @@ func (st *State) ExtractWGSL(b []byte) ([][]byte, bool) { li-- inHlsl = false case inHlsl: - del := false switch { case bytes.HasPrefix(ln, stComment) || bytes.HasPrefix(ln, edComment): lines = slices.Delete(lines, li, li+1) li-- - del = true case bytes.HasPrefix(ln, comment): lines[li] = ln[3:] } - if !del { - if bytes.HasPrefix(lines[li], main) { - hasMain = true - } - } case isKey && bytes.HasPrefix(keyStr, wgsl): inHlsl = true lines = slices.Delete(lines, li, li+1) @@ -186,5 +179,5 @@ func (st *State) ExtractWGSL(b []byte) ([][]byte, bool) { noHlslStart = li } } - return lines, hasMain + return lines } diff --git a/goal/gosl/gotosl/files.go b/goal/gosl/gotosl/files.go index 5023873349..d8885b4c5a 100644 --- a/goal/gosl/gotosl/files.go +++ b/goal/gosl/gotosl/files.go @@ -7,6 +7,7 @@ package gotosl import ( "bytes" "fmt" + "io" "io/fs" "log" "os" @@ -41,9 +42,8 @@ func ReadFileLines(fn string) ([][]byte, error) { } func (st *State) WriteFileLines(fn string, lines [][]byte) error { - outfn := filepath.Join(st.Config.Output, fn) res := bytes.Join(lines, []byte("\n")) - return os.WriteFile(outfn, res, 0644) + return os.WriteFile(fn, res, 0644) } // HasGoslTag returns true if given file has a //gosl: tag @@ -58,10 +58,6 @@ func HasGoslTag(lines [][]byte) bool { return false } -// LoadedPackageNames are single prefix names of packages that were -// loaded in the list of files to process -var LoadedPackageNames = map[string]bool{} - func IsGoFile(f fs.DirEntry) bool { name := f.Name() return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") && !f.IsDir() @@ -72,15 +68,10 @@ func IsWGSLFile(f fs.DirEntry) bool { return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".wgsl") && !f.IsDir() } -func IsSPVFile(f fs.DirEntry) bool { - name := f.Name() - return !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".spv") && !f.IsDir() -} - // ProjectFiles gets the files in the current directory. func (st *State) ProjectFiles() { fls := fsx.Filenames(".", ".go") - st.Files = make(map[string]*File) + st.GoFiles = make(map[string]*File) for _, fn := range fls { fl := &File{Name: fn} var err error @@ -91,7 +82,7 @@ func (st *State) ProjectFiles() { if !HasGoslTag(fl.Lines) { continue } - st.Files[fn] = fl + st.GoFiles[fn] = fl st.ImportFiles(fl.Lines) } } @@ -112,7 +103,7 @@ func (st *State) ImportFiles(lines [][]byte) { if impath[len(impath)-1] == '"' { impath = impath[:len(impath)-1] } - _, ok := st.Imports[impath] + _, ok := st.GoImports[impath] if ok { continue } @@ -124,7 +115,7 @@ func (st *State) ImportFiles(lines [][]byte) { continue } pfls := make(map[string]*File) - st.Imports[impath] = pfls + st.GoImports[impath] = pfls pkg := pkgs[0] gofls := pkg.GoFiles if len(gofls) == 0 { @@ -143,7 +134,7 @@ func (st *State) ImportFiles(lines [][]byte) { st.ImportFiles(lns) // fmt.Printf("added file: %s from package: %s\n", gf, impath) } - st.Imports[impath] = pfls + st.GoImports[impath] = pfls } } @@ -153,7 +144,7 @@ func RemoveGenFiles(dir string) { if err != nil { return err } - if IsGoFile(f) || IsWGSLFile(f) || IsSPVFile(f) { + if IsGoFile(f) || IsWGSLFile(f) { os.Remove(path) } return nil @@ -162,3 +153,50 @@ func RemoveGenFiles(dir string) { log.Println(err) } } + +// CopyPackageFile copies given file name from given package path +// into the current imports directory. +// e.g., "slrand.wgsl", "cogentcore.org/core/goal/gosl/slrand" +func (st *State) CopyPackageFile(fnm, packagePath string) error { + tofn := filepath.Join(st.ImportsDir, fnm) + pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles}, packagePath) + if err != nil { + fmt.Println(err) + return err + } + if len(pkgs) != 1 { + err = fmt.Errorf("%s package not found", packagePath) + fmt.Println(err) + return err + } + pkg := pkgs[0] + var fn string + if len(pkg.GoFiles) > 0 { + fn = pkg.GoFiles[0] + } else if len(pkg.OtherFiles) > 0 { + fn = pkg.GoFiles[0] + } else { + err = fmt.Errorf("No files found in package: %s", packagePath) + fmt.Println(err) + return err + } + dir, _ := filepath.Split(fn) + fmfn := filepath.Join(dir, fnm) + CopyFile(fmfn, tofn) + return nil +} + +func CopyFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, in) + return err +} diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go new file mode 100644 index 0000000000..807cdaf8fe --- /dev/null +++ b/goal/gosl/gotosl/gotosl.go @@ -0,0 +1,121 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gotosl + +import ( + "errors" + "os" + "path/filepath" + "strings" +) + +// System represents a ComputeSystem, and its kernels and variables. +type System struct { + Name string + + // Kernels are the kernels using this compute system. + Kernels map[string]*Kernel + + // Groups are the variables for this compute system. + Groups []*Group +} + +func NewSystem(name string) *System { + sy := &System{Name: name} + sy.Kernels = make(map[string]*Kernel) + sy.Groups = append(sy.Groups, &Group{}) + return sy +} + +// Kernel represents a kernel function, which is the basis for +// each wgsl generated code file. +type Kernel struct { + Name string + + // accumulating lines of code for the wgsl file. + FileLines [][]byte +} + +// Var represents one variable +type Var struct { + Name string + + Type string +} + +// Group represents one variable group. +type Group struct { + Vars []*Var +} + +// File has contents of a file as lines of bytes. +type File struct { + Name string + Lines [][]byte +} + +// State holds the current Go -> WGSL processing state. +type State struct { + // Config options. + Config *Config + + // path to shaders/imports directory. + ImportsDir string + + // GoFiles are all the files with gosl content in current directory. + GoFiles map[string]*File + + // GoImports has all the imported files. + GoImports map[string]map[string]*File + + // ImportPackages has short package names, to remove from go code + // so everything lives in same main package. + ImportPackages map[string]bool + + // Systems has the kernels and variables for each system. + // There is an initial "Default" system when system is not specified. + Systems map[string]*System + + // SLImportFiles are all the extracted and translated WGSL files in shaders/imports, + // which are copied into the generated shader kernel files. + SLImportFiles []*File + + // ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation. + ExcludeMap map[string]bool +} + +func (st *State) Init(cfg *Config) { + st.Config = cfg + st.GoImports = make(map[string]map[string]*File) + st.Systems = make(map[string]*System) + st.ExcludeMap = make(map[string]bool) + ex := strings.Split(cfg.Exclude, ",") + for _, fn := range ex { + st.ExcludeMap[fn] = true + } + st.Systems["Default"] = NewSystem("Default") +} + +func (st *State) Run() error { + if gomod := os.Getenv("GO111MODULE"); gomod == "off" { + err := errors.New("gosl only works in go modules mode, but GO111MODULE=off") + return err + } + if st.Config.Output == "" { + st.Config.Output = "shaders" + } + st.ImportsDir = filepath.Join(st.Config.Output, "imports") + os.MkdirAll(st.Config.Output, 0755) + os.MkdirAll(st.ImportsDir, 0755) + RemoveGenFiles(st.Config.Output) + RemoveGenFiles(st.ImportsDir) + + st.ProjectFiles() // get list of all files, recursively gets imports etc. + st.ExtractFiles() // get .go from project files + st.ExtractImports() // get .go from imports + st.TranslateDir("./" + st.ImportsDir) + + return nil +} diff --git a/goal/gosl/gotosl/sledits.go b/goal/gosl/gotosl/sledits.go index 020aa4c6b4..3361b2c869 100644 --- a/goal/gosl/gotosl/sledits.go +++ b/goal/gosl/gotosl/sledits.go @@ -21,19 +21,15 @@ func MoveLines(lines *[][]byte, to, st, ed int) { *lines = nln } -// SlEdits performs post-generation edits for wgsl -// * moves wgsl segments around, e.g., methods -// into their proper classes -// * fixes printf, slice other common code +// SlEdits performs post-generation edits for wgsl, +// replacing type names, slbool, function calls, etc. // returns true if a slrand. or sltype. prefix was found, // driveing copying of those files. -func SlEdits(src []byte) ([]byte, bool, bool) { - // return src // uncomment to show original without edits +func SlEdits(src []byte) (lines [][]byte, hasSlrand bool, hasSltype bool) { nl := []byte("\n") - lines := bytes.Split(src, nl) - hasSlrand, hasSltype := SlEditsReplace(lines) - - return bytes.Join(lines, nl), hasSlrand, hasSltype + lines = bytes.Split(src, nl) + hasSlrand, hasSltype = SlEditsReplace(lines) + return } type Replace struct { diff --git a/goal/gosl/gotosl/state.go b/goal/gosl/gotosl/state.go deleted file mode 100644 index ee4e04da19..0000000000 --- a/goal/gosl/gotosl/state.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package gotosl - -import ( - "errors" - "os" - "path/filepath" - "strings" -) - -// Var represents one variable -type Var struct { - Name string - - Type string -} - -// Group represents one variable group. -type Group struct { - Vars []*Var -} - -// System represents a ComputeSystem, and its variables. -type System struct { - Name string - - Groups []*Group -} - -// Kernel represents a kernel function, which is the basis for -// each wgsl generated code file. -type Kernel struct { - Name string - - // accumulating lines of code for the wgsl file. - FileLines [][]byte -} - -// File has info for a file being processed -type File struct { - Name string - - Lines [][]byte -} - -// State holds the current processing state -type State struct { - // Config options - Config *Config - - // files with gosl content in current directory - Files map[string]*File - - // has all the imports - Imports map[string]map[string]*File - - // short package names - ImportPackages map[string]bool - - Kernels map[string]*Kernel - - Systems map[string]*System - - // ExcludeMap is the compiled map of functions to exclude. - ExcludeMap map[string]bool -} - -func (st *State) Init(cfg *Config) { - st.Config = cfg - st.Imports = make(map[string]map[string]*File) - st.Kernels = make(map[string]*Kernel) - st.Systems = make(map[string]*System) - st.ExcludeMap = make(map[string]bool) - ex := strings.Split(cfg.Exclude, ",") - for _, fn := range ex { - st.ExcludeMap[fn] = true - } - - sy := &System{Name: "Default"} - sy.Groups = append(sy.Groups, &Group{}) - st.Systems["Default"] = sy -} - -func (st *State) Run() error { - if gomod := os.Getenv("GO111MODULE"); gomod == "off" { - err := errors.New("gosl only works in go modules mode, but GO111MODULE=off") - return err - } - if st.Config.Output == "" { - st.Config.Output = "shaders" - } - imps := filepath.Join(st.Config.Output, "imports") - os.MkdirAll(st.Config.Output, 0755) - os.MkdirAll(imps, 0755) - RemoveGenFiles(st.Config.Output) - RemoveGenFiles(imps) - - st.ProjectFiles() // recursively gets imports etc. - st.ExtractImports() // save all the import files - - return nil -} diff --git a/goal/gosl/gotosl/process.go b/goal/gosl/gotosl/translate.go similarity index 63% rename from goal/gosl/gotosl/process.go rename to goal/gosl/gotosl/translate.go index b9b66d45d6..476f7c5ebb 100644 --- a/goal/gosl/gotosl/process.go +++ b/goal/gosl/gotosl/translate.go @@ -22,8 +22,8 @@ import ( "golang.org/x/tools/go/packages" ) -// ProcessDir process files in given directory. -func (st *State) ProcessDir(pf string) error { +// TranslateDir translate all .Go files in given directory to WGSL. +func (st *State) TranslateDir(pf string) error { nl := []byte("\n") pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesSizes | packages.NeedTypesInfo}, pf) if err != nil { @@ -42,9 +42,6 @@ func (st *State) ProcessDir(pf string) error { // return nil, err files := pkg.GoFiles - // map of files with a main function that needs to be compiled - needsCompile := map[string]bool{} - serr := alignsl.CheckPackage(pkg) if serr != nil { @@ -58,7 +55,7 @@ func (st *State) ProcessDir(pf string) error { _, gofn := filepath.Split(gofp) wgfn := wgslFile(gofn) if st.Config.Debug { - fmt.Printf("###################################\nProcessing Go file: %s\n", gofn) + fmt.Printf("###################################\nTranslating Go file: %s\n", gofn) } var afile *ast.File @@ -73,7 +70,7 @@ func (st *State) ProcessDir(pf string) error { } } if afile == nil { - fmt.Printf("Warning: File named: %s not found in processed package\n", gofn) + fmt.Printf("Warning: File named: %s not found in Loaded package\n", gofn) continue } @@ -82,76 +79,31 @@ func (st *State) ProcessDir(pf string) error { pcfg.Fprint(&buf, pkg, afile) // ioutil.WriteFile(filepath.Join(*outDir, fn+".tmp"), buf.Bytes(), 0644) slfix, hasSltype, hasSlrand := SlEdits(buf.Bytes()) - _ = slfix if hasSlrand && !slrandCopied { hasSltype = true if st.Config.Debug { fmt.Printf("\tcopying slrand.wgsl to shaders\n") } - // st.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/goal/gosl/slrand") + st.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/goal/gosl/slrand") slrandCopied = true } if hasSltype && !sltypeCopied { if st.Config.Debug { fmt.Printf("\tcopying sltype.wgsl to shaders\n") } - // st.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/goal/gosl/sltype") + st.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/goal/gosl/sltype") sltypeCopied = true } - exsl, hasMain := st.ExtractWGSL(slfix) - _ = hasMain - // gosls[fn] = exsl + exsl := st.ExtractWGSL(slfix) - // if hasMain { - // needsCompile[fn] = true - // } if !st.Config.Keep { os.Remove(fpos.Filename) } - // add wgsl code - // for _, slfn := range wgslFiles { - // if fn+".wgsl" != slfn { - // continue - // } - // buf, err := os.ReadFile(slfn) - // if err != nil { - // fmt.Println(err) - // continue - // } - // exsl = append(exsl, []byte(fmt.Sprintf("\n// from file: %s\n", slfn))...) - // exsl = append(exsl, buf...) - // gosls[fn] = exsl - // needsCompile[fn] = true // assume any standalone has main - // break - // } - slfn := filepath.Join(st.Config.Output, wgfn) ioutil.WriteFile(slfn, bytes.Join(exsl, nl), 0644) - } - - // check for wgsl files that had no go equivalent - - // for _, slfn := range wgslFiles { - // hasGo := false - // for fn := range gosls { - // if fn+".wgsl" == slfn { - // hasGo = true - // break - // } - // } - // if hasGo { - // continue - // } - // _, slfno := filepath.Split(slfn) // could be in a subdir - // tofn := filepath.Join(st.Config.Output, slfno) - // CopyFile(slfn, tofn) - // fn := strings.TrimSuffix(slfno, ".wgsl") - // needsCompile[fn] = true // assume any standalone wgsl is a main - // } - for fn := range needsCompile { - st.CompileFile(fn + ".wgsl") + st.SLImportFiles = append(st.SLImportFiles, &File{Name: wgfn, Lines: exsl}) } return nil } From e9b5c4eb202180cf3cf4ad00b9edbca2ec5d5c6c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 15:46:25 -0700 Subject: [PATCH 187/311] extracting args and kernels --- goal/GPU.md | 9 ++- goal/gosl/examples/basic/compute.go | 7 +- goal/gosl/gotosl/extract.go | 18 +++++ goal/gosl/gotosl/gotosl.go | 29 +++++++- goal/gosl/gotosl/nodes.go | 100 ++++++++++++++++++++++++++-- goal/gosl/gotosl/printer.go | 8 ++- goal/gosl/gotosl/translate.go | 2 +- 7 files changed, 153 insertions(+), 20 deletions(-) diff --git a/goal/GPU.md b/goal/GPU.md index 800fa7afe8..3bf5e950fd 100644 --- a/goal/GPU.md +++ b/goal/GPU.md @@ -90,19 +90,18 @@ The following comment directive can be used in any kernel file to specify which To define the global variables for each system, use a standard Go `var` block declaration (with optional system name qualifier): ```Go -var ( //gosl:vars [system name] - - //gosl:group -uniform Params - +//gosl:vars [system name] +var ( // Layer-level parameters + //gosl:group -uniform Params Layers []LayerParam // note: struct with appropriate memory alignment // Path-level parameters Paths []PathParam - //gosl:group Units // Unit state values + //gosl:group Units Units tensor.Float32 // Synapse weight state values diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index e6d7984ddf..1bca9f40c6 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -9,11 +9,12 @@ import "cogentcore.org/core/math32" //gosl:start //gosl:import "cogentcore.org/core/math32" -var ( //gosl:vars - // Params are the parameters for the computation +//gosl:vars +var ( + // Params are the parameters for the computation. Params []ParamStruct - // Data is the data on which the computation operates + // Data is the data on which the computation operates. Data []DataStruct ) diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index ba6cb1ab5d..14254d3441 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -6,7 +6,9 @@ package gotosl import ( "bytes" + "fmt" "path/filepath" + "strings" "slices" ) @@ -47,6 +49,8 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { nowgsl := []byte("nowgsl") end := []byte("end") imp := []byte("import") + kernel := []byte("//gosl:kernel") + fnc := []byte("func") inReg := false inHlsl := false @@ -74,6 +78,20 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { ln = bytes.ReplaceAll(ln, []byte(pkg+"."), []byte{}) } } + if bytes.HasPrefix(ln, fnc) && bytes.Contains(ln, kernel) { + sysnm := strings.TrimSpace(string(ln[bytes.LastIndex(ln, kernel)+len(kernel):])) + sy := st.System(sysnm) + fcall := string(ln[5:]) + lp := strings.Index(fcall, "(") + rp := strings.LastIndex(fcall, ")") + args := fcall[lp+1 : rp] + fnm := fcall[:lp] + kn := &Kernel{Name: fnm, Args: args} + sy.Kernels[fnm] = kn + if st.Config.Debug { + fmt.Println("\tAdded kernel:", fnm, "args:", args, "system:", sy.Name) + } + } outLns = append(outLns, ln) case isKey && bytes.HasPrefix(keyStr, start): inReg = true diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 807cdaf8fe..e8ae912042 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -25,7 +25,6 @@ type System struct { func NewSystem(name string) *System { sy := &System{Name: name} sy.Kernels = make(map[string]*Kernel) - sy.Groups = append(sy.Groups, &Group{}) return sy } @@ -34,19 +33,30 @@ func NewSystem(name string) *System { type Kernel struct { Name string + Args string + // accumulating lines of code for the wgsl file. FileLines [][]byte } -// Var represents one variable +// Var represents one global system buffer variable. type Var struct { Name string + // Type of variable: either []Type or tensor.Float32, tensor.Int32 Type string } // Group represents one variable group. type Group struct { + Name string + + // comment docs about this group + Doc string + + // Uniform indicates a uniform group; else default is Storage + Uniform bool + Vars []*Var } @@ -119,3 +129,18 @@ func (st *State) Run() error { return nil } + +// System returns the given system by name, making if not made. +// if name is empty, "Default" is used. +func (st *State) System(sysname string) *System { + if sysname == "" { + sysname = "Default" + } + sy, ok := st.Systems[sysname] + if ok { + return sy + } + sy = NewSystem(sysname) + st.Systems[sysname] = sy + return sy +} diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 49d1942c00..639aa5ba62 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -27,6 +27,8 @@ import ( "strings" "unicode" "unicode/utf8" + + "cogentcore.org/core/base/errors" ) // Formatting issues: @@ -71,6 +73,21 @@ func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (nbre return } +// gosl: find any gosl directive in given comments, returns directive and remaining docs +func (p *printer) findDirective(g *ast.CommentGroup) (dir string, docs string) { + if g == nil { + return + } + for _, c := range g.List { + if strings.HasPrefix(c.Text, "//gosl:") { + dir = c.Text[7:] + } else { + docs += c.Text + " " + } + } + return +} + // setComment sets g as the next comment if g != nil and if node comments // are enabled - this mode is used when printing source code fragments such // as exports only. It assumes that there is no pending comment in p.comments @@ -1432,7 +1449,7 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa break } err = fmt.Errorf("gosl methodPath ERROR: path for method call must be simple list of fields, not %#v:", cur.X) - fmt.Println(err.Error()) + errors.Log(err) return } if p.isPtrArg(baseRecv) { @@ -1452,7 +1469,7 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa f := fieldByName(curt, p) if f == nil { err = fmt.Errorf("gosl ERROR: field not found %q in type: %q:", p, curt.String()) - fmt.Println(err.Error()) + errors.Log(err) return } if pi == 0 { @@ -1503,8 +1520,7 @@ func getStructType(typ types.Type) (*types.Struct, error) { } } err := fmt.Errorf("gosl ERROR: type is not a struct and it should be: %q %+t", typ.String(), typ) - fmt.Println(err.Error()) - return nil, err + return nil, errors.Log(err) } func (p *printer) methodExpr(x *ast.CallExpr, depth int) { @@ -1537,7 +1553,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { } } else { err := fmt.Errorf("gosl methodExpr ERROR: path expression for method call must be simple list of fields, not %#v:", path.X) - fmt.Println(err.Error()) + errors.Log(err) return } if pathIsPackage { @@ -2233,6 +2249,69 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, tok token.Token) { } } +// gosl: process system vars +func (p *printer) systemVars(d *ast.GenDecl, sysname string) { + sy := p.GoToSL.System(sysname) + var gp *Group + for _, s := range d.Specs { + vs := s.(*ast.ValueSpec) + dir, docs := p.findDirective(vs.Doc) + if strings.HasPrefix(dir, "group") { + gpnm := strings.TrimSpace(dir[5:]) + if gpnm == "" { + gp = &Group{Name: fmt.Sprintf("Group_%d", len(sy.Groups)), Doc: docs} + sy.Groups = append(sy.Groups, gp) + } else { + gps := strings.Fields(gpnm) + gp = &Group{Doc: docs} + if gps[0] == "-uniform" { + gp.Uniform = true + if len(gps) > 1 { + gp.Name = gps[1] + } + } else { + gp.Name = gps[0] + } + sy.Groups = append(sy.Groups, gp) + } + } + if gp == nil { + gp = &Group{Name: fmt.Sprintf("Group_%d", len(sy.Groups)), Doc: docs} + sy.Groups = append(sy.Groups, gp) + } + if len(vs.Names) != 1 { + errors.Log(fmt.Errorf("gosl: system %q: vars must have only 1 variable per line", sysname)) + } + nm := vs.Names[0].Name + typ := "" + if sl, ok := vs.Type.(*ast.ArrayType); ok { + id, ok := sl.Elt.(*ast.Ident) + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: Var type not recognized: %#v", sysname, sl.Elt)) + continue + } + typ = "[]" + id.Name + } else { + sel, ok := vs.Type.(*ast.SelectorExpr) + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Int32", sysname)) + continue + } + sid, ok := sel.X.(*ast.Ident) + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: Var type selector is not recognized: %#v", sysname, sel.X)) + continue + } + typ = sid.Name + "." + sel.Sel.Name + } + vr := &Var{Name: nm, Type: typ} + gp.Vars = append(gp.Vars, vr) + if p.GoToSL.Config.Debug { + fmt.Println("\tAdded var:", nm, typ, "to group:", gp.Name) + } + } +} + func (p *printer) genDecl(d *ast.GenDecl) { p.setComment(d.Doc) // note: critical to print here to trigger comment generation in right place @@ -2251,6 +2330,13 @@ func (p *printer) genDecl(d *ast.GenDecl) { // p.print(indent, formfeed) if n > 1 && (d.Tok == token.CONST || d.Tok == token.VAR) { // two or more grouped const/var declarations: + if d.Tok == token.VAR { + dir, _ := p.findDirective(d.Doc) + if strings.HasPrefix(dir, "vars") { + p.systemVars(d, strings.TrimSpace(dir[4:])) + return + } + } // determine if the type column must be kept keepType := keepTypeColumn(d.Specs) firstSpec := d.Specs[0].(*ast.ValueSpec) @@ -2264,11 +2350,13 @@ func (p *printer) genDecl(d *ast.GenDecl) { } var line int for i, s := range d.Specs { + vs := s.(*ast.ValueSpec) + // p.findDirective(vs.Doc) if i > 0 { p.linebreak(p.lineFor(s.Pos()), 1, ignore, p.linesFrom(line) > 0) } p.recordLine(&line) - p.valueSpec(s.(*ast.ValueSpec), keepType[i], d.Tok, firstSpec, isIota, i) + p.valueSpec(vs, keepType[i], d.Tok, firstSpec, isIota, i) } } else { var line int diff --git a/goal/gosl/gotosl/printer.go b/goal/gosl/gotosl/printer.go index cf341330e3..1c1a78aca4 100644 --- a/goal/gosl/gotosl/printer.go +++ b/goal/gosl/gotosl/printer.go @@ -62,6 +62,7 @@ type commentInfo struct { type printer struct { // Configuration (does not change after initialization) PrintConfig + fset *token.FileSet pkg *packages.Package // gosl: extra @@ -1330,9 +1331,10 @@ const ( // A PrintConfig node controls the output of Fprint. type PrintConfig struct { - Mode Mode // default: 0 - Tabwidth int // default: 8 - Indent int // default: 0 (all code is indented at least by this much) + Mode Mode // default: 0 + Tabwidth int // default: 8 + Indent int // default: 0 (all code is indented at least by this much) + GoToSL *State // gosl: ExcludeFunctions map[string]bool } diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 476f7c5ebb..5de72be420 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -75,7 +75,7 @@ func (st *State) TranslateDir(pf string) error { } var buf bytes.Buffer - pcfg := PrintConfig{Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: st.ExcludeMap} + pcfg := PrintConfig{GoToSL: st, Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: st.ExcludeMap} pcfg.Fprint(&buf, pkg, afile) // ioutil.WriteFile(filepath.Join(*outDir, fn+".tmp"), buf.Bytes(), 0644) slfix, hasSltype, hasSlrand := SlEdits(buf.Bytes()) From 2a36add51ea73fe2fb8a40b3715e7b0bfd8bb4c9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 16:58:45 -0700 Subject: [PATCH 188/311] generating kernel code --- goal/gosl/gotosl/extract.go | 14 +++++++++++--- goal/gosl/gotosl/gotosl.go | 16 ++++++++++++++-- goal/gosl/gotosl/translate.go | 2 +- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index 14254d3441..459ecd15f6 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022, Cogent Core. All rights reserved. +// Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -56,7 +56,7 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { inHlsl := false inNoHlsl := false var outLns [][]byte - for _, ln := range lines { + for li, ln := range lines { tln := bytes.TrimSpace(ln) isKey := bytes.HasPrefix(tln, key) var keyStr []byte @@ -86,7 +86,15 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { rp := strings.LastIndex(fcall, ")") args := fcall[lp+1 : rp] fnm := fcall[:lp] - kn := &Kernel{Name: fnm, Args: args} + funcode := "" + for ki := li + 1; ki < len(lines); ki++ { + kl := lines[ki] + if len(kl) > 0 && kl[0] == '}' { + break + } + funcode += string(kl) + "\n" + } + kn := &Kernel{Name: fnm, Args: args, FuncCode: funcode} sy.Kernels[fnm] = kn if st.Config.Debug { fmt.Println("\tAdded kernel:", fnm, "args:", args, "system:", sy.Name) diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index e8ae912042..78158d28ee 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -35,14 +35,20 @@ type Kernel struct { Args string - // accumulating lines of code for the wgsl file. - FileLines [][]byte + // function code + FuncCode string + + // Lines is full shader code + Lines [][]byte } // Var represents one global system buffer variable. type Var struct { Name string + // comment docs about this var. + Doc string + // Type of variable: either []Type or tensor.Float32, tensor.Int32 Type string } @@ -127,6 +133,12 @@ func (st *State) Run() error { st.ExtractImports() // get .go from imports st.TranslateDir("./" + st.ImportsDir) + for _, sy := range st.Systems { + for _, kn := range sy.Kernels { + st.GenKernel(sy, kn) + } + } + return nil } diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 5de72be420..480451324f 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -100,7 +100,7 @@ func (st *State) TranslateDir(pf string) error { os.Remove(fpos.Filename) } - slfn := filepath.Join(st.Config.Output, wgfn) + slfn := filepath.Join(st.ImportsDir, wgfn) ioutil.WriteFile(slfn, bytes.Join(exsl, nl), 0644) st.SLImportFiles = append(st.SLImportFiles, &File{Name: wgfn, Lines: exsl}) From 7603f84db178720eda4585d0b13e9e48c04d936c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 17:23:24 -0700 Subject: [PATCH 189/311] kernel gen working --- goal/GPU.md | 2 +- goal/gosl/examples/basic/compute.go | 8 ++++++-- goal/gosl/examples/basic/main.go | 2 +- goal/gosl/gotosl/translate.go | 14 +------------- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/goal/GPU.md b/goal/GPU.md index 3bf5e950fd..e28155fd0f 100644 --- a/goal/GPU.md +++ b/goal/GPU.md @@ -64,7 +64,7 @@ The GPU code can only handle a highly restricted _subset_ of Go code, with data ```Go // Compute does the main computation. -func Compute(i int32) { //gosl:kernel +func Compute(i uint32) { //gosl:kernel Params[0].IntegFromRaw(&Data[i]) } ``` diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 1bca9f40c6..5f44bdfcee 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -54,8 +54,12 @@ func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) { } // Compute does the main computation -func Compute(i int32) { //gosl:kernel - Params[0].IntegFromRaw(&Data[i]) +func Compute(i uint32) { //gosl:kernel + // Params[0].IntegFromRaw(&Data[i]) + var params = Params[0] + var data = Data[i] + params.IntegFromRaw(&data) + Data[i] = data } //gosl:end diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index 5bf40ca88d..4e0d40fb72 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -19,7 +19,7 @@ import ( //go:generate gosl . -//go:embed shaders/basic.wgsl shaders/fastexp.wgsl +//go:embed shaders/Compute.wgsl var shaders embed.FS func init() { diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 480451324f..82edfb09f3 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -9,7 +9,6 @@ import ( "fmt" "go/ast" "go/token" - "io/fs" "io/ioutil" "log" "os" @@ -18,7 +17,6 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/goal/gosl/alignsl" - "cogentcore.org/core/gpu" "golang.org/x/tools/go/packages" ) @@ -110,17 +108,7 @@ func (st *State) TranslateDir(pf string) error { func (st *State) CompileFile(fn string) error { dir, _ := filepath.Abs(st.Config.Output) - fsys := os.DirFS(dir) - b, err := fs.ReadFile(fsys, fn) - if errors.Log(err) != nil { - return err - } - is := gpu.IncludeFS(fsys, "", string(b)) - ofn := filepath.Join(dir, fn) - err = os.WriteFile(ofn, []byte(is), 0666) - if errors.Log(err) != nil { - return err - } + // cmd := exec.Command("naga", "--compact", fn, fn) // produces some pretty weird code actually cmd := exec.Command("naga", fn) cmd.Dir = dir out, err := cmd.CombinedOutput() From 92fb26d23fdd786d21e240e812ee952f4821e6cc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 3 Oct 2024 21:04:39 -0700 Subject: [PATCH 190/311] add genkernel file! --- goal/gosl/gotosl/genkernel.go | 68 +++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 goal/gosl/gotosl/genkernel.go diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go new file mode 100644 index 0000000000..06d56b23df --- /dev/null +++ b/goal/gosl/gotosl/genkernel.go @@ -0,0 +1,68 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gotosl + +import ( + "bytes" + "fmt" + "path/filepath" + "strings" +) + +// GenKernel generates and writes the WGSL kernel code for given kernel +func (st *State) GenKernel(sy *System, kn *Kernel) { + hdr := st.GenKernelHeader(sy, kn) + + lines := bytes.Split([]byte(hdr), []byte("\n")) + for _, im := range st.SLImportFiles { + lines = append(lines, []byte("")) + lines = append(lines, []byte(fmt.Sprintf("///////////// import: %q", im.Name))) + lines = append(lines, im.Lines...) + } + kn.Lines = lines + kfn := kn.Name + ".wgsl" + fn := filepath.Join(st.Config.Output, kfn) + st.WriteFileLines(fn, lines) + + st.CompileFile(kfn) +} + +// GenKernelHeader returns the novel generated WGSL kernel code +// for given kernel, which goes at the top of the resulting file. +func (st *State) GenKernelHeader(sy *System, kn *Kernel) string { + var b strings.Builder + b.WriteString("// Code generated by \"gosl\"; DO NOT EDIT\n") + b.WriteString("// kernel: " + kn.Name + "\n\n") + + for gi, gp := range sy.Groups { + if gp.Doc != "" { + b.WriteString("// " + gp.Doc + "\n") + } + str := "storage, read_write" + if gp.Uniform { + str = "uniform" + } + for vi, vr := range gp.Vars { + if vr.Doc != "" { + b.WriteString("// " + vr.Doc + "\n") + } + b.WriteString(fmt.Sprintf("@group(%d) @binding(%d)\n", gi, vi)) + b.WriteString(fmt.Sprintf("var<%s> %s: ", str, vr.Name)) + if vr.Type[:2] == "[]" { + b.WriteString(fmt.Sprintf("array<%s>;\n", vr.Type[2:])) + } else { + // todo: tensor type + } + } + } + + b.WriteString("\n") + b.WriteString("@compute @workgroup_size(64, 1, 1)\n") + // todo: conditional on different index dimensionality + b.WriteString("fn main(@builtin(global_invocation_id) idx: vec3) {\n") + b.WriteString(fmt.Sprintf("\t%s(idx.x);\n", kn.Name)) + b.WriteString("}\n") + return b.String() +} From 3e33b229a5d18f4653266e35a00c972325971ae0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 4 Oct 2024 03:34:57 -0700 Subject: [PATCH 191/311] gpu code generation mostly working; read sync not working due to missing read buffers. --- goal/gosl/examples/basic/main.go | 62 ++------ goal/gosl/gotosl/gengpu.go | 259 +++++++++++++++++++++++++++++++ goal/gosl/gotosl/genkernel.go | 1 + goal/gosl/gotosl/gotosl.go | 8 + gpu/compute.go | 17 +- gpu/cpipeline.go | 1 + gpu/examples/compute/compute.go | 2 +- 7 files changed, 292 insertions(+), 58 deletions(-) create mode 100644 goal/gosl/gotosl/gengpu.go diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index 4e0d40fb72..3a6b4e3d01 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -7,49 +7,22 @@ package main import ( - "embed" "fmt" "math/rand" "runtime" - "unsafe" "cogentcore.org/core/base/timer" - "cogentcore.org/core/gpu" ) //go:generate gosl . -//go:embed shaders/Compute.wgsl -var shaders embed.FS - func init() { // must lock main thread for gpu! runtime.LockOSThread() } func main() { - gpu.Debug = true - gp := gpu.NewComputeGPU() - fmt.Printf("Running on GPU: %s\n", gp.DeviceName) - - // gp.PropertiesString(true) // print - - sy := gpu.NewComputeSystem(gp, "compute") - pl := gpu.NewComputePipelineShaderFS(shaders, "shaders/basic.wgsl", sy) - vars := sy.Vars() - sgp := vars.AddGroup(gpu.Storage) - n := 2000000 // note: not necc to spec up-front, but easier if so - threads := 64 - - pv := sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) - dv := sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), n, gpu.ComputeShader) - - sgp.SetNValues(1) - sy.Config() - - pvl := pv.Values.Values[0] - dvl := dv.Values.Values[0] Params = make([]ParamStruct, 1) Params[0].Defaults() @@ -61,16 +34,17 @@ func main() { sd := make([]DataStruct, n) for i := range sd { - sd[i].Raw = cd[i].Raw + sd[i].Raw = Data[i].Raw } + GPUInit() + cpuTmr := timer.Time{} cpuTmr.Start() - gosl.UseCPU() - for i := range parallel(Data) { - Compute(i) - } + UseGPU = false + RunCompute(n) + cpuTmr.Stop() cd := Data @@ -79,31 +53,16 @@ func main() { gpuFullTmr := timer.Time{} gpuFullTmr.Start() - gosl.ToGPU(Params, Data) - // gpu.SetValueFrom(pvl, pars) - // gpu.SetValueFrom(dvl, sd) - // sgp.CreateReadBuffers() + ToGPU(ParamsVar, DataVar) gpuTmr := timer.Time{} gpuTmr.Start() - gosl.UseGPU() - for i := range parallel(Data) { - Compute(i) - } - - // ce, _ := sy.BeginComputePass() - // pl.Dispatch1D(ce, n, threads) - // ce.End() - // dvl.GPUToRead(sy.CommandEncoder) - // sy.EndComputePass(ce) + UseGPU = true + RunCompute(n, DataVar) gpuTmr.Stop() - gosl.FromCPU(Data) - // dvl.ReadSync() - // gpu.ReadToBytes(dvl, sd) - gpuFullTmr.Stop() mx := min(n, 5) @@ -118,6 +77,5 @@ func main() { gpuFull := gpuFullTmr.Total fmt.Printf("N: %d\t CPU: %v\t GPU: %v\t Full: %v\t CPU/GPU: %6.4g\n", n, cpu, gpu, gpuFull, float64(cpu)/float64(gpu)) - sy.Release() - gp.Release() + GPURelease() } diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go new file mode 100644 index 0000000000..aee3768a5d --- /dev/null +++ b/goal/gosl/gotosl/gengpu.go @@ -0,0 +1,259 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gotosl + +import ( + "fmt" + "os" + "strings" +) + +// genSysName is the name to use for system in generating code. +// if only one system, the name is empty +func (st *State) genSysName(sy *System) string { + if len(st.Systems) == 1 { + return "" + } + return sy.Name +} + +// genSysVar is the name to use for system in generating code. +// if only one system, the name is empty +func (st *State) genSysVar(sy *System) string { + return fmt.Sprintf("GPU%sSystem", st.genSysName(sy)) +} + +// GenGPU generates and writes the Go GPU helper code +func (st *State) GenGPU() { + var b strings.Builder + + header := `// Code generated by "gosl"; DO NOT EDIT + +package main + +import ( + "embed" + "unsafe" + "cogentcore.org/core/gpu" +) + +//go:embed %s/*.wgsl +var shaders embed.FS + +// GPU is the compute gpu device +var ComputeGPU *gpu.GPU + +// UseGPU indicates whether to use GPU vs. CPU. +var UseGPU bool + +` + + b.WriteString(fmt.Sprintf(header, st.Config.Output)) + + for _, sy := range st.Systems { + b.WriteString(fmt.Sprintf("var %s *gpu.ComputeSystem\n", st.genSysVar(sy))) + } + + venum := ` +// GPUVars is an enum for GPU variables, for specifying what to sync. +type GPUVars int32 //enums:enum + +const ( +` + + b.WriteString(venum) + + vidx := 0 + for _, sy := range st.Systems { + for _, gp := range sy.Groups { + for _, vr := range gp.Vars { + b.WriteString(fmt.Sprintf("\t%sVar GPUVars = %d\n", vr.Name, vidx)) + vidx++ + } + } + } + b.WriteString(")\n") + + initf := ` +// GPUInit initializes the GPU compute system +// Configuring Systems, variables and kernels. +func GPUInit() { + gp := gpu.NewComputeGPU() + ComputeGPU = gp +` + + b.WriteString(initf) + + for _, sy := range st.Systems { + b.WriteString(st.GenGPUSystemInit(sy)) + } + b.WriteString("}\n\n") + + release := ` +// GPURelease releases the GPU compute system. +func GPURelease() { +` + + b.WriteString(release) + + for _, sy := range st.Systems { + b.WriteString(fmt.Sprintf("\t%s.Release()\n", st.genSysVar(sy))) + } + b.WriteString("\tComputeGPU.Release()\n") + b.WriteString("}\n\n") + + for _, sy := range st.Systems { + b.WriteString(st.GenGPUSystemOps(sy)) + } + + gs := b.String() + fn := "gosl.go" + os.WriteFile(fn, []byte(gs), 0644) +} + +// GenGPUSystemInit generates GPU Init code for given system. +func (st *State) GenGPUSystemInit(sy *System) string { + var b strings.Builder + + syvar := st.genSysVar(sy) + + b.WriteString("\t{\n") + b.WriteString(fmt.Sprintf("\t\tsy := gpu.NewComputeSystem(gp, %q)\n", sy.Name)) + b.WriteString(fmt.Sprintf("\t\t%s = sy\n", syvar)) + for _, kn := range sy.Kernels { + b.WriteString(fmt.Sprintf("\t\tgpu.NewComputePipelineShaderFS(shaders, %q, sy)\n", kn.Filename)) + } + b.WriteString("\t\tvars := sy.Vars()\n") + for _, gp := range sy.Groups { + b.WriteString("\t\t{\n") + gtyp := "gpu.Storage" + if gp.Uniform { + gtyp = "gpu.Uniform" + } + b.WriteString(fmt.Sprintf("\t\t\tsgp := vars.AddGroup(%s)\n", gtyp)) + for _, vr := range gp.Vars { + // todo: tensor + b.WriteString(fmt.Sprintf("\t\t\tsgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), len(%s), gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) + } + b.WriteString("\t\t\tsgp.SetNValues(1)\n") + if !gp.Uniform { + b.WriteString("\t\t\tsgp.CreateReadBuffers()\n") + } + b.WriteString("\t\t}\n") + } + b.WriteString("\t\tsy.Config()\n") + b.WriteString("\t}\n") + return b.String() +} + +// GenGPUSystemOps generates GPU helper functions for given system. +func (st *State) GenGPUSystemOps(sy *System) string { + var b strings.Builder + + syvar := st.genSysVar(sy) + synm := st.genSysName(sy) + + // 1 = kernel, 2 = system, 3 = sysname + run := ` +// Run%[1]s runs the %[1]s kernel with given number of items, +// on either the CPU or GPU depending on the UseGPU. +// Pass *Var variable names to sync those variables back from the GPU +// after running (irrelevant for CPU). +func Run%[1]s(n int, syncVars ...GPUVars) { + if UseGPU { + Run%[1]sGPU(n, syncVars...) + } else { + Run%[1]sCPU(n) + } +} + +// Run%[1]sGPU runs the %[1]s kernel on the GPU. +func Run%[1]sGPU(n int, syncVars ...GPUVars) { + sy := %[2]s + pl := sy.ComputePipelines[%[1]q] + ce, _ := sy.BeginComputePass() + pl.Dispatch1D(ce, n, 64) + ce.End() + %[3]sReadFromGPU(syncVars...) + sy.EndComputePass() + %[3]sSyncFromGPU(syncVars...) +} + +// Run%[1]sCPU runs the %[1]s kernel on the CPU. +func Run%[1]sCPU(n int) { + // todo: need flops, need threaded api -- not tensor + for i := range n { + %[1]s(uint32(i)) + } +} + +` + + for _, kn := range sy.Kernels { + b.WriteString(fmt.Sprintf(run, kn.Name, syvar, synm)) + } + + toGPU := ` +// %[1]sToGPU copies given variables to the GPU for the system. +func %[1]sToGPU(vars ...GPUVars) { + sy := %[2]s + syVars := sy.Vars() + for _, v := range vars { + switch v { +` + b.WriteString(fmt.Sprintf(toGPU, synm, syvar)) + + for gi, gp := range sy.Groups { + for _, vr := range gp.Vars { + b.WriteString(fmt.Sprintf("\t\tcase %sVar:\n", vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tv, _ := syVars.ValueByIndex(%d, %q, 0)\n", gi, vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tgpu.SetValueFrom(v, %s)\n", vr.Name)) + } + } + b.WriteString("\t\t}\n\t}\n}\n") + + fmGPU := ` +// %[1]sReadFromGPU starts the process of copying vars to the GPU. +func %[1]sReadFromGPU(vars ...GPUVars) { + sy := %[2]s + syVars := sy.Vars() + for _, v := range vars { + switch v { +` + + b.WriteString(fmt.Sprintf(fmGPU, synm, syvar)) + + for gi, gp := range sy.Groups { + for _, vr := range gp.Vars { + b.WriteString(fmt.Sprintf("\t\tcase %sVar:\n", vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tv, _ := syVars.ValueByIndex(%d, %q, 0)\n", gi, vr.Name)) + b.WriteString("\t\t\tv.GPUToRead(sy.CommandEncoder)\n") + } + } + b.WriteString("\t\t}\n\t}\n}\n") + + syncGPU := ` +// %[1]sSyncFromGPU synchronizes vars from the GPU to the actual variable. +func %[1]sSyncFromGPU(vars ...GPUVars) { + sy := %[2]s + syVars := sy.Vars() + for _, v := range vars { + switch v { +` + + b.WriteString(fmt.Sprintf(syncGPU, synm, syvar)) + + for gi, gp := range sy.Groups { + for _, vr := range gp.Vars { + b.WriteString(fmt.Sprintf("\t\tcase %sVar:\n", vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tv, _ := syVars.ValueByIndex(%d, %q, 0)\n", gi, vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tv.ReadSync()\n")) + b.WriteString(fmt.Sprintf("\t\t\tgpu.ReadToBytes(v, %s)\n", vr.Name)) + } + } + b.WriteString("\t\t}\n\t}\n}\n") + + return b.String() +} diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index 06d56b23df..8abebcc9f2 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -24,6 +24,7 @@ func (st *State) GenKernel(sy *System, kn *Kernel) { kn.Lines = lines kfn := kn.Name + ".wgsl" fn := filepath.Join(st.Config.Output, kfn) + kn.Filename = fn st.WriteFileLines(fn, lines) st.CompileFile(kfn) diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 78158d28ee..1566adcf73 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -35,6 +35,9 @@ type Kernel struct { Args string + // Filename is the name of the kernel shader file, e.g., shaders/Compute.wgsl + Filename string + // function code FuncCode string @@ -98,6 +101,9 @@ type State struct { // which are copied into the generated shader kernel files. SLImportFiles []*File + // generated Go GPU gosl.go file contents + GPUFile File + // ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation. ExcludeMap map[string]bool } @@ -139,6 +145,8 @@ func (st *State) Run() error { } } + st.GenGPU() + return nil } diff --git a/gpu/compute.go b/gpu/compute.go index a9302909b5..dd65f39027 100644 --- a/gpu/compute.go +++ b/gpu/compute.go @@ -25,9 +25,13 @@ type ComputeSystem struct { // Access through the System.Vars() method. vars Vars - // ComputePipelines by name + // ComputePipelines by name. ComputePipelines map[string]*ComputePipeline + // ComputeEncoder is the compute specific command encoder for the + // current [BeginComputePass], and released in [EndComputePass]. + ComputeEncoder *wgpu.ComputePassEncoder + // CommandEncoder is the command encoder created in // [BeginComputePass], and released in [EndComputePass]. CommandEncoder *wgpu.CommandEncoder @@ -122,16 +126,19 @@ func (sy *ComputeSystem) BeginComputePass() (*wgpu.ComputePassEncoder, error) { return nil, err } sy.CommandEncoder = cmd - return cmd.BeginComputePass(nil), nil // note: optional name in the descriptor + sy.ComputeEncoder = cmd.BeginComputePass(nil) // optional name in the encoder + return sy.ComputeEncoder, nil } // EndComputePass submits the current compute commands to the device -// Queue and releases the [CommandEncoder] and the given -// ComputePassEncoder. You must call ce.End prior to calling this. +// Queue and releases the [ComputeSystem.CommandEncoder] and +// [ComputeSystem.ComputeEncoder]. You must call ce.End prior to calling this. // Can insert other commands after ce.End, e.g., to copy data back // from the GPU, prior to calling EndComputePass. -func (sy *ComputeSystem) EndComputePass(ce *wgpu.ComputePassEncoder) error { +func (sy *ComputeSystem) EndComputePass() error { + ce := sy.ComputeEncoder cmd := sy.CommandEncoder + sy.ComputeEncoder = nil sy.CommandEncoder = nil cmdBuffer, err := cmd.Finish(nil) if errors.Log(err) != nil { diff --git a/gpu/cpipeline.go b/gpu/cpipeline.go index 290919c9f6..68168b403c 100644 --- a/gpu/cpipeline.go +++ b/gpu/cpipeline.go @@ -43,6 +43,7 @@ func NewComputePipelineShaderFS(fsys fs.FS, fname string, sy *ComputeSystem) *Co sh := pl.AddShader(name) errors.Log(sh.OpenFileFS(fsys, fname)) pl.AddEntry(sh, ComputeShader, "main") + sy.ComputePipelines[pl.Name] = pl return pl } diff --git a/gpu/examples/compute/compute.go b/gpu/examples/compute/compute.go index 5737215b71..6a6cdbf465 100644 --- a/gpu/examples/compute/compute.go +++ b/gpu/examples/compute/compute.go @@ -65,7 +65,7 @@ func main() { pl.Dispatch1D(ce, n, threads) ce.End() dvl.GPUToRead(sy.CommandEncoder) - sy.EndComputePass(ce) + sy.EndComputePass() dvl.ReadSync() gpu.ReadToBytes(dvl, sd) From fdb620f30939a47a2cafd9e2da1b0a47823a0af9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 4 Oct 2024 12:32:14 -0700 Subject: [PATCH 192/311] gpu: major update to arrayn variable creation: actual array size is determined when value buffer is set, and also added ReadOnly flag, so it auto-creates read buffer for all !ReadOnly Storage variables. --- goal/gosl/examples/basic/compute.go | 1 + goal/gosl/examples/basic/gosl.go | 137 ++++++++++++++++++++++++++++ goal/gosl/examples/basic/main.go | 6 +- goal/gosl/examples/rand/main.go | 2 - goal/gosl/gotosl/gengpu.go | 19 ++-- goal/gosl/gotosl/gotosl.go | 5 + gpu/examples/compute/compute.go | 2 - gpu/value.go | 39 ++++++-- gpu/values.go | 12 --- gpu/var.go | 16 ++-- gpu/vargroup.go | 12 --- gpu/vars.go | 21 ----- 12 files changed, 197 insertions(+), 75 deletions(-) create mode 100644 goal/gosl/examples/basic/gosl.go diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 5f44bdfcee..512afd1490 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -12,6 +12,7 @@ import "cogentcore.org/core/math32" //gosl:vars var ( // Params are the parameters for the computation. + //gosl:read-only Params []ParamStruct // Data is the data on which the computation operates. diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go new file mode 100644 index 0000000000..a3a0277014 --- /dev/null +++ b/goal/gosl/examples/basic/gosl.go @@ -0,0 +1,137 @@ +// Code generated by "gosl"; DO NOT EDIT + +package main + +import ( + "embed" + "unsafe" + + "cogentcore.org/core/gpu" +) + +//go:embed shaders/*.wgsl +var shaders embed.FS + +// GPU is the compute gpu device +var ComputeGPU *gpu.GPU + +// UseGPU indicates whether to use GPU vs. CPU. +var UseGPU bool + +var GPUSystem *gpu.ComputeSystem + +// GPUVars is an enum for GPU variables, for specifying what to sync. +type GPUVars int32 //enums:enum + +const ( + ParamsVar GPUVars = 0 + DataVar GPUVars = 1 +) + +// GPUInit initializes the GPU compute system +// Configuring Systems, variables and kernels. +func GPUInit() { + gp := gpu.NewComputeGPU() + ComputeGPU = gp + { + sy := gpu.NewComputeSystem(gp, "Default") + GPUSystem = sy + gpu.NewComputePipelineShaderFS(shaders, "shaders/Compute.wgsl", sy) + vars := sy.Vars() + { + sgp := vars.AddGroup(gpu.Storage) + sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), len(Params), gpu.ComputeShader) + sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), len(Data), gpu.ComputeShader) + sgp.SetNValues(1) + } + sy.Config() + } +} + +// GPURelease releases the GPU compute system. +func GPURelease() { + GPUSystem.Release() + ComputeGPU.Release() +} + +// RunCompute runs the Compute kernel with given number of items, +// on either the CPU or GPU depending on the UseGPU. +// Pass *Var variable names to sync those variables back from the GPU +// after running (irrelevant for CPU). +func RunCompute(n int, syncVars ...GPUVars) { + if UseGPU { + RunComputeGPU(n, syncVars...) + } else { + RunComputeCPU(n) + } +} + +// RunComputeGPU runs the Compute kernel on the GPU. +func RunComputeGPU(n int, syncVars ...GPUVars) { + sy := GPUSystem + pl := sy.ComputePipelines["Compute"] + ce, _ := sy.BeginComputePass() + pl.Dispatch1D(ce, n, 64) + ce.End() + ReadFromGPU(syncVars...) + sy.EndComputePass() + SyncFromGPU(syncVars...) +} + +// RunComputeCPU runs the Compute kernel on the CPU. +func RunComputeCPU(n int) { + // todo: need flops, need threaded api -- not tensor + for i := range n { + Compute(uint32(i)) + } +} + +// ToGPU copies given variables to the GPU for the system. +func ToGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, v := range vars { + switch v { + case ParamsVar: + v, _ := syVars.ValueByIndex(0, "Params", 0) + gpu.SetValueFrom(v, Params) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + gpu.SetValueFrom(v, Data) + } + } +} + +// ReadFromGPU starts the process of copying vars to the GPU. +func ReadFromGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, v := range vars { + switch v { + case ParamsVar: + v, _ := syVars.ValueByIndex(0, "Params", 0) + v.GPUToRead(sy.CommandEncoder) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + v.GPUToRead(sy.CommandEncoder) + } + } +} + +// SyncFromGPU synchronizes vars from the GPU to the actual variable. +func SyncFromGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, v := range vars { + switch v { + case ParamsVar: + v, _ := syVars.ValueByIndex(0, "Params", 0) + v.ReadSync() + gpu.ReadToBytes(v, Params) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + v.ReadSync() + gpu.ReadToBytes(v, Data) + } + } +} diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index 3a6b4e3d01..632b52eda4 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -12,6 +12,7 @@ import ( "runtime" "cogentcore.org/core/base/timer" + "cogentcore.org/core/gpu" ) //go:generate gosl . @@ -22,6 +23,9 @@ func init() { } func main() { + gpu.Debug = true + GPUInit() + n := 2000000 // note: not necc to spec up-front, but easier if so Params = make([]ParamStruct, 1) @@ -37,8 +41,6 @@ func main() { sd[i].Raw = Data[i].Raw } - GPUInit() - cpuTmr := timer.Time{} cpuTmr.Start() diff --git a/goal/gosl/examples/rand/main.go b/goal/gosl/examples/rand/main.go index f053e96766..99a93dcaf9 100644 --- a/goal/gosl/examples/rand/main.go +++ b/goal/gosl/examples/rand/main.go @@ -70,8 +70,6 @@ func main() { gpu.SetValueFrom(cvl, []uint64{seed}) gpu.SetValueFrom(dvl, dataG) - sgp.CreateReadBuffers() - gpuTmr := timer.Time{} gpuTmr.Start() diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index aee3768a5d..e75998d8e2 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -42,7 +42,7 @@ import ( //go:embed %s/*.wgsl var shaders embed.FS -// GPU is the compute gpu device +// ComputeGPU is the compute gpu device var ComputeGPU *gpu.GPU // UseGPU indicates whether to use GPU vs. CPU. @@ -53,6 +53,7 @@ var UseGPU bool b.WriteString(fmt.Sprintf(header, st.Config.Output)) for _, sy := range st.Systems { + b.WriteString(fmt.Sprintf("// %s is a GPU compute System with kernels operating on the\nsame set of data variables.\n", st.genSysVar(sy))) b.WriteString(fmt.Sprintf("var %s *gpu.ComputeSystem\n", st.genSysVar(sy))) } @@ -77,8 +78,8 @@ const ( b.WriteString(")\n") initf := ` -// GPUInit initializes the GPU compute system -// Configuring Systems, variables and kernels. +// GPUInit initializes the GPU compute system, +// configuring system(s), variables and kernels. func GPUInit() { gp := gpu.NewComputeGPU() ComputeGPU = gp @@ -92,7 +93,8 @@ func GPUInit() { b.WriteString("}\n\n") release := ` -// GPURelease releases the GPU compute system. +// GPURelease releases the GPU compute system resources. +// Call this at program exit. func GPURelease() { ` @@ -138,9 +140,6 @@ func (st *State) GenGPUSystemInit(sy *System) string { b.WriteString(fmt.Sprintf("\t\t\tsgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), len(%s), gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) } b.WriteString("\t\t\tsgp.SetNValues(1)\n") - if !gp.Uniform { - b.WriteString("\t\t\tsgp.CreateReadBuffers()\n") - } b.WriteString("\t\t}\n") } b.WriteString("\t\tsy.Config()\n") @@ -157,9 +156,9 @@ func (st *State) GenGPUSystemOps(sy *System) string { // 1 = kernel, 2 = system, 3 = sysname run := ` -// Run%[1]s runs the %[1]s kernel with given number of items, -// on either the CPU or GPU depending on the UseGPU. -// Pass *Var variable names to sync those variables back from the GPU +// Run%[1]s runs the %[1]s kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// Pass *Var variable enums to sync those variables back from the GPU // after running (irrelevant for CPU). func Run%[1]s(n int, syncVars ...GPUVars) { if UseGPU { diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 1566adcf73..2cc552f5a0 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -54,6 +54,11 @@ type Var struct { // Type of variable: either []Type or tensor.Float32, tensor.Int32 Type string + + // ReadOnly indicates that this variable is never read back from GPU, + // specified by the gosl:read-only property in the variable comments. + // It is important to optimize GPU memory usage to indicate this. + ReadOnly bool } // Group represents one variable group. diff --git a/gpu/examples/compute/compute.go b/gpu/examples/compute/compute.go index 6a6cdbf465..cfb8351807 100644 --- a/gpu/examples/compute/compute.go +++ b/gpu/examples/compute/compute.go @@ -59,8 +59,6 @@ func main() { } gpu.SetValueFrom(dvl, sd) - sgp.CreateReadBuffers() - ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, threads) ce.End() diff --git a/gpu/value.go b/gpu/value.go index 922f8cc261..0f7fab3167 100644 --- a/gpu/value.go +++ b/gpu/value.go @@ -30,10 +30,17 @@ type Value struct { // index of this value within the Var list of values Index int - // VarSize is the size of each Var element, which includes any fixed ArrayN + // VarSize is the size of each Var element, which includes any fixed Var.ArrayN // array size specified on the Var. + // The actual buffer size is VarSize * Value.ArrayN (or DynamicN for dynamic). VarSize int + // ArrayN is the actual number of array elements, for Uniform or Storage + // variables without a fixed array size (i.e., the Var ArrayN = 1). + // This is set when the buffer is actually created, based on the data, + // or can be set directly prior to buffer creation. + ArrayN int + // DynamicIndex is the current index into a DynamicOffset variable // to use for the SetBindGroup call. Note that this is an index, // not an offset, so it indexes the DynamicN Vars in the Value, @@ -112,6 +119,7 @@ func (vl *Value) init(vr *Var, dev *Device, idx int) { vl.Index = idx vl.Name = fmt.Sprintf("%s_%d", vr.Name, vl.Index) vl.VarSize = vr.MemSize() + vl.ArrayN = 1 vl.alignBytes = vr.alignBytes vl.AlignVarSize = MemSizeAlign(vl.VarSize, vl.alignBytes) vl.isDynamic = vl.role == Vertex || vl.role == Index || vr.DynamicOffset @@ -129,11 +137,12 @@ func (vl *Value) MemSize() int { if vl.isDynamic { return vl.AlignVarSize * vl.dynamicN } - return vl.VarSize + return vl.ArrayN * vl.VarSize } // CreateBuffer creates the GPU buffer for this value if it does not // yet exist or is not the right size. +// For !ReadOnly [Storage] buffers, calls [Value.CreateReadBuffer]. func (vl *Value) CreateBuffer() error { if vl.role == SampledTexture { return nil @@ -159,6 +168,9 @@ func (vl *Value) CreateBuffer() error { } vl.AllocSize = sz vl.buffer = buf + if vl.role == Storage && !vl.vvar.ReadOnly { + vl.CreateReadBuffer() + } return nil } @@ -214,6 +226,9 @@ func (vl *Value) SetDynamicN(n int) { // SetValueFrom copies given values into value buffer memory, // making the buffer if it has not yet been constructed. +// The actual ArrayN size of Storage or Uniform variables will +// be computed based on the size of the from bytes, relative to +// the variable size. // IMPORTANT: do not use this for dynamic offset Uniform or // Storage variables, as the alignment will not be correct; // See [SetDynamicFromBytes]. @@ -223,6 +238,7 @@ func SetValueFrom[E any](vl *Value, from []E) error { // SetFromBytes copies given bytes into value buffer memory, // making the buffer if it has not yet been constructed. +// For !ReadOnly [Storage] buffers, calls [Value.CreateReadBuffer]. // IMPORTANT: do not use this for dynamic offset Uniform or // Storage variables, as the alignment will not be correct; // See [SetDynamicFromBytes]. @@ -232,12 +248,19 @@ func (vl *Value) SetFromBytes(from []byte) error { return errors.Log(err) } nb := len(from) + an := nb / vl.VarSize + aover := nb % vl.VarSize + if aover != 0 { + err := fmt.Errorf("gpu.Value SetFromBytes %s, Size passed: %d is not an even multiple of the variable size: %d", vl.Name, nb, vl.VarSize) + return errors.Log(err) + } if vl.isDynamic { // Vertex, Index at this point - dn := nb / vl.VarSize - vl.SetDynamicN(dn) + vl.SetDynamicN(an) + } else { + vl.ArrayN = an } tb := vl.MemSize() - if nb != tb { + if nb != tb { // this should never happen, but justin case err := fmt.Errorf("gpu.Value SetFromBytes %s, Size passed: %d != Size expected %d", vl.Name, nb, tb) return errors.Log(err) } @@ -254,6 +277,9 @@ func (vl *Value) SetFromBytes(from []byte) error { } vl.buffer = buf vl.AllocSize = nb + if vl.role == Storage && !vl.vvar.ReadOnly { + vl.CreateReadBuffer() + } } else { err := vl.device.Queue.WriteBuffer(vl.buffer, 0, from) if errors.Log(err) != nil { @@ -406,9 +432,8 @@ func (vl *Value) SetFromTexture(tx *Texture) *Texture { } // CreateReadBuffer creates a read buffer for this value, -// if it does not yet exist or is not the right size. +// for [Storage] values only. Automatically called for !ReadOnly. // Read buffer is needed for reading values back from the GPU. -// Only for Storage role variables. func (vl *Value) CreateReadBuffer() error { if !(vl.role == Storage || vl.role == StorageTexture) { return nil diff --git a/gpu/values.go b/gpu/values.go index 695db559ed..193c6da697 100644 --- a/gpu/values.go +++ b/gpu/values.go @@ -147,18 +147,6 @@ func (vs *Values) MemSize() int { return tsz } -// CreateReadBuffers creates read buffers for all values. -func (vs *Values) CreateReadBuffers() error { - var errs []error - for _, vl := range vs.Values { - err := vl.CreateReadBuffer() - if err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - // bindGroupEntry returns the BindGroupEntry for Current // value for this variable. func (vs *Values) bindGroupEntry(vr *Var) []wgpu.BindGroupEntry { diff --git a/gpu/var.go b/gpu/var.go index 0845aa28ab..7e59acca9f 100644 --- a/gpu/var.go +++ b/gpu/var.go @@ -35,13 +35,12 @@ type Var struct { // automatically be sent as 4 interleaved Float32Vector4 chuncks. Type Types - // number of elements, which is 1 for a single element, or a constant - // number for a fixed array of elements. For Vertex variables, the - // number is dynamic and does not need to be specified in advance, - // so you can leave it at 1. There can be alignment issues with arrays + // ArrayN is the number of elements in an array, only if there is a + // fixed array size. Otherwise, for single elements or dynamic arrays + // use a value of 1. There can be alignment issues with arrays // so make sure your elemental types are compatible. // Note that DynamicOffset variables can have Value buffers with multiple - // instances of the variable (with proper alignment stride), which is + // instances of the variable (with proper alignment stride), // which goes on top of any array value for the variable itself. ArrayN int @@ -87,6 +86,11 @@ type Var struct { // Only for Uniform and Storage variables. DynamicOffset bool + // ReadOnly applies only to [Storage] variables, and indicates that + // they are never read back from the GPU, so the additional staging + // buffers needed to do so are not created for these variables. + ReadOnly bool + // Values is the the array of Values allocated for this variable. // Each value has its own corresponding Buffer or Texture. // The currently-active Value is specified by the Current index, @@ -145,7 +149,6 @@ func (vr *Var) MemSize() int { if vr.ArrayN < 1 { vr.ArrayN = 1 } - // todo: may need to diagnose alignments here.. switch { case vr.Role >= SampledTexture: return 0 @@ -157,7 +160,6 @@ func (vr *Var) MemSize() int { // Release resets the MemPtr for values, resets any self-owned resources (Textures) func (vr *Var) Release() { vr.Values.Release() - // todo: free anything in var } // SetNValues sets specified number of Values for this var. diff --git a/gpu/vargroup.go b/gpu/vargroup.go index a676ad6610..54c022a5f2 100644 --- a/gpu/vargroup.go +++ b/gpu/vargroup.go @@ -166,18 +166,6 @@ func (vg *VarGroup) SetAllCurrentValue(i int) { } } -// CreateReadBuffers creates read buffers for all values. -func (vg *VarGroup) CreateReadBuffers() error { - var errs []error - for _, vr := range vg.Vars { - err := vr.Values.CreateReadBuffers() - if err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - // Config must be called after all variables have been added. // Configures binding / location for all vars based on sequential order. // also does validation and returns error message. diff --git a/gpu/vars.go b/gpu/vars.go index 9580a95ce3..2e1720b94d 100644 --- a/gpu/vars.go +++ b/gpu/vars.go @@ -169,27 +169,6 @@ func (vs *Vars) SetDynamicIndex(group int, name string, dynamicIndex int) *Var { return vr } -// CreateReadBuffers creates read buffers for all Storage variables. -// This is needed to be able to read values back from GPU (e.g., for Compute). -func (vs *Vars) CreateReadBuffers() error { - var errs []error - ns := vs.NGroups() - for gi := vs.StartGroup(); gi < ns; gi++ { - vg := vs.Groups[gi] - if vg == nil { - continue - } - if vg.Role != Storage { - continue - } - err := vg.CreateReadBuffers() - if err != nil { - errs = append(errs, err) - } - } - return errors.Join(errs...) -} - // Config must be called after all variables have been added. // Configures all Groups and also does validation, returning error // does DescLayout too, so all ready for Pipeline config. From 62c272f7692539b1b1181d4fc9d003319150aad0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 4 Oct 2024 13:35:21 -0700 Subject: [PATCH 193/311] fix incorrect imports. type prefixing and add & to recv arg for method call. basic now builds and runs directly from gosl build. --- goal/gosl/examples/basic/gosl.go | 23 ++++++++++++++--------- goal/gosl/gotosl/nodes.go | 3 +++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index a3a0277014..eeb18691e1 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -5,19 +5,20 @@ package main import ( "embed" "unsafe" - "cogentcore.org/core/gpu" ) //go:embed shaders/*.wgsl var shaders embed.FS -// GPU is the compute gpu device +// ComputeGPU is the compute gpu device var ComputeGPU *gpu.GPU // UseGPU indicates whether to use GPU vs. CPU. var UseGPU bool +// GPUSystem is a GPU compute System with kernels operating on the +same set of data variables. var GPUSystem *gpu.ComputeSystem // GPUVars is an enum for GPU variables, for specifying what to sync. @@ -25,11 +26,11 @@ type GPUVars int32 //enums:enum const ( ParamsVar GPUVars = 0 - DataVar GPUVars = 1 + DataVar GPUVars = 1 ) -// GPUInit initializes the GPU compute system -// Configuring Systems, variables and kernels. +// GPUInit initializes the GPU compute system, +// configuring system(s), variables and kernels. func GPUInit() { gp := gpu.NewComputeGPU() ComputeGPU = gp @@ -48,15 +49,18 @@ func GPUInit() { } } -// GPURelease releases the GPU compute system. + +// GPURelease releases the GPU compute system resources. +// Call this at program exit. func GPURelease() { GPUSystem.Release() ComputeGPU.Release() } -// RunCompute runs the Compute kernel with given number of items, -// on either the CPU or GPU depending on the UseGPU. -// Pass *Var variable names to sync those variables back from the GPU + +// RunCompute runs the Compute kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// Pass *Var variable enums to sync those variables back from the GPU // after running (irrelevant for CPU). func RunCompute(n int, syncVars ...GPUVars) { if UseGPU { @@ -86,6 +90,7 @@ func RunComputeCPU(n int) { } } + // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { sy := GPUSystem diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 639aa5ba62..cd7c1d2b83 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1546,6 +1546,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { recvType = id.Name // is a package path } else { pathType = typ + recvPath = "&" + recvPath } } else { pathIsPackage = true @@ -1556,11 +1557,13 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { errors.Log(err) return } + // fmt.Println(pathIsPackage, recvType, methName, recvPath) if pathIsPackage { p.print(recvType + "." + methName) p.setPos(x.Lparen) p.print(token.LPAREN) } else { + recvType = strings.TrimPrefix(recvType, "imports.") // no! p.print(recvType + "_" + methName) p.setPos(x.Lparen) p.print(token.LPAREN) From 1f916d5ec17ccbe423821237d7b24c631143aeaa Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 4 Oct 2024 14:26:02 -0700 Subject: [PATCH 194/311] break up Run and Done commands to allow sequencing. --- goal/gosl/examples/basic/gosl.go | 54 +++++++++++++++++++++++------- goal/gosl/examples/basic/main.go | 4 +-- goal/gosl/gotosl/gengpu.go | 56 ++++++++++++++++++++++++-------- gpu/compute.go | 7 ++++ 4 files changed, 94 insertions(+), 27 deletions(-) diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index eeb18691e1..259bae2908 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -18,7 +18,7 @@ var ComputeGPU *gpu.GPU var UseGPU bool // GPUSystem is a GPU compute System with kernels operating on the -same set of data variables. +// same set of data variables. var GPUSystem *gpu.ComputeSystem // GPUVars is an enum for GPU variables, for specifying what to sync. @@ -60,36 +60,66 @@ func GPURelease() { // RunCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. -// Pass *Var variable enums to sync those variables back from the GPU -// after running (irrelevant for CPU). -func RunCompute(n int, syncVars ...GPUVars) { +// Can call multiple Run* kernels in a row, which are then all launched +// in the same command submission on the GPU, which is by far the most efficient. +// MUST call RunDone (with optional vars to sync) after all Run calls. +// Alternatively, a single-shot RunOneCompute call does Run and Done for a +// single run-and-sync case. +func RunCompute(n int) { if UseGPU { - RunComputeGPU(n, syncVars...) + RunComputeGPU(n) } else { RunComputeCPU(n) } } -// RunComputeGPU runs the Compute kernel on the GPU. -func RunComputeGPU(n int, syncVars ...GPUVars) { +// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. +func RunComputeGPU(n int) { sy := GPUSystem pl := sy.ComputePipelines["Compute"] ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, 64) - ce.End() - ReadFromGPU(syncVars...) - sy.EndComputePass() - SyncFromGPU(syncVars...) } // RunComputeCPU runs the Compute kernel on the CPU. func RunComputeCPU(n int) { - // todo: need flops, need threaded api -- not tensor + // todo: need threaded api -- not tensor for i := range n { Compute(uint32(i)) } } +// RunOneCompute runs the Compute kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// This version then calls RunDone with the given variables to sync +// after the Run, for a single-shot Run-and-Done call. If multiple kernels +// can be run in sequence, it is much more efficient to do multiple Run* +// calls followed by a RunDone call. +func RunOneCompute(n int, syncVars ...GPUVars) { + if UseGPU { + RunComputeGPU(n) + RunDone(syncVars...) + } else { + RunComputeCPU(n) + } +} + +// RunDone must be called after Run* calls to start compute kernels. +// This actually submits the kernel jobs to the GPU, and adds commands +// to synchronize the given variables back from the GPU to the CPU. +// After this function completes, the GPU results will be available in +// the specified variables. +func RunDone(syncVars ...GPUVars) { + if !UseGPU { + return + } + sy := GPUSystem + sy.ComputeEncoder.End() + ReadFromGPU(syncVars...) + sy.EndComputePass() + SyncFromGPU(syncVars...) +} + // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index 632b52eda4..0ded3a0e06 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -45,7 +45,7 @@ func main() { cpuTmr.Start() UseGPU = false - RunCompute(n) + RunOneCompute(n) cpuTmr.Stop() @@ -61,7 +61,7 @@ func main() { gpuTmr.Start() UseGPU = true - RunCompute(n, DataVar) + RunOneCompute(n, DataVar) gpuTmr.Stop() diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index e75998d8e2..2adab7f71c 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -53,7 +53,7 @@ var UseGPU bool b.WriteString(fmt.Sprintf(header, st.Config.Output)) for _, sy := range st.Systems { - b.WriteString(fmt.Sprintf("// %s is a GPU compute System with kernels operating on the\nsame set of data variables.\n", st.genSysVar(sy))) + b.WriteString(fmt.Sprintf("// %s is a GPU compute System with kernels operating on the\n// same set of data variables.\n", st.genSysVar(sy))) b.WriteString(fmt.Sprintf("var %s *gpu.ComputeSystem\n", st.genSysVar(sy))) } @@ -154,40 +154,70 @@ func (st *State) GenGPUSystemOps(sy *System) string { syvar := st.genSysVar(sy) synm := st.genSysName(sy) - // 1 = kernel, 2 = system, 3 = sysname + // 1 = kernel, 2 = system var, 3 = sysname (blank for 1 default) run := ` // Run%[1]s runs the %[1]s kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. -// Pass *Var variable enums to sync those variables back from the GPU -// after running (irrelevant for CPU). -func Run%[1]s(n int, syncVars ...GPUVars) { +// Can call multiple Run* kernels in a row, which are then all launched +// in the same command submission on the GPU, which is by far the most efficient. +// MUST call RunDone (with optional vars to sync) after all Run calls. +// Alternatively, a single-shot RunOne%[1]s call does Run and Done for a +// single run-and-sync case. +func Run%[1]s(n int) { if UseGPU { - Run%[1]sGPU(n, syncVars...) + Run%[1]sGPU(n) } else { Run%[1]sCPU(n) } } -// Run%[1]sGPU runs the %[1]s kernel on the GPU. -func Run%[1]sGPU(n int, syncVars ...GPUVars) { +// Run%[1]sGPU runs the %[1]s kernel on the GPU. See [Run%[1]s] for more info. +func Run%[1]sGPU(n int) { sy := %[2]s pl := sy.ComputePipelines[%[1]q] ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, 64) - ce.End() - %[3]sReadFromGPU(syncVars...) - sy.EndComputePass() - %[3]sSyncFromGPU(syncVars...) } // Run%[1]sCPU runs the %[1]s kernel on the CPU. func Run%[1]sCPU(n int) { - // todo: need flops, need threaded api -- not tensor + // todo: need threaded api -- not tensor for i := range n { %[1]s(uint32(i)) } } +// RunOne%[1]s runs the %[1]s kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// This version then calls RunDone with the given variables to sync +// after the Run, for a single-shot Run-and-Done call. If multiple kernels +// can be run in sequence, it is much more efficient to do multiple Run* +// calls followed by a RunDone call. +func RunOne%[1]s(n int, syncVars ...GPUVars) { + if UseGPU { + Run%[1]sGPU(n) + RunDone%[3]s(syncVars...) + } else { + Run%[1]sCPU(n) + } +} + +// RunDone%[3]s must be called after Run* calls to start compute kernels. +// This actually submits the kernel jobs to the GPU, and adds commands +// to synchronize the given variables back from the GPU to the CPU. +// After this function completes, the GPU results will be available in +// the specified variables. +func RunDone%[3]s(syncVars ...GPUVars) { + if !UseGPU { + return + } + sy := %[2]s + sy.ComputeEncoder.End() + %[3]sReadFromGPU(syncVars...) + sy.EndComputePass() + %[3]sSyncFromGPU(syncVars...) +} + ` for _, kn := range sy.Kernels { diff --git a/gpu/compute.go b/gpu/compute.go index dd65f39027..e6f5c822d4 100644 --- a/gpu/compute.go +++ b/gpu/compute.go @@ -120,7 +120,14 @@ func (sy *ComputeSystem) NewCommandEncoder() (*wgpu.CommandEncoder, error) { // to start the compute pass, returning the encoder object // to which further compute commands should be added. // Call [EndComputePass] when done. +// If an existing [ComputeSystem.ComputeEncoder] is already set from +// a prior BeginComputePass call, then that is returned, so this +// is safe and efficient to call for every compute shader dispatch, +// where the first call will create and the rest add to the ongoing job. func (sy *ComputeSystem) BeginComputePass() (*wgpu.ComputePassEncoder, error) { + if sy.ComputeEncoder != nil { + return sy.ComputeEncoder, nil + } cmd, err := sy.NewCommandEncoder() if errors.Log(err) != nil { return nil, err From a7b0e5ac16800c9062f58c24c3e4a446b8ee401f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 4 Oct 2024 16:33:28 -0700 Subject: [PATCH 195/311] actually parse read-only flag and use in generated code --- goal/gosl/examples/basic/gosl.go | 20 ++--- goal/gosl/examples/basic/main.go | 2 +- goal/gosl/examples/basic/shaders/Compute.wgsl | 86 +++++++++++++++++++ goal/gosl/gotosl/gengpu.go | 24 +++--- goal/gosl/gotosl/nodes.go | 6 +- 5 files changed, 115 insertions(+), 23 deletions(-) create mode 100644 goal/gosl/examples/basic/shaders/Compute.wgsl diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index 259bae2908..eedc8f6899 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -41,15 +41,16 @@ func GPUInit() { vars := sy.Vars() { sgp := vars.AddGroup(gpu.Storage) - sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), len(Params), gpu.ComputeShader) - sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), len(Data), gpu.ComputeShader) + var vr *gpu.Var + vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), len(Params), gpu.ComputeShader) + vr.ReadOnly = true + vr = sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), len(Data), gpu.ComputeShader) sgp.SetNValues(1) } sy.Config() } } - // GPURelease releases the GPU compute system resources. // Call this at program exit. func GPURelease() { @@ -57,7 +58,6 @@ func GPURelease() { ComputeGPU.Release() } - // RunCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched @@ -125,8 +125,8 @@ func RunDone(syncVars ...GPUVars) { func ToGPU(vars ...GPUVars) { sy := GPUSystem syVars := sy.Vars() - for _, v := range vars { - switch v { + for _, vr := range vars { + switch vr { case ParamsVar: v, _ := syVars.ValueByIndex(0, "Params", 0) gpu.SetValueFrom(v, Params) @@ -141,8 +141,8 @@ func ToGPU(vars ...GPUVars) { func ReadFromGPU(vars ...GPUVars) { sy := GPUSystem syVars := sy.Vars() - for _, v := range vars { - switch v { + for _, vr := range vars { + switch vr { case ParamsVar: v, _ := syVars.ValueByIndex(0, "Params", 0) v.GPUToRead(sy.CommandEncoder) @@ -157,8 +157,8 @@ func ReadFromGPU(vars ...GPUVars) { func SyncFromGPU(vars ...GPUVars) { sy := GPUSystem syVars := sy.Vars() - for _, v := range vars { - switch v { + for _, vr := range vars { + switch vr { case ParamsVar: v, _ := syVars.ValueByIndex(0, "Params", 0) v.ReadSync() diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index 0ded3a0e06..ee364ce7d4 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -15,7 +15,7 @@ import ( "cogentcore.org/core/gpu" ) -//go:generate gosl . +//go:generate gosl func init() { // must lock main thread for gpu! diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl new file mode 100644 index 0000000000..1fc197f492 --- /dev/null +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -0,0 +1,86 @@ +// Code generated by "gosl"; DO NOT EDIT +// kernel: Compute + +// // Params are the parameters for the computation. +@group(0) @binding(0) +var Params: array; +@group(0) @binding(1) +var Data: array; + +@compute @workgroup_size(64, 1, 1) +fn main(@builtin(global_invocation_id) idx: vec3) { + Compute(idx.x); +} + + +///////////// import: "compute.wgsl" + +//gosl:import "cogentcore.org/core/math32" + +//gosl:vars + +// Params are the parameters for the computation. +//gosl:read-only + +// Data is the data on which the computation operates. + +// DataStruct has the test data +struct DataStruct { + + // raw value + Raw: f32, + + // integrated value + Integ: f32, + + // exp of integ + Exp: f32, + + // must pad to multiple of 4 floats for arrays + pad: f32, +} + +// ParamStruct has the test params +struct ParamStruct { + + // rate constant in msec + Tau: f32, + + // 1/Tau + Dt: f32, + + pad: f32, + pad1: f32, +} + +// IntegFromRaw computes integrated value from current raw value +fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr) { + (*ds).Integ += (*ps).Dt * ((*ds).Raw - (*ds).Integ); + (*ds).Exp = FastExp(-(*ds).Integ); +} + +// Compute does the main computation +fn Compute(i: u32) { //gosl:kernel + // Params[0].IntegFromRaw(&Data[i]) + var params = Params[0]; + var data = Data[i]; + ParamStruct_IntegFromRaw(¶ms, &data); + Data[i] = data; +} + + +///////////// import: "fastexp.wgsl" + +// FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph +// It does not have any of the sanity checking of a standard method -- returns +// nonsense when arg is out of range. Runs in 2.23ns vs. 6.3ns for 64bit which is faster +// than Exp actually. +fn FastExp(x: f32) -> f32 { + if (x <= -88.02969) { // this doesn't add anything and -exp is main use-case anyway + return f32(0.0); + } + var i = i32(12102203*x) + i32(127)*(i32(1)<<23); + var m = i >> 7 & 0xFFFF; // copy mantissa + i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11); + return bitcast(u32(i)); +} diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index 2adab7f71c..148e3a6f64 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -92,8 +92,7 @@ func GPUInit() { } b.WriteString("}\n\n") - release := ` -// GPURelease releases the GPU compute system resources. + release := `// GPURelease releases the GPU compute system resources. // Call this at program exit. func GPURelease() { ` @@ -135,9 +134,13 @@ func (st *State) GenGPUSystemInit(sy *System) string { gtyp = "gpu.Uniform" } b.WriteString(fmt.Sprintf("\t\t\tsgp := vars.AddGroup(%s)\n", gtyp)) + b.WriteString("\t\t\tvar vr *gpu.Var\n") for _, vr := range gp.Vars { // todo: tensor - b.WriteString(fmt.Sprintf("\t\t\tsgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), len(%s), gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), len(%s), gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) + if vr.ReadOnly { + b.WriteString("\t\t\tvr.ReadOnly = true\n") + } } b.WriteString("\t\t\tsgp.SetNValues(1)\n") b.WriteString("\t\t}\n") @@ -155,8 +158,7 @@ func (st *State) GenGPUSystemOps(sy *System) string { synm := st.genSysName(sy) // 1 = kernel, 2 = system var, 3 = sysname (blank for 1 default) - run := ` -// Run%[1]s runs the %[1]s kernel with given number of elements, + run := `// Run%[1]s runs the %[1]s kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched // in the same command submission on the GPU, which is by far the most efficient. @@ -229,8 +231,8 @@ func RunDone%[3]s(syncVars ...GPUVars) { func %[1]sToGPU(vars ...GPUVars) { sy := %[2]s syVars := sy.Vars() - for _, v := range vars { - switch v { + for _, vr := range vars { + switch vr { ` b.WriteString(fmt.Sprintf(toGPU, synm, syvar)) @@ -248,8 +250,8 @@ func %[1]sToGPU(vars ...GPUVars) { func %[1]sReadFromGPU(vars ...GPUVars) { sy := %[2]s syVars := sy.Vars() - for _, v := range vars { - switch v { + for _, vr := range vars { + switch vr { ` b.WriteString(fmt.Sprintf(fmGPU, synm, syvar)) @@ -268,8 +270,8 @@ func %[1]sReadFromGPU(vars ...GPUVars) { func %[1]sSyncFromGPU(vars ...GPUVars) { sy := %[2]s syVars := sy.Vars() - for _, v := range vars { - switch v { + for _, vr := range vars { + switch vr { ` b.WriteString(fmt.Sprintf(syncGPU, synm, syvar)) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index cd7c1d2b83..09da0e27a9 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -2259,6 +2259,10 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { for _, s := range d.Specs { vs := s.(*ast.ValueSpec) dir, docs := p.findDirective(vs.Doc) + readOnly := false + if strings.Contains(dir, "read-only") { + readOnly = true + } if strings.HasPrefix(dir, "group") { gpnm := strings.TrimSpace(dir[5:]) if gpnm == "" { @@ -2307,7 +2311,7 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { } typ = sid.Name + "." + sel.Sel.Name } - vr := &Var{Name: nm, Type: typ} + vr := &Var{Name: nm, Type: typ, ReadOnly: readOnly} gp.Vars = append(gp.Vars, vr) if p.GoToSL.Config.Debug { fmt.Println("\tAdded var:", nm, typ, "to group:", gp.Name) From cbd7d4d07361cf137fb053c76cff59d9b463c76b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 4 Oct 2024 17:38:03 -0700 Subject: [PATCH 196/311] start on auto-processing of global var exprs with tmp vars. one very bad aspect of WGSL.. --- goal/gosl/examples/basic/compute.go | 10 ++-- goal/gosl/examples/basic/shaders/Compute.wgsl | 10 ++-- goal/gosl/gotosl/gotosl.go | 14 +++++ goal/gosl/gotosl/nodes.go | 55 +++++++++++++++++++ goal/gosl/slbool/slboolcore/slboolcore.go | 2 +- goal/gosl/slrand/slrand.go | 2 +- 6 files changed, 81 insertions(+), 12 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 512afd1490..84c7a07bc4 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -56,11 +56,11 @@ func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) { // Compute does the main computation func Compute(i uint32) { //gosl:kernel - // Params[0].IntegFromRaw(&Data[i]) - var params = Params[0] - var data = Data[i] - params.IntegFromRaw(&data) - Data[i] = data + Params[0].IntegFromRaw(&Data[i]) + // var params = Params[0] + // var data = Data[i] + // params.IntegFromRaw(&data) + // Data[i] = data } //gosl:end diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 1fc197f492..b2a1fbc77e 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -61,11 +61,11 @@ fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr Date: Sat, 5 Oct 2024 01:26:34 -0700 Subject: [PATCH 197/311] calling functions with global vars now automatically creates tmp vars and does the assignment for writable vars. --- goal/gosl/examples/basic/compute.go | 4 - goal/gosl/examples/basic/shaders/Compute.wgsl | 8 +- goal/gosl/gotosl/nodes.go | 131 ++++++++++++++---- 3 files changed, 106 insertions(+), 37 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 84c7a07bc4..222f7655c9 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -57,10 +57,6 @@ func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) { // Compute does the main computation func Compute(i uint32) { //gosl:kernel Params[0].IntegFromRaw(&Data[i]) - // var params = Params[0] - // var data = Data[i] - // params.IntegFromRaw(&data) - // Data[i] = data } //gosl:end diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index b2a1fbc77e..c357aba2da 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -61,11 +61,9 @@ fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr 0 { + p.print(formfeed) + } + } // fmt.Println(pathIsPackage, recvType, methName, recvPath) if pathIsPackage { p.print(recvType + "." + methName) @@ -1627,16 +1711,6 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { p.print(token.COMMA, blank) } } - args := x.Args - if pathType != nil { - meth, _, _ := types.LookupFieldOrMethod(pathType, true, p.pkg.Types, methName) - if meth != nil { - if ft, ok := meth.(*types.Func); ok { - sig := ft.Type().(*types.Signature) - args = p.matchLiteralArgs(x.Args, sig.Params()) - } - } - } if x.Ellipsis.IsValid() { p.exprList(x.Lparen, args, depth, 0, x.Ellipsis, false) p.setPos(x.Ellipsis) @@ -1650,6 +1724,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { p.setPos(x.Rparen) p.print(token.RPAREN) + p.assignRwArgs(rwargs) } func (p *printer) expr0(x ast.Expr, depth int) { From 157954b1fe87b1771025e6d4d21a0c1735f0f55a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 5 Oct 2024 12:33:36 -0700 Subject: [PATCH 198/311] complete working mockup of how to do tensors in gosl. added Header offset in tensor.Shape to support encoding the strides in the first NumDims dimensions of a tensor. --- goal/gosl/examples/basic/compute.go | 39 +++++++------- goal/gosl/examples/basic/gosl.go | 14 ++--- goal/gosl/examples/basic/main.go | 23 ++++---- goal/gosl/examples/basic/shaders/Compute.wgsl | 54 +++++++++++-------- goal/gosl/gotosl/gengpu.go | 3 +- goal/gosl/gotosl/gotosl.go | 14 ++++- goal/gosl/sltensor/README.md | 7 +++ goal/gosl/sltensor/sltensor.go | 48 +++++++++++++++++ goal/gosl/sltensor/sltensor.wgsl | 18 +++++++ tensor/base.go | 16 +++--- tensor/number.go | 50 ++++++++++------- tensor/shape.go | 11 ++-- tensor/string.go | 50 ++++++++--------- tensor/tensor.go | 2 +- tensor/values.go | 2 + 15 files changed, 234 insertions(+), 117 deletions(-) create mode 100644 goal/gosl/sltensor/README.md create mode 100644 goal/gosl/sltensor/sltensor.go create mode 100644 goal/gosl/sltensor/sltensor.wgsl diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 222f7655c9..f0e110a0d3 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -4,7 +4,10 @@ package main -import "cogentcore.org/core/math32" +import ( + "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" +) //gosl:start //gosl:import "cogentcore.org/core/math32" @@ -16,24 +19,16 @@ var ( Params []ParamStruct // Data is the data on which the computation operates. - Data []DataStruct + // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + //gosl:2D + Data tensor.Float32 ) -// DataStruct has the test data -type DataStruct struct { - - // raw value - Raw float32 - - // integrated value - Integ float32 - - // exp of integ - Exp float32 - - // must pad to multiple of 4 floats for arrays - pad float32 -} +const ( + Raw int = iota + Integ + Exp +) // ParamStruct has the test params type ParamStruct struct { @@ -49,14 +44,16 @@ type ParamStruct struct { } // IntegFromRaw computes integrated value from current raw value -func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) { - ds.Integ += ps.Dt * (ds.Raw - ds.Integ) - ds.Exp = math32.FastExp(-ds.Integ) +func (ps *ParamStruct) IntegFromRaw(idx int) { + integ := Data.Value(idx, Integ) + integ += ps.Dt * (Data.Value(idx, Raw) - integ) + Data.Set(integ, idx, Integ) + Data.Set(math32.FastExp(-integ), idx, Exp) } // Compute does the main computation func Compute(i uint32) { //gosl:kernel - Params[0].IntegFromRaw(&Data[i]) + Params[0].IntegFromRaw(int(i)) } //gosl:end diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index eedc8f6899..52b572ef15 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -5,6 +5,7 @@ package main import ( "embed" "unsafe" + "cogentcore.org/core/gpu" ) @@ -26,7 +27,7 @@ type GPUVars int32 //enums:enum const ( ParamsVar GPUVars = 0 - DataVar GPUVars = 1 + DataVar GPUVars = 1 ) // GPUInit initializes the GPU compute system, @@ -42,9 +43,9 @@ func GPUInit() { { sgp := vars.AddGroup(gpu.Storage) var vr *gpu.Var - vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), len(Params), gpu.ComputeShader) + vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) vr.ReadOnly = true - vr = sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), len(Data), gpu.ComputeShader) + vr = sgp.Add("Data", gpu.Float32, 1, gpu.ComputeShader) sgp.SetNValues(1) } sy.Config() @@ -107,7 +108,7 @@ func RunOneCompute(n int, syncVars ...GPUVars) { // RunDone must be called after Run* calls to start compute kernels. // This actually submits the kernel jobs to the GPU, and adds commands // to synchronize the given variables back from the GPU to the CPU. -// After this function completes, the GPU results will be available in +// After this function completes, the GPU results will be available in // the specified variables. func RunDone(syncVars ...GPUVars) { if !UseGPU { @@ -120,7 +121,6 @@ func RunDone(syncVars ...GPUVars) { SyncFromGPU(syncVars...) } - // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { sy := GPUSystem @@ -132,7 +132,7 @@ func ToGPU(vars ...GPUVars) { gpu.SetValueFrom(v, Params) case DataVar: v, _ := syVars.ValueByIndex(0, "Data", 0) - gpu.SetValueFrom(v, Data) + gpu.SetValueFrom(v, Data.Values) } } } @@ -166,7 +166,7 @@ func SyncFromGPU(vars ...GPUVars) { case DataVar: v, _ := syVars.ValueByIndex(0, "Data", 0) v.ReadSync() - gpu.ReadToBytes(v, Data) + gpu.ReadToBytes(v, Data.Values) } } } diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index ee364ce7d4..c69829e81b 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -12,7 +12,9 @@ import ( "runtime" "cogentcore.org/core/base/timer" + "cogentcore.org/core/goal/gosl/sltensor" "cogentcore.org/core/gpu" + "cogentcore.org/core/tensor" ) //go:generate gosl @@ -26,19 +28,22 @@ func main() { gpu.Debug = true GPUInit() - n := 2000000 // note: not necc to spec up-front, but easier if so + n := 20_000 // note: not necc to spec up-front, but easier if so Params = make([]ParamStruct, 1) Params[0].Defaults() - Data = make([]DataStruct, n) - for i := range Data { - Data[i].Raw = rand.Float32() + sltensor.SetShapeSizes(&Data, n, 3) // critically, makes GPU compatible Header with strides + nt := Data.Len() + + for i := range nt { + Data.Set1D(rand.Float32(), i) } - sd := make([]DataStruct, n) - for i := range sd { - sd[i].Raw = Data[i].Raw + var sd tensor.Float32 + sltensor.SetShapeSizes(&sd, n, 3) + for i := range nt { + sd.Set1D(Data.Value1D(i), i) } cpuTmr := timer.Time{} @@ -69,8 +74,8 @@ func main() { mx := min(n, 5) for i := 0; i < mx; i++ { - d := cd[i].Exp - sd[i].Exp - fmt.Printf("%d\t Raw: %g\t Integ: %g\t Exp: %6.4g\tTrg: %6.4g\tDiff: %g\n", i, sd[i].Raw, sd[i].Integ, sd[i].Exp, cd[i].Exp, d) + d := cd.Value(i, Exp) - sd.Value(i, Exp) + fmt.Printf("%d\t Raw: %g\t Integ: %g\t Exp: %6.4g\tTrg: %6.4g\tDiff: %g\n", i, sd.Value(i, Raw), sd.Value(i, Integ), sd.Value(i, Exp), cd.Value(i, Exp), d) } fmt.Printf("\n") diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index c357aba2da..69a492b7b7 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -5,8 +5,7 @@ @group(0) @binding(0) var Params: array; @group(0) @binding(1) -var Data: array; - +var Data: array; @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { Compute(idx.x); @@ -23,22 +22,12 @@ fn main(@builtin(global_invocation_id) idx: vec3) { //gosl:read-only // Data is the data on which the computation operates. +// 2D: outer index is data, inner index is: Raw, Integ, Exp vars. +//gosl:2D -// DataStruct has the test data -struct DataStruct { - - // raw value - Raw: f32, - - // integrated value - Integ: f32, - - // exp of integ - Exp: f32, - - // must pad to multiple of 4 floats for arrays - pad: f32, -} +const Raw: u32 = 0; +const Integ: u32 = 1; +const Exp: u32 = 2; // ParamStruct has the test params struct ParamStruct { @@ -54,16 +43,16 @@ struct ParamStruct { } // IntegFromRaw computes integrated value from current raw value -fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr) { - (*ds).Integ += (*ps).Dt * ((*ds).Raw - (*ds).Integ); - (*ds).Exp = FastExp(-(*ds).Integ); +fn ParamStruct_IntegFromRaw(ps: ptr, idx: u32) { + var integ = Data[F32Index2D(Data[0], Data[1], idx, Integ)]; // .Value(idx, Integ); + integ += (*ps).Dt * (Data[F32Index2D(Data[0], Data[1], idx, Raw)] - integ); // .Value(idx, Raw) - integ); + Data[F32Index2D(Data[0], Data[1], idx, Integ)] = integ; // .Set(integ, idx, Integ); + Data[F32Index2D(Data[0], Data[1], idx, Exp)] = FastExp(-integ); // .Set(FastExp(-integ), idx, Exp); } // Compute does the main computation fn Compute(i: u32) { //gosl:kernel - var params=Params[0]; var data=Data[i]; - ParamStruct_IntegFromRaw(¶ms, &data); - Data[i]=data; + var params=Params[0]; ParamStruct_IntegFromRaw(¶ms, i); } @@ -82,3 +71,22 @@ fn FastExp(x: f32) -> f32 { i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11); return bitcast(u32(i)); } + +// sltensor indexing functions + +fn F32Index2D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { + return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; +} + +fn F32Index3D(s0: f32, s1: f32, s2: f32, i0: u32, i1: u32, i2: u32) -> u32 { + return u32(3) + bitcast(s0) * i0 + bitcast(s1) * i1 + bitcast(s2) * i2; +} + +fn U32Index2D(s0: u32, s1: u32, i0: u32, i1: u32) -> u32 { + return u32(2) + s0 * i0 + s1 * i1; +} + +fn U32Index3D(s0: u32, s1: u32, s2: u32, i0: u32, i1: u32, i2: u32) -> u32 { + return u32(3) + s0 * i0 + s1 * i1 + s2 * i2; +} + diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index 148e3a6f64..1479f5d2d1 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -136,8 +136,9 @@ func (st *State) GenGPUSystemInit(sy *System) string { b.WriteString(fmt.Sprintf("\t\t\tsgp := vars.AddGroup(%s)\n", gtyp)) b.WriteString("\t\t\tvar vr *gpu.Var\n") for _, vr := range gp.Vars { + if // todo: tensor - b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), len(%s), gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) + b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), 1, gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) if vr.ReadOnly { b.WriteString("\t\t\tvr.ReadOnly = true\n") } diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 2c88cefb4b..04744eb06a 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -52,13 +52,19 @@ type Var struct { // comment docs about this var. Doc string - // Type of variable: either []Type or tensor.Float32, tensor.Int32 + // Type of variable: either []Type or F32, U32 for tensors Type string // ReadOnly indicates that this variable is never read back from GPU, // specified by the gosl:read-only property in the variable comments. // It is important to optimize GPU memory usage to indicate this. ReadOnly bool + + // True if a tensor type + Tensor bool + + // Number of dimensions + TensorDims int } // Group represents one variable group. @@ -172,6 +178,12 @@ func (st *State) System(sysname string) *System { // GlobalVar returns global variable of given name, if found. func (st *State) GlobalVar(vrnm string) *Var { + if st == nil { + return nil + } + if st.Systems == nil { + return nil + } for _, sy := range st.Systems { for _, gp := range sy.Groups { for _, vr := range gp.Vars { diff --git a/goal/gosl/sltensor/README.md b/goal/gosl/sltensor/README.md new file mode 100644 index 0000000000..da6db63506 --- /dev/null +++ b/goal/gosl/sltensor/README.md @@ -0,0 +1,7 @@ +# sltensor: tensor access in WGSL + +sltensor provides tensor indexing functions used by gosl to translate tensor access functions into direct code, for `array` and `array` global variables, which encode the strides in their first NumDims values. The 1D case just uses direct indexing with no strides. + +Strides are always encoded for all dimensions to allow complete flexibility in memory organization. + + diff --git a/goal/gosl/sltensor/sltensor.go b/goal/gosl/sltensor/sltensor.go new file mode 100644 index 0000000000..05c7ad9278 --- /dev/null +++ b/goal/gosl/sltensor/sltensor.go @@ -0,0 +1,48 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sltensor + +import ( + "math" + "reflect" + + "cogentcore.org/core/base/num" + "cogentcore.org/core/base/reflectx" + "cogentcore.org/core/base/slicesx" + "cogentcore.org/core/tensor" +) + +// SetShapeSizes sets the shape of a [tensor.Number] tensor +// for use as a GPU data variable, with strides encoded as uint32 +// values in the first NumDims Header values. Must use this instead +// of the SetShapeSizes method on the tensor. +func SetShapeSizes[T num.Number](tsr *tensor.Number[T], sizes ...int) { + tsr.Shape().SetShapeSizes(sizes...) + tsr.Shape().Header = tsr.Shape().NumDims() + nln := tsr.Shape().Header + tsr.Shape().Len() + tsr.Values = slicesx.SetLength(tsr.Values, nln) + SetStrides(tsr) +} + +// SetStrides sets the strides of a [tensor.Number] tensor +// for use as a GPU data variable, with strides encoded as uint32 +// values in the first NumDims Header values. Must have already +// used [SetShapeSizes] to reserve header space. +func SetStrides[T num.Number](tsr *tensor.Number[T]) { + switch { + case reflectx.KindIsInt(tsr.DataType()): + for i, d := range tsr.Shape().Strides { + tsr.Values[i] = T(d) + } + case tsr.DataType() == reflect.Float32: + for i, d := range tsr.Shape().Strides { + tsr.Values[i] = T(math.Float32frombits(uint32(d))) + } + case tsr.DataType() == reflect.Float64: + for i, d := range tsr.Shape().Strides { + tsr.Values[i] = T(math.Float64frombits(uint64(d))) + } + } +} diff --git a/goal/gosl/sltensor/sltensor.wgsl b/goal/gosl/sltensor/sltensor.wgsl new file mode 100644 index 0000000000..5aab6a961f --- /dev/null +++ b/goal/gosl/sltensor/sltensor.wgsl @@ -0,0 +1,18 @@ +// sltensor indexing functions + +fn F32Index2D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { + return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; +} + +fn F32Index3D(s0: f32, s1: f32, s2: f32, i0: u32, i1: u32, i2: u32) -> u32 { + return u32(3) + bitcast(s0) * i0 + bitcast(s1) * i1 + bitcast(s2) * i2; +} + +fn U32Index2D(s0: u32, s1: u32, i0: u32, i1: u32) -> u32 { + return u32(2) + s0 * i0 + s1 * i1; +} + +fn U32Index3D(s0: u32, s1: u32, s2: u32, i0: u32, i1: u32, i2: u32) -> u32 { + return u32(3) + s0 * i0 + s1 * i1 + s2 * i2; +} + diff --git a/tensor/base.go b/tensor/base.go index 9e0d7bf590..4a6a2d95b2 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -35,7 +35,7 @@ func (tsr *Base[T]) ShapeSizes() []int { return slices.Clone(tsr.shape.Sizes) } // backing storage appropriately, retaining all existing data that fits. func (tsr *Base[T]) SetShapeSizes(sizes ...int) { tsr.shape.SetShapeSizes(sizes...) - nln := tsr.Len() + nln := tsr.shape.Header + tsr.shape.Len() tsr.Values = slicesx.SetLength(tsr.Values, nln) } @@ -68,13 +68,13 @@ func (tsr *Base[T]) Value(i ...int) T { return tsr.Values[tsr.shape.IndexTo1D(i...)] } -func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[i] } - func (tsr *Base[T]) Set(val T, i ...int) { tsr.Values[tsr.shape.IndexTo1D(i...)] = val } -func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[i] = val } +func (tsr *Base[T]) Value1D(i int) T { return tsr.Values[tsr.shape.Header+i] } + +func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[tsr.shape.Header+i] = val } // SetNumRows sets the number of rows (outermost dimension) in a RowMajor organized tensor. // It is safe to set this to 0. For incrementally growing tensors (e.g., a log) @@ -82,7 +82,7 @@ func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[i] = val } // full amount of memory, and then set to 0 and grow incrementally. func (tsr *Base[T]) SetNumRows(rows int) { _, cells := tsr.shape.RowCellSize() - nln := rows * cells + nln := tsr.shape.Header + rows*cells tsr.shape.Sizes[0] = rows tsr.Values = slicesx.SetLength(tsr.Values, nln) } @@ -121,11 +121,13 @@ func (tsr *Base[T]) StringValue(i ...int) string { return reflectx.ToString(tsr.Values[tsr.shape.IndexTo1D(i...)]) } -func (tsr *Base[T]) String1D(off int) string { return reflectx.ToString(tsr.Values[off]) } +func (tsr *Base[T]) String1D(i int) string { + return reflectx.ToString(tsr.Values[tsr.shape.Header+i]) +} func (tsr *Base[T]) StringRowCell(row, cell int) string { _, sz := tsr.shape.RowCellSize() - return reflectx.ToString(tsr.Values[row*sz+cell]) + return reflectx.ToString(tsr.Values[tsr.shape.Header+row*sz+cell]) } // Label satisfies the core.Labeler interface for a summary description of the tensor. diff --git a/tensor/number.go b/tensor/number.go index 8d9b9d66fa..965e3ccc3e 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -29,6 +29,9 @@ type Int = Number[int] // Int32 is an alias for Number[int32]. type Int32 = Number[int32] +// Uint32 is an alias for Number[uint32]. +type Uint32 = Number[uint32] + // Byte is an alias for Number[byte]. type Byte = Number[byte] @@ -56,6 +59,12 @@ func NewInt32(sizes ...int) *Int32 { return New[int32](sizes...).(*Int32) } +// NewUint32 returns a new Uint32 tensor +// with the given sizes per dimension (shape). +func NewUint32(sizes ...int) *Uint32 { + return New[uint32](sizes...).(*Uint32) +} + // NewByte returns a new Byte tensor // with the given sizes per dimension (shape). func NewByte(sizes ...int) *Byte { @@ -76,7 +85,7 @@ func NewNumber[T num.Number](sizes ...int) *Number[T] { func NewNumberShape[T num.Number](shape *Shape) *Number[T] { tsr := &Number[T]{} tsr.shape.CopyFrom(shape) - tsr.Values = make([]T, tsr.Len()) + tsr.Values = make([]T, shape.Header+tsr.Len()) return tsr } @@ -109,16 +118,16 @@ func (tsr *Number[T]) SetString(val string, i ...int) { } } -func (tsr Number[T]) SetString1D(val string, off int) { +func (tsr Number[T]) SetString1D(val string, i int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { - tsr.Values[off] = T(fv) + tsr.Values[tsr.shape.Header+i] = T(fv) } } func (tsr *Number[T]) SetStringRowCell(val string, row, cell int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { _, sz := tsr.shape.RowCellSize() - tsr.Values[row*sz+cell] = T(fv) + tsr.Values[tsr.shape.Header+row*sz+cell] = T(fv) } } @@ -149,22 +158,22 @@ func (tsr *Number[T]) SetFloat(val float64, i ...int) { } func (tsr *Number[T]) Float1D(i int) float64 { - return float64(tsr.Values[i]) + return float64(tsr.Values[tsr.shape.Header+i]) } func (tsr *Number[T]) SetFloat1D(val float64, i int) { - tsr.Values[i] = T(val) + tsr.Values[tsr.shape.Header+i] = T(val) } func (tsr *Number[T]) FloatRowCell(row, cell int) float64 { _, sz := tsr.shape.RowCellSize() i := row*sz + cell - return float64(tsr.Values[i]) + return float64(tsr.Values[tsr.shape.Header+i]) } func (tsr *Number[T]) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() - tsr.Values[row*sz+cell] = T(val) + tsr.Values[tsr.shape.Header+row*sz+cell] = T(val) } // FloatRow returns the value at given row (outermost dimension). @@ -195,22 +204,22 @@ func (tsr *Number[T]) SetInt(val int, i ...int) { } func (tsr *Number[T]) Int1D(i int) int { - return int(tsr.Values[i]) + return int(tsr.Values[tsr.shape.Header+i]) } func (tsr *Number[T]) SetInt1D(val int, i int) { - tsr.Values[i] = T(val) + tsr.Values[tsr.shape.Header+i] = T(val) } func (tsr *Number[T]) IntRowCell(row, cell int) int { _, sz := tsr.shape.RowCellSize() i := row*sz + cell - return int(tsr.Values[i]) + return int(tsr.Values[tsr.shape.Header+i]) } func (tsr *Number[T]) SetIntRowCell(val int, row, cell int) { _, sz := tsr.shape.RowCellSize() - tsr.Values[row*sz+cell] = T(val) + tsr.Values[tsr.shape.Header+row*sz+cell] = T(val) } // IntRow returns the value at given row (outermost dimension). @@ -231,7 +240,8 @@ func (tsr *Number[T]) SetIntRow(val int, row int) { // SetZeros is simple convenience function initialize all values to 0 func (tsr *Number[T]) SetZeros() { - for j := range tsr.Values { + n := len(tsr.Values) + for j := tsr.shape.Header; j < n; j++ { tsr.Values[j] = 0 } } @@ -249,17 +259,17 @@ func (tsr *Number[T]) Clone() Values { // otherwise it goes through appropriate standard type. func (tsr *Number[T]) CopyFrom(frm Values) { if fsm, ok := frm.(*Number[T]); ok { - copy(tsr.Values, fsm.Values) + copy(tsr.Values[tsr.shape.Header:], fsm.Values[fsm.shape.Header:]) return } - sz := min(len(tsr.Values), frm.Len()) + sz := min(tsr.Len(), frm.Len()) if reflectx.KindIsInt(tsr.DataType()) { for i := range sz { - tsr.Values[i] = T(frm.Int1D(i)) + tsr.Values[tsr.shape.Header+i] = T(frm.Int1D(i)) } } else { for i := range sz { - tsr.Values[i] = T(frm.Float1D(i)) + tsr.Values[tsr.shape.Header+i] = T(frm.Float1D(i)) } } } @@ -276,10 +286,10 @@ func (tsr *Number[T]) AppendFrom(frm Values) error { return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) } tsr.SetNumRows(rows + frows) - st := rows * cell + st := tsr.shape.Header + rows*cell fsz := frows * fcell if fsm, ok := frm.(*Number[T]); ok { - copy(tsr.Values[st:st+fsz], fsm.Values) + copy(tsr.Values[st:st+fsz], fsm.Values[fsm.shape.Header:]) return nil } for i := 0; i < fsz; i++ { @@ -294,7 +304,9 @@ func (tsr *Number[T]) AppendFrom(frm Values) error { // values to copy. Uses an optimized implementation if the other tensor is // of the same type, and otherwise it goes through appropriate standard type. func (tsr *Number[T]) CopyCellsFrom(frm Values, to, start, n int) { + to += tsr.shape.Header if fsm, ok := frm.(*Number[T]); ok { + start += fsm.shape.Header copy(tsr.Values[to:to+n], fsm.Values[start:start+n]) return } diff --git a/tensor/shape.go b/tensor/shape.go index c49abcca11..c2d311a7cb 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -21,6 +21,11 @@ type Shape struct { // offsets for each dimension. Strides []int `display:"-"` + + // Header is an extra indexing offset at the start of the slice + // for special data. This is used for GPU tensor access, to store + // the strides at the start. Not supported for bool type. + Header int } // NewShape returns a new shape with given sizes. @@ -54,6 +59,7 @@ func (sh *Shape) SizesAsTensor() *Int { func (sh *Shape) CopyFrom(cp *Shape) { sh.Sizes = slices.Clone(cp.Sizes) sh.Strides = slices.Clone(cp.Strides) + sh.Header = cp.Header } // Len returns the total length of elements in the tensor @@ -74,9 +80,6 @@ func (sh *Shape) NumDims() int { return len(sh.Sizes) } // DimSize returns the size of given dimension. func (sh *Shape) DimSize(i int) int { - // if sh.Sizes == nil { - // return 0 - // } return sh.Sizes[i] } @@ -127,7 +130,7 @@ func (sh *Shape) RowCellSize() (rows, cells int) { // No checking is done on the length or size of the index values relative // to the shape of the tensor. func (sh *Shape) IndexTo1D(index ...int) int { - var oned int + oned := sh.Header for i, v := range index { oned += v * sh.Strides[i] } diff --git a/tensor/string.go b/tensor/string.go index ed4cacf06d..8ee6e1fd85 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -30,7 +30,7 @@ func NewString(sizes ...int) *String { func NewStringShape(shape *Shape) *String { tsr := &String{} tsr.shape.CopyFrom(shape) - tsr.Values = make([]string, tsr.Len()) + tsr.Values = make([]string, shape.Header+tsr.Len()) return tsr } @@ -62,17 +62,16 @@ func (tsr *String) AsValues() Values { return tsr } ///////////////////// Strings func (tsr *String) SetString(val string, i ...int) { - j := tsr.shape.IndexTo1D(i...) - tsr.Values[j] = val + tsr.Values[tsr.shape.IndexTo1D(i...)] = val } -func (tsr String) SetString1D(val string, off int) { - tsr.Values[off] = val +func (tsr String) SetString1D(val string, i int) { + tsr.Values[tsr.shape.Header+i] = val } func (tsr *String) SetStringRowCell(val string, row, cell int) { _, sz := tsr.shape.RowCellSize() - tsr.Values[row*sz+cell] = val + tsr.Values[tsr.shape.Header+row*sz+cell] = val } func (tsr *String) StringRow(row int) string { @@ -93,22 +92,22 @@ func (tsr *String) SetFloat(val float64, i ...int) { tsr.Values[tsr.shape.IndexTo1D(i...)] = Float64ToString(val) } -func (tsr *String) Float1D(off int) float64 { - return StringToFloat64(tsr.Values[off]) +func (tsr *String) Float1D(i int) float64 { + return StringToFloat64(tsr.Values[tsr.shape.Header+i]) } -func (tsr *String) SetFloat1D(val float64, off int) { - tsr.Values[off] = Float64ToString(val) +func (tsr *String) SetFloat1D(val float64, i int) { + tsr.Values[tsr.shape.Header+i] = Float64ToString(val) } func (tsr *String) FloatRowCell(row, cell int) float64 { _, sz := tsr.shape.RowCellSize() - return StringToFloat64(tsr.Values[row*sz+cell]) + return StringToFloat64(tsr.Values[tsr.shape.Header+row*sz+cell]) } func (tsr *String) SetFloatRowCell(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() - tsr.Values[row*sz+cell] = Float64ToString(val) + tsr.Values[tsr.shape.Header+row*sz+cell] = Float64ToString(val) } func (tsr *String) FloatRow(row int) float64 { @@ -129,22 +128,22 @@ func (tsr *String) SetInt(val int, i ...int) { tsr.Values[tsr.shape.IndexTo1D(i...)] = strconv.Itoa(val) } -func (tsr *String) Int1D(off int) int { - return errors.Ignore1(strconv.Atoi(tsr.Values[off])) +func (tsr *String) Int1D(i int) int { + return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.Header+i])) } -func (tsr *String) SetInt1D(val int, off int) { - tsr.Values[off] = strconv.Itoa(val) +func (tsr *String) SetInt1D(val int, i int) { + tsr.Values[tsr.shape.Header+i] = strconv.Itoa(val) } func (tsr *String) IntRowCell(row, cell int) int { _, sz := tsr.shape.RowCellSize() - return errors.Ignore1(strconv.Atoi(tsr.Values[row*sz+cell])) + return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.Header+row*sz+cell])) } func (tsr *String) SetIntRowCell(val int, row, cell int) { _, sz := tsr.shape.RowCellSize() - tsr.Values[row*sz+cell] = strconv.Itoa(val) + tsr.Values[tsr.shape.Header+row*sz+cell] = strconv.Itoa(val) } func (tsr *String) IntRow(row int) int { @@ -158,7 +157,8 @@ func (tsr *String) SetIntRow(val int, row int) { // SetZeros is a simple convenience function initialize all values to the // zero value of the type (empty strings for string type). func (tsr *String) SetZeros() { - for j := range tsr.Values { + n := len(tsr.Values) + for j := tsr.shape.Header; j < n; j++ { tsr.Values[j] = "" } } @@ -177,12 +177,12 @@ func (tsr *String) Clone() Values { // otherwise it goes through appropriate standard type. func (tsr *String) CopyFrom(frm Values) { if fsm, ok := frm.(*String); ok { - copy(tsr.Values, fsm.Values) + copy(tsr.Values[tsr.shape.Header:], fsm.Values[fsm.shape.Header:]) return } - sz := min(len(tsr.Values), frm.Len()) + sz := min(tsr.Len(), frm.Len()) for i := 0; i < sz; i++ { - tsr.Values[i] = Float64ToString(frm.Float1D(i)) + tsr.Values[tsr.shape.Header+i] = Float64ToString(frm.Float1D(i)) } } @@ -198,10 +198,10 @@ func (tsr *String) AppendFrom(frm Values) error { return fmt.Errorf("tensor.AppendFrom: cell sizes do not match: %d != %d", cell, fcell) } tsr.SetNumRows(rows + frows) - st := rows * cell + st := tsr.shape.Header + rows*cell fsz := frows * fcell if fsm, ok := frm.(*String); ok { - copy(tsr.Values[st:st+fsz], fsm.Values) + copy(tsr.Values[st:st+fsz], fsm.Values[fsm.shape.Header:]) return nil } for i := 0; i < fsz; i++ { @@ -216,7 +216,9 @@ func (tsr *String) AppendFrom(frm Values) error { // values to copy. Uses an optimized implementation if the other tensor is // of the same type, and otherwise it goes through appropriate standard type. func (tsr *String) CopyCellsFrom(frm Values, to, start, n int) { + to += tsr.shape.Header if fsm, ok := frm.(*String); ok { + start += fsm.shape.Header for i := 0; i < n; i++ { tsr.Values[to+i] = fsm.Values[start+i] } diff --git a/tensor/tensor.go b/tensor/tensor.go index 2aab1e56c5..425cdf5558 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -17,7 +17,7 @@ import ( // Any numerical type can also be used. bool is represented using an // efficient bit slice. type DataTypes interface { - string | bool | float32 | float64 | int | int32 | byte + string | bool | float32 | float64 | int | int32 | uint32 | byte } // MaxSprintLength is the default maximum length of a String() representation diff --git a/tensor/values.go b/tensor/values.go index 03701b731b..2d614e9b28 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -81,6 +81,8 @@ func New[T DataTypes](sizes ...int) Values { return NewNumber[int](sizes...) case int32: return NewNumber[int32](sizes...) + case uint32: + return NewNumber[uint32](sizes...) case byte: return NewNumber[byte](sizes...) default: From 18a80d203e08a418a886bafdb26334b06f9eab54 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 5 Oct 2024 17:12:37 -0700 Subject: [PATCH 199/311] generating correct gosl.go code and parsing various tensor properties. --- goal/gosl/examples/basic/compute.go | 2 +- goal/gosl/examples/basic/gosl.go | 6 +- goal/gosl/examples/basic/shaders/Compute.wgsl | 42 +++-------- goal/gosl/gotosl/gengpu.go | 21 ++++-- goal/gosl/gotosl/gotosl.go | 4 + goal/gosl/gotosl/nodes.go | 73 +++++++++++++++---- 6 files changed, 96 insertions(+), 52 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index f0e110a0d3..c7f71b5a5e 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -20,7 +20,7 @@ var ( // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. - //gosl:2D + //gosl:dims 2 Data tensor.Float32 ) diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index 52b572ef15..d1a5889eb2 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -5,7 +5,6 @@ package main import ( "embed" "unsafe" - "cogentcore.org/core/gpu" ) @@ -27,7 +26,7 @@ type GPUVars int32 //enums:enum const ( ParamsVar GPUVars = 0 - DataVar GPUVars = 1 + DataVar GPUVars = 1 ) // GPUInit initializes the GPU compute system, @@ -108,7 +107,7 @@ func RunOneCompute(n int, syncVars ...GPUVars) { // RunDone must be called after Run* calls to start compute kernels. // This actually submits the kernel jobs to the GPU, and adds commands // to synchronize the given variables back from the GPU to the CPU. -// After this function completes, the GPU results will be available in +// After this function completes, the GPU results will be available in // the specified variables. func RunDone(syncVars ...GPUVars) { if !UseGPU { @@ -121,6 +120,7 @@ func RunDone(syncVars ...GPUVars) { SyncFromGPU(syncVars...) } + // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { sy := GPUSystem diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 69a492b7b7..83f7be37ce 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -5,7 +5,7 @@ @group(0) @binding(0) var Params: array; @group(0) @binding(1) -var Data: array; +var Data: @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { Compute(idx.x); @@ -23,11 +23,12 @@ fn main(@builtin(global_invocation_id) idx: vec3) { // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. -//gosl:2D +//gosl:dims 2 -const Raw: u32 = 0; -const Integ: u32 = 1; -const Exp: u32 = 2; +const Raw: i32 = 0; +const Integ: i32 = 1; + +const Exp: i32 = 2; // ParamStruct has the test params struct ParamStruct { @@ -43,16 +44,16 @@ struct ParamStruct { } // IntegFromRaw computes integrated value from current raw value -fn ParamStruct_IntegFromRaw(ps: ptr, idx: u32) { - var integ = Data[F32Index2D(Data[0], Data[1], idx, Integ)]; // .Value(idx, Integ); - integ += (*ps).Dt * (Data[F32Index2D(Data[0], Data[1], idx, Raw)] - integ); // .Value(idx, Raw) - integ); - Data[F32Index2D(Data[0], Data[1], idx, Integ)] = integ; // .Set(integ, idx, Integ); - Data[F32Index2D(Data[0], Data[1], idx, Exp)] = FastExp(-integ); // .Set(FastExp(-integ), idx, Exp); +fn ParamStruct_IntegFromRaw(ps: ptr, idx: i32) { + var integ = Data.Value(idx, Integ); + integ += (*ps).Dt * (Data.Value(idx, Raw) - integ); + Data.Set(integ, idx, Integ); + Data.Set(FastExp(-integ), idx, Exp); } // Compute does the main computation fn Compute(i: u32) { //gosl:kernel - var params=Params[0]; ParamStruct_IntegFromRaw(¶ms, i); + var params=Params[0]; ParamStruct_IntegFromRaw(¶ms, i32(i)); } @@ -71,22 +72,3 @@ fn FastExp(x: f32) -> f32 { i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11); return bitcast(u32(i)); } - -// sltensor indexing functions - -fn F32Index2D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { - return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; -} - -fn F32Index3D(s0: f32, s1: f32, s2: f32, i0: u32, i1: u32, i2: u32) -> u32 { - return u32(3) + bitcast(s0) * i0 + bitcast(s1) * i1 + bitcast(s2) * i2; -} - -fn U32Index2D(s0: u32, s1: u32, i0: u32, i1: u32) -> u32 { - return u32(2) + s0 * i0 + s1 * i1; -} - -fn U32Index3D(s0: u32, s1: u32, s2: u32, i0: u32, i1: u32, i2: u32) -> u32 { - return u32(3) + s0 * i0 + s1 * i1 + s2 * i2; -} - diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index 1479f5d2d1..2709ff64f7 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -136,9 +136,12 @@ func (st *State) GenGPUSystemInit(sy *System) string { b.WriteString(fmt.Sprintf("\t\t\tsgp := vars.AddGroup(%s)\n", gtyp)) b.WriteString("\t\t\tvar vr *gpu.Var\n") for _, vr := range gp.Vars { - if - // todo: tensor - b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), 1, gpu.ComputeShader)\n", vr.Name, vr.Type[2:], vr.Name)) + if vr.Tensor { + typ := strings.TrimPrefix(vr.Type, "tensor.") + b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.Add(%q, gpu.%s, 1, gpu.ComputeShader)\n", vr.Name, typ)) + } else { + b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), 1, gpu.ComputeShader)\n", vr.Name, vr.Type[2:])) + } if vr.ReadOnly { b.WriteString("\t\t\tvr.ReadOnly = true\n") } @@ -241,7 +244,11 @@ func %[1]sToGPU(vars ...GPUVars) { for _, vr := range gp.Vars { b.WriteString(fmt.Sprintf("\t\tcase %sVar:\n", vr.Name)) b.WriteString(fmt.Sprintf("\t\t\tv, _ := syVars.ValueByIndex(%d, %q, 0)\n", gi, vr.Name)) - b.WriteString(fmt.Sprintf("\t\t\tgpu.SetValueFrom(v, %s)\n", vr.Name)) + vv := vr.Name + if vr.Tensor { + vv += ".Values" + } + b.WriteString(fmt.Sprintf("\t\t\tgpu.SetValueFrom(v, %s)\n", vv)) } } b.WriteString("\t\t}\n\t}\n}\n") @@ -282,7 +289,11 @@ func %[1]sSyncFromGPU(vars ...GPUVars) { b.WriteString(fmt.Sprintf("\t\tcase %sVar:\n", vr.Name)) b.WriteString(fmt.Sprintf("\t\t\tv, _ := syVars.ValueByIndex(%d, %q, 0)\n", gi, vr.Name)) b.WriteString(fmt.Sprintf("\t\t\tv.ReadSync()\n")) - b.WriteString(fmt.Sprintf("\t\t\tgpu.ReadToBytes(v, %s)\n", vr.Name)) + vv := vr.Name + if vr.Tensor { + vv += ".Values" + } + b.WriteString(fmt.Sprintf("\t\t\tgpu.ReadToBytes(v, %s)\n", vv)) } } b.WriteString("\t\t}\n\t}\n}\n") diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 04744eb06a..bdc0cd1b41 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -8,6 +8,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" ) @@ -65,6 +66,9 @@ type Var struct { // Number of dimensions TensorDims int + + // data kind of the tensor + TensorKind reflect.Kind } // Group represents one variable group. diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 00dd6cfb28..fd77fee5c7 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -22,6 +22,7 @@ import ( "go/types" "math" "path" + "reflect" "slices" "strconv" "strings" @@ -73,14 +74,14 @@ func (p *printer) linebreak(line, min int, ws whiteSpace, newSection bool) (nbre return } -// gosl: find any gosl directive in given comments, returns directive and remaining docs -func (p *printer) findDirective(g *ast.CommentGroup) (dir string, docs string) { +// gosl: find any gosl directive in given comments, returns directive(s) and remaining docs +func (p *printer) findDirective(g *ast.CommentGroup) (dirs []string, docs string) { if g == nil { return } for _, c := range g.List { if strings.HasPrefix(c.Text, "//gosl:") { - dir = c.Text[7:] + dirs = append(dirs, c.Text[7:]) } else { docs += c.Text + " " } @@ -88,6 +89,27 @@ func (p *printer) findDirective(g *ast.CommentGroup) (dir string, docs string) { return } +// gosl: hasDirective returns whether directive(s) contains string +func hasDirective(dirs []string, dir string) bool { + for _, d := range dirs { + if strings.Contains(d, dir) { + return true + } + } + return false +} + +// gosl: directiveAfter returns the directive after given leading text, +// and a bool indicating if the string was found. +func directiveAfter(dirs []string, dir string) (string, bool) { + for _, d := range dirs { + if strings.HasPrefix(d, dir) { + return strings.TrimSpace(strings.TrimPrefix(d, dir)), true + } + } + return "", false +} + // setComment sets g as the next comment if g != nil and if node comments // are enabled - this mode is used when printing source code fragments such // as exports only. It assumes that there is no pending comment in p.comments @@ -2382,19 +2404,18 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, tok token.Token) { } } -// gosl: process system vars +// gosl: process system global vars func (p *printer) systemVars(d *ast.GenDecl, sysname string) { sy := p.GoToSL.System(sysname) var gp *Group for _, s := range d.Specs { vs := s.(*ast.ValueSpec) - dir, docs := p.findDirective(vs.Doc) + dirs, docs := p.findDirective(vs.Doc) readOnly := false - if strings.Contains(dir, "read-only") { + if hasDirective(dirs, "read-only") { readOnly = true } - if strings.HasPrefix(dir, "group") { - gpnm := strings.TrimSpace(dir[5:]) + if gpnm, ok := directiveAfter(dirs, "group"); ok { if gpnm == "" { gp = &Group{Name: fmt.Sprintf("Group_%d", len(sy.Groups)), Doc: docs} sy.Groups = append(sy.Groups, gp) @@ -2431,7 +2452,7 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { } else { sel, ok := vs.Type.(*ast.SelectorExpr) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Int32", sysname)) + errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname)) continue } sid, ok := sel.X.(*ast.Ident) @@ -2442,6 +2463,33 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { typ = sid.Name + "." + sel.Sel.Name } vr := &Var{Name: nm, Type: typ, ReadOnly: readOnly} + if strings.HasPrefix(typ, "tensor.") { + vr.Tensor = true + kindStr := strings.TrimPrefix(typ, "tensor.") + kind := reflect.Float32 + switch kindStr { + case "Float32": + kind = reflect.Float32 + case "Uint32": + kind = reflect.Uint32 + case "Int32": + kind = reflect.Int32 + default: + errors.Log(fmt.Errorf("gosl: system %q: variable %q type is not supported: %q", sysname, nm, kindStr)) + continue + } + dstr, ok := directiveAfter(dirs, "dims") + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: variable %q tensor vars require //gosl:dims to specify number of dimensions", sysname, nm)) + continue + } + dims, err := strconv.Atoi(dstr) + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: variable %q tensor dims parse error: %s", sysname, nm, err.Error())) + } + vr.TensorKind = kind + vr.TensorDims = dims + } gp.Vars = append(gp.Vars, vr) if p.GoToSL.Config.Debug { fmt.Println("\tAdded var:", nm, typ, "to group:", gp.Name) @@ -2468,9 +2516,9 @@ func (p *printer) genDecl(d *ast.GenDecl) { if n > 1 && (d.Tok == token.CONST || d.Tok == token.VAR) { // two or more grouped const/var declarations: if d.Tok == token.VAR { - dir, _ := p.findDirective(d.Doc) - if strings.HasPrefix(dir, "vars") { - p.systemVars(d, strings.TrimSpace(dir[4:])) + dirs, _ := p.findDirective(d.Doc) + if sysname, ok := directiveAfter(dirs, "vars"); ok { + p.systemVars(d, sysname) return } } @@ -2488,7 +2536,6 @@ func (p *printer) genDecl(d *ast.GenDecl) { var line int for i, s := range d.Specs { vs := s.(*ast.ValueSpec) - // p.findDirective(vs.Doc) if i > 0 { p.linebreak(p.lineFor(s.Pos()), 1, ignore, p.linesFrom(line) > 0) } From a28cb78890905b958203959dc2a5ba53cc5f9bee Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 02:06:52 -0700 Subject: [PATCH 200/311] gosl tensor translation full working --- goal/gosl/examples/basic/main.go | 8 +-- goal/gosl/examples/basic/shaders/Compute.wgsl | 15 +++-- goal/gosl/gotosl/gengpu.go | 2 +- goal/gosl/gotosl/genkernel.go | 58 +++++++++++++++++-- goal/gosl/gotosl/gotosl.go | 43 +++++++++++++- goal/gosl/gotosl/nodes.go | 53 ++++++++++++----- 6 files changed, 147 insertions(+), 32 deletions(-) diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index c69829e81b..bab4d70454 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -28,7 +28,7 @@ func main() { gpu.Debug = true GPUInit() - n := 20_000 // note: not necc to spec up-front, but easier if so + n := 2_000_000 // note: not necc to spec up-front, but easier if so Params = make([]ParamStruct, 1) Params[0].Defaults() @@ -66,16 +66,16 @@ func main() { gpuTmr.Start() UseGPU = true - RunOneCompute(n, DataVar) - + RunCompute(n) gpuTmr.Stop() + RunDone(DataVar) gpuFullTmr.Stop() mx := min(n, 5) for i := 0; i < mx; i++ { d := cd.Value(i, Exp) - sd.Value(i, Exp) - fmt.Printf("%d\t Raw: %g\t Integ: %g\t Exp: %6.4g\tTrg: %6.4g\tDiff: %g\n", i, sd.Value(i, Raw), sd.Value(i, Integ), sd.Value(i, Exp), cd.Value(i, Exp), d) + fmt.Printf("%d\t Raw: %6.4g\t Integ: %6.4g\t Exp: %6.4g\tTrg: %6.4g\tDiff: %g\n", i, sd.Value(i, Raw), sd.Value(i, Integ), sd.Value(i, Exp), cd.Value(i, Exp), d) } fmt.Printf("\n") diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 83f7be37ce..dfbdfa8413 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -5,12 +5,17 @@ @group(0) @binding(0) var Params: array; @group(0) @binding(1) -var Data: +var Data: array; + @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { Compute(idx.x); } +fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { + return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; +} + ///////////// import: "compute.wgsl" @@ -45,10 +50,10 @@ struct ParamStruct { // IntegFromRaw computes integrated value from current raw value fn ParamStruct_IntegFromRaw(ps: ptr, idx: i32) { - var integ = Data.Value(idx, Integ); - integ += (*ps).Dt * (Data.Value(idx, Raw) - integ); - Data.Set(integ, idx, Integ); - Data.Set(FastExp(-integ), idx, Exp); + var integ = Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))]; + integ += (*ps).Dt * (Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Raw))] - integ); + Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))]=integ; + Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Exp))]=FastExp(-integ); } // Compute does the main computation diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index 2709ff64f7..fc8587ff0a 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -140,7 +140,7 @@ func (st *State) GenGPUSystemInit(sy *System) string { typ := strings.TrimPrefix(vr.Type, "tensor.") b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.Add(%q, gpu.%s, 1, gpu.ComputeShader)\n", vr.Name, typ)) } else { - b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), 1, gpu.ComputeShader)\n", vr.Name, vr.Type[2:])) + b.WriteString(fmt.Sprintf("\t\t\tvr = sgp.AddStruct(%q, int(unsafe.Sizeof(%s{})), 1, gpu.ComputeShader)\n", vr.Name, vr.SLType())) } if vr.ReadOnly { b.WriteString("\t\t\tvr.ReadOnly = true\n") diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index 8abebcc9f2..235746eb78 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "path/filepath" + "reflect" "strings" ) @@ -51,11 +52,7 @@ func (st *State) GenKernelHeader(sy *System, kn *Kernel) string { } b.WriteString(fmt.Sprintf("@group(%d) @binding(%d)\n", gi, vi)) b.WriteString(fmt.Sprintf("var<%s> %s: ", str, vr.Name)) - if vr.Type[:2] == "[]" { - b.WriteString(fmt.Sprintf("array<%s>;\n", vr.Type[2:])) - } else { - // todo: tensor type - } + b.WriteString(fmt.Sprintf("array<%s>;\n", vr.SLType())) } } @@ -65,5 +62,56 @@ func (st *State) GenKernelHeader(sy *System, kn *Kernel) string { b.WriteString("fn main(@builtin(global_invocation_id) idx: vec3) {\n") b.WriteString(fmt.Sprintf("\t%s(idx.x);\n", kn.Name)) b.WriteString("}\n") + b.WriteString(st.GenTensorFuncs(sy)) + return b.String() +} + +// GenTensorFuncs returns the generated WGSL code +// for indexing the tensors in given system. +func (st *State) GenTensorFuncs(sy *System) string { + var b strings.Builder + + done := make(map[string]bool) + + for _, gp := range sy.Groups { + for _, vr := range gp.Vars { + if !vr.Tensor { + continue + } + typ := vr.SLType() + fn := vr.IndexFunc() + if _, ok := done[fn]; ok { + continue + } + tconv := "" + switch vr.TensorKind { + case reflect.Float32: + tconv = "bitcast(" + case reflect.Int32: + tconv = "u32(" + } + tend := "" + if tconv != "" { + tend = ")" + } + b.WriteString("\nfn " + fn + "(") + nd := vr.TensorDims + for d := range nd { + b.WriteString(fmt.Sprintf("s%d: %s, ", d, typ)) + } + for d := range nd { + b.WriteString(fmt.Sprintf("i%d: u32", d)) + if d < nd-1 { + b.WriteString(", ") + } + } + b.WriteString(") -> u32 {\n\treturn ") + b.WriteString(fmt.Sprintf("u32(%d)", vr.TensorDims)) + for d := range nd { + b.WriteString(fmt.Sprintf(" + %ss%d%s * i%d", tconv, d, tend, d)) + } + b.WriteString(";\n}\n") + } + } return b.String() } diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index bdc0cd1b41..bba346480e 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -5,11 +5,13 @@ package gotosl import ( - "errors" + "fmt" "os" "path/filepath" "reflect" "strings" + + "cogentcore.org/core/base/errors" ) // System represents a ComputeSystem, and its kernels and variables. @@ -71,6 +73,45 @@ type Var struct { TensorKind reflect.Kind } +func (vr *Var) SetTensorKind() { + kindStr := strings.TrimPrefix(vr.Type, "tensor.") + kind := reflect.Float32 + switch kindStr { + case "Float32": + kind = reflect.Float32 + case "Uint32": + kind = reflect.Uint32 + case "Int32": + kind = reflect.Int32 + default: + errors.Log(fmt.Errorf("gosl: variable %q type is not supported: %q", vr.Name, kindStr)) + } + vr.TensorKind = kind +} + +// SLType returns the WGSL type string +func (vr *Var) SLType() string { + if vr.Tensor { + switch vr.TensorKind { + case reflect.Float32: + return "f32" + case reflect.Int32: + return "i32" + case reflect.Uint32: + return "u32" + } + } else { + return vr.Type[2:] + } + return "" +} + +// IndexFunc returns the index function name +func (vr *Var) IndexFunc() string { + typ := strings.ToUpper(vr.SLType()) + return fmt.Sprintf("Index%s%dD", typ, vr.TensorDims) +} + // Group represents one variable group. type Group struct { Name string diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index fd77fee5c7..bb710a573c 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -22,7 +22,6 @@ import ( "go/types" "math" "path" - "reflect" "slices" "strconv" "strings" @@ -1636,7 +1635,7 @@ func (p *printer) globalVar(idx *ast.IndexExpr) (isGlobal bool, tmpVar, typName if err == nil { vtyp = nmd } - typName = gvr.Type[2:] + typName = gvr.SLType() p.print("var ", tmpVar, token.ASSIGN) p.expr(idx) p.print(token.SEMICOLON, blank) @@ -1665,6 +1664,36 @@ func (p *printer) methodIndex(idx *ast.IndexExpr) (recvPath, recvType string, pa return } +func (p *printer) tensorMethod(x *ast.CallExpr, vr *Var, methName string) { + args := x.Args + + stArg := 0 + if methName == "Set" { + stArg = 1 + } + p.print(vr.Name, token.LBRACK) + p.print(vr.IndexFunc(), token.LPAREN) + nd := vr.TensorDims + for d := range nd { + p.print(vr.Name, token.LBRACK, strconv.Itoa(d), token.RBRACK, token.COMMA, blank) + } + n := len(args) + for i := stArg; i < n; i++ { + ag := args[i] + p.print("u32", token.LPAREN) + p.expr(ag) + p.print(token.RPAREN) + if i < n-1 { + p.print(token.COMMA) + } + } + p.print(token.RPAREN, token.RBRACK) + if methName == "Set" { + p.print(token.ASSIGN) + p.expr(args[0]) + } +} + func (p *printer) methodExpr(x *ast.CallExpr, depth int) { path := x.Fun.(*ast.SelectorExpr) // we know fun is selector methName := path.Sel.Name @@ -1679,6 +1708,11 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { return } } else if id, ok := path.X.(*ast.Ident); ok { + gvr := p.GoToSL.GlobalVar(id.Name) + if gvr != nil && gvr.Tensor { + p.tensorMethod(x, gvr, methName) + return + } recvPath = id.Name typ := p.getIdType(id) if typ != nil { @@ -2465,19 +2499,6 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { vr := &Var{Name: nm, Type: typ, ReadOnly: readOnly} if strings.HasPrefix(typ, "tensor.") { vr.Tensor = true - kindStr := strings.TrimPrefix(typ, "tensor.") - kind := reflect.Float32 - switch kindStr { - case "Float32": - kind = reflect.Float32 - case "Uint32": - kind = reflect.Uint32 - case "Int32": - kind = reflect.Int32 - default: - errors.Log(fmt.Errorf("gosl: system %q: variable %q type is not supported: %q", sysname, nm, kindStr)) - continue - } dstr, ok := directiveAfter(dirs, "dims") if !ok { errors.Log(fmt.Errorf("gosl: system %q: variable %q tensor vars require //gosl:dims to specify number of dimensions", sysname, nm)) @@ -2487,7 +2508,7 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { if !ok { errors.Log(fmt.Errorf("gosl: system %q: variable %q tensor dims parse error: %s", sysname, nm, err.Error())) } - vr.TensorKind = kind + vr.SetTensorKind() vr.TensorDims = dims } gp.Vars = append(gp.Vars, vr) From e7f078378768718fe8972f53e9ea739004832bfe Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 03:38:10 -0700 Subject: [PATCH 201/311] rand updated to new gosl code --- goal/gosl/examples/basic/shaders/Compute.wgsl | 4 +- goal/gosl/examples/rand/gosl.go | 172 ++++++++ goal/gosl/examples/rand/main.go | 60 +-- goal/gosl/examples/rand/rand.go | 28 +- goal/gosl/examples/rand/shaders/rand.wgsl | 371 ------------------ goal/gosl/examples/rand/shaders/slrand.wgsl | 201 ---------- goal/gosl/examples/rand/shaders/sltype.wgsl | 119 ------ goal/gosl/gotosl/extract.go | 4 +- goal/gosl/gotosl/files.go | 23 +- goal/gosl/gotosl/genkernel.go | 2 +- goal/gosl/gotosl/gotosl.go | 8 +- goal/gosl/gotosl/nodes.go | 17 +- goal/gosl/slrand/slrand.wgsl | 2 +- goal/gosl/sltensor/README.md | 8 +- goal/gosl/sltensor/sltensor.wgsl | 18 - 15 files changed, 248 insertions(+), 789 deletions(-) create mode 100644 goal/gosl/examples/rand/gosl.go delete mode 100644 goal/gosl/examples/rand/shaders/rand.wgsl delete mode 100644 goal/gosl/examples/rand/shaders/slrand.wgsl delete mode 100644 goal/gosl/examples/rand/shaders/sltype.wgsl delete mode 100644 goal/gosl/sltensor/sltensor.wgsl diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index dfbdfa8413..333b3a221a 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -52,8 +52,8 @@ struct ParamStruct { fn ParamStruct_IntegFromRaw(ps: ptr, idx: i32) { var integ = Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))]; integ += (*ps).Dt * (Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Raw))] - integ); - Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))]=integ; - Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Exp))]=FastExp(-integ); + Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))] = integ; + Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Exp))] = FastExp(-integ); } // Compute does the main computation diff --git a/goal/gosl/examples/rand/gosl.go b/goal/gosl/examples/rand/gosl.go new file mode 100644 index 0000000000..ae8340b836 --- /dev/null +++ b/goal/gosl/examples/rand/gosl.go @@ -0,0 +1,172 @@ +// Code generated by "gosl"; DO NOT EDIT + +package main + +import ( + "embed" + "unsafe" + "cogentcore.org/core/gpu" +) + +//go:embed shaders/*.wgsl +var shaders embed.FS + +// ComputeGPU is the compute gpu device +var ComputeGPU *gpu.GPU + +// UseGPU indicates whether to use GPU vs. CPU. +var UseGPU bool + +// GPUSystem is a GPU compute System with kernels operating on the +// same set of data variables. +var GPUSystem *gpu.ComputeSystem + +// GPUVars is an enum for GPU variables, for specifying what to sync. +type GPUVars int32 //enums:enum + +const ( + SeedVar GPUVars = 0 + DataVar GPUVars = 1 +) + +// GPUInit initializes the GPU compute system, +// configuring system(s), variables and kernels. +func GPUInit() { + gp := gpu.NewComputeGPU() + ComputeGPU = gp + { + sy := gpu.NewComputeSystem(gp, "Default") + GPUSystem = sy + gpu.NewComputePipelineShaderFS(shaders, "shaders/Compute.wgsl", sy) + vars := sy.Vars() + { + sgp := vars.AddGroup(gpu.Storage) + var vr *gpu.Var + vr = sgp.AddStruct("Seed", int(unsafe.Sizeof(Seeds{})), 1, gpu.ComputeShader) + vr.ReadOnly = true + vr = sgp.AddStruct("Data", int(unsafe.Sizeof(Rnds{})), 1, gpu.ComputeShader) + sgp.SetNValues(1) + } + sy.Config() + } +} + +// GPURelease releases the GPU compute system resources. +// Call this at program exit. +func GPURelease() { + GPUSystem.Release() + ComputeGPU.Release() +} + +// RunCompute runs the Compute kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// Can call multiple Run* kernels in a row, which are then all launched +// in the same command submission on the GPU, which is by far the most efficient. +// MUST call RunDone (with optional vars to sync) after all Run calls. +// Alternatively, a single-shot RunOneCompute call does Run and Done for a +// single run-and-sync case. +func RunCompute(n int) { + if UseGPU { + RunComputeGPU(n) + } else { + RunComputeCPU(n) + } +} + +// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. +func RunComputeGPU(n int) { + sy := GPUSystem + pl := sy.ComputePipelines["Compute"] + ce, _ := sy.BeginComputePass() + pl.Dispatch1D(ce, n, 64) +} + +// RunComputeCPU runs the Compute kernel on the CPU. +func RunComputeCPU(n int) { + // todo: need threaded api -- not tensor + for i := range n { + Compute(uint32(i)) + } +} + +// RunOneCompute runs the Compute kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// This version then calls RunDone with the given variables to sync +// after the Run, for a single-shot Run-and-Done call. If multiple kernels +// can be run in sequence, it is much more efficient to do multiple Run* +// calls followed by a RunDone call. +func RunOneCompute(n int, syncVars ...GPUVars) { + if UseGPU { + RunComputeGPU(n) + RunDone(syncVars...) + } else { + RunComputeCPU(n) + } +} + +// RunDone must be called after Run* calls to start compute kernels. +// This actually submits the kernel jobs to the GPU, and adds commands +// to synchronize the given variables back from the GPU to the CPU. +// After this function completes, the GPU results will be available in +// the specified variables. +func RunDone(syncVars ...GPUVars) { + if !UseGPU { + return + } + sy := GPUSystem + sy.ComputeEncoder.End() + ReadFromGPU(syncVars...) + sy.EndComputePass() + SyncFromGPU(syncVars...) +} + + +// ToGPU copies given variables to the GPU for the system. +func ToGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, vr := range vars { + switch vr { + case SeedVar: + v, _ := syVars.ValueByIndex(0, "Seed", 0) + gpu.SetValueFrom(v, Seed) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + gpu.SetValueFrom(v, Data) + } + } +} + +// ReadFromGPU starts the process of copying vars to the GPU. +func ReadFromGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, vr := range vars { + switch vr { + case SeedVar: + v, _ := syVars.ValueByIndex(0, "Seed", 0) + v.GPUToRead(sy.CommandEncoder) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + v.GPUToRead(sy.CommandEncoder) + } + } +} + +// SyncFromGPU synchronizes vars from the GPU to the actual variable. +func SyncFromGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, vr := range vars { + switch vr { + case SeedVar: + v, _ := syVars.ValueByIndex(0, "Seed", 0) + v.ReadSync() + gpu.ReadToBytes(v, Seed) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + v.ReadSync() + gpu.ReadToBytes(v, Data) + } + } +} diff --git a/goal/gosl/examples/rand/main.go b/goal/gosl/examples/rand/main.go index 99a93dcaf9..ff526e256e 100644 --- a/goal/gosl/examples/rand/main.go +++ b/goal/gosl/examples/rand/main.go @@ -5,23 +5,15 @@ package main import ( - "embed" "fmt" "runtime" - "unsafe" "log/slog" "cogentcore.org/core/base/timer" - "cogentcore.org/core/gpu" ) -// note: standard one to use is plain "gosl" which should be go install'd - -//go:generate ../../gosl rand.go rand.wgsl - -//go:embed shaders/*.wgsl -var shaders embed.FS +//go:generate gosl func init() { // must lock main thread for gpu! @@ -29,61 +21,40 @@ func init() { } func main() { - gpu.Debug = true - gp := gpu.NewComputeGPU() - fmt.Printf("Running on GPU: %s\n", gp.DeviceName) + GPUInit() // n := 10 n := 4_000_000 // 5_000_000 is too much -- 256_000_000 -- up against buf size limit - threads := 64 + + UseGPU = false + + Seed = make([]Seeds, 1) dataC := make([]Rnds, n) dataG := make([]Rnds, n) + Data = dataC + cpuTmr := timer.Time{} cpuTmr.Start() - - seed := uint64(0) - for i := range dataC { - d := &dataC[i] - d.RndGen(seed, uint32(i)) - } + RunOneCompute(n) cpuTmr.Stop() - sy := gpu.NewComputeSystem(gp, "slrand") - pl := gpu.NewComputePipelineShaderFS(shaders, "shaders/rand.wgsl", sy) - vars := sy.Vars() - sgp := vars.AddGroup(gpu.Storage) - - ctrv := sgp.AddStruct("Counter", int(unsafe.Sizeof(seed)), 1, gpu.ComputeShader) - datav := sgp.AddStruct("Data", int(unsafe.Sizeof(Rnds{})), n, gpu.ComputeShader) - - sgp.SetNValues(1) - sy.Config() - - cvl := ctrv.Values.Values[0] - dvl := datav.Values.Values[0] + UseGPU = true + Data = dataG gpuFullTmr := timer.Time{} gpuFullTmr.Start() - gpu.SetValueFrom(cvl, []uint64{seed}) - gpu.SetValueFrom(dvl, dataG) + ToGPU(SeedVar, DataVar) gpuTmr := timer.Time{} gpuTmr.Start() - ce, _ := sy.BeginComputePass() - pl.Dispatch1D(ce, n, threads) - ce.End() - dvl.GPUToRead(sy.CommandEncoder) - sy.EndComputePass(ce) - + RunCompute(n) gpuTmr.Stop() - dvl.ReadSync() - gpu.ReadToBytes(dvl, dataG) - + RunDone(DataVar) gpuFullTmr.Stop() anyDiffEx := false @@ -126,6 +97,5 @@ func main() { gpu := gpuTmr.Total fmt.Printf("N: %d\t CPU: %v\t GPU: %v\t Full: %v\t CPU/GPU: %6.4g\n", n, cpu, gpu, gpuFullTmr.Total, float64(cpu)/float64(gpu)) - sy.Release() - gp.Release() + GPURelease() } diff --git a/goal/gosl/examples/rand/rand.go b/goal/gosl/examples/rand/rand.go index 2f28690f8c..cc7bf7dac2 100644 --- a/goal/gosl/examples/rand/rand.go +++ b/goal/gosl/examples/rand/rand.go @@ -3,16 +3,26 @@ package main import ( "fmt" - "cogentcore.org/core/gpu/gosl/slrand" - "cogentcore.org/core/gpu/gosl/sltype" + "cogentcore.org/core/goal/gosl/slrand" + "cogentcore.org/core/goal/gosl/sltype" "cogentcore.org/core/math32" ) -//gosl:wgsl rand -// #include "slrand.wgsl" -//gosl:end rand +//gosl:start -//gosl:start rand +//gosl:vars +var ( + //gosl:read-only + Seed []Seeds + + // Data + Data []Rnds +) + +type Seeds struct { + Seed uint64 + pad, pad1 int32 +} type Rnds struct { Uints sltype.Uint32Vec2 @@ -39,7 +49,11 @@ func (r *Rnds) RndGen(counter uint64, idx uint32) { r.Gauss = slrand.Float32NormVec2(counter, uint32(3), idx) } -//gosl:end rand +func Compute(i uint32) { //gosl:kernel + Data[i].RndGen(Seed[0].Seed, i) +} + +//gosl:end const Tol = 1.0e-4 // fails at lower tol eventually -- -6 works for many diff --git a/goal/gosl/examples/rand/shaders/rand.wgsl b/goal/gosl/examples/rand/shaders/rand.wgsl deleted file mode 100644 index baf5d96be3..0000000000 --- a/goal/gosl/examples/rand/shaders/rand.wgsl +++ /dev/null @@ -1,371 +0,0 @@ - -// #include "slrand.wgsl" -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/slrand -// See README.md there for documentation. - -// These random number generation (RNG) functions are optimized for -// use on the GPU, with equivalent Go versions available in slrand.go. -// This is using the Philox2x32 counter-based RNG. - -// #include "sltype.wgsl" -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/sltype -// See README.md there for documentation. - -// This file emulates uint64 (u64) using 2 uint32 integers. -// and defines conversions between uint and float. - -// define a u64 type as an alias. -// if / when u64 actually happens, will make it easier to update. -alias su64 = vec2; - -// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using vec2). -fn Uint32Mul64(a: u32, b: u32) -> su64 { - let LOMASK = (((u32(1))<<16)-1); - var r: su64; - r.x = a * b; /* full low multiply */ - let ahi = a >> 16; - let alo = a & LOMASK; - let bhi = b >> 16; - let blo = b & LOMASK; - - let ahbl = ahi * blo; - let albh = alo * bhi; - - let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); - var hit = ahi*bhi + (ahbl>>16) + (albh>>16); - hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ - /* carry from the sum with alo*blo */ - if ((r.x >> u32(16)) < (ahbl_albh&LOMASK)) { - hit += u32(1); - } - r.y = hit; - return r; -} - -/* -// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using su64). -fn Uint32Mul64(a: u32, b: u32) -> su64 { - return su64(a) * su64(b); -} -*/ - - -// Uint64Add32 adds given uint32 number to given uint64 (using vec2). -fn Uint64Add32(a: su64, b: u32) -> su64 { - if (b == 0) { - return a; - } - var s = a; - if (s.x > u32(0xffffffff) - b) { - s.y++; - s.x = (b - 1) - (u32(0xffffffff) - s.x); - } else { - s.x += b; - } - return s; -} - -// Uint64Incr returns increment of the given uint64 (using vec2). -fn Uint64Incr(a: su64) -> su64 { - var s = a; - if(s.x == 0xffffffff) { - s.y++; - s.x = u32(0); - } else { - s.x++; - } - return s; -} - -// Uint32ToFloat32 converts a uint32 integer into a float32 -// in the (0,1) interval (i.e., exclusive of 1). -// This differs from the Go standard by excluding 0, which is handy for passing -// directly to Log function, and from the reference Philox code by excluding 1 -// which is in the Go standard and most other standard RNGs. -fn Uint32ToFloat32(val: u32) -> f32 { - let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); - let halffactor = f32(0.5) * factor; - var f = f32(val) * factor + halffactor; - if (f == 1.0) { // exclude 1 - return bitcast(0x3F7FFFFF); - } - return f; -} - -// note: there is no overloading of user-defined functions -// https://github.com/gpuweb/gpuweb/issues/876 - -// Uint32ToFloat32Vec2 converts two uint 32 bit integers -// into two corresponding 32 bit f32 values -// in the (0,1) interval (i.e., exclusive of 1). -fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { - var r: vec2; - r.x = Uint32ToFloat32(val.x); - r.y = Uint32ToFloat32(val.y); - return r; -} - -// Uint32ToFloat32Range11 converts a uint32 integer into a float32 -// in the [-1..1] interval (inclusive of -1 and 1, never identically == 0). -fn Uint32ToFloat32Range11(val: u32) -> f32 { - let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); - let halffactor = f32(0.5) * factor; - return (f32(val) * factor + halffactor); -} - -// Uint32ToFloat32Range11Vec2 converts two uint32 integers into two float32 -// in the [-1,1] interval (inclusive of -1 and 1, never identically == 0). -fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { - var r: vec2; - r.x = Uint32ToFloat32Range11(val.x); - r.y = Uint32ToFloat32Range11(val.y); - return r; -} - - - - -// Philox2x32round does one round of updating of the counter. -fn Philox2x32round(counter: su64, key: u32) -> su64 { - let mul = Uint32Mul64(u32(0xD256D193), counter.x); - var ctr: su64; - ctr.x = mul.y ^ key ^ counter.y; - ctr.y = mul.x; - return ctr; -} - -// Philox2x32bumpkey does one round of updating of the key -fn Philox2x32bumpkey(key: u32) -> u32 { - return key + u32(0x9E3779B9); -} - -// Philox2x32 implements the stateless counter-based RNG algorithm -// returning a random number as two uint32 values, given a -// counter and key input that determine the result. -// The input counter is not modified. -fn Philox2x32(counter: su64, key: u32) -> vec2 { - // this is an unrolled loop of 10 updates based on initial counter and key, - // which produces the random deviation deterministically based on these inputs. - var ctr = Philox2x32round(counter, key); // 1 - var ky = Philox2x32bumpkey(key); - ctr = Philox2x32round(ctr, ky); // 2 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 3 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 4 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 5 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 6 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 7 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 8 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 9 - ky = Philox2x32bumpkey(ky); - - return Philox2x32round(ctr, ky); // 10 -} - -//////////////////////////////////////////////////////////// -// Methods below provide a standard interface with more -// readable names, mapping onto the Go rand methods. -// -// They assume a global shared counter, which is then -// incremented by a function index, defined for each function -// consuming random numbers that _could_ be called within a parallel -// processing loop. At the end of the loop, the global counter should -// be incremented by the total possible number of such functions. -// This results in fully resproducible results, invariant to -// specific processing order, and invariant to whether any one function -// actually calls the random number generator. - -// RandUint32Vec2 returns two uniformly distributed 32 unsigned integers, -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - return Philox2x32(Uint64Add32(counter, funcIndex), key); -} - -// RandUint32 returns a uniformly distributed 32 unsigned integer, -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { - return Philox2x32(Uint64Add32(counter, funcIndex), key).x; -} - -// RandFloat32Vec2 returns two uniformly distributed float32 values in range (0,1), -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); -} - -// RandFloat32 returns a uniformly distributed float32 value in range (0,1), -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { - return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); -} - -// RandFloat32Range11Vec2 returns two uniformly distributed float32 values in range [-1,1], -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); -} - -// RandFloat32Range11 returns a uniformly distributed float32 value in range [-1,1], -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { - return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); -} - -// RandBoolP returns a bool true value with probability p -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { - return (RandFloat32(counter, funcIndex, key) < p); -} - -fn sincospi(x: f32) -> vec2 { - let PIf = 3.1415926535897932; - var r: vec2; - r.x = cos(PIf*x); - r.y = sin(PIf*x); - return r; -} - -// RandFloat32NormVec2 returns two random float32 numbers -// distributed according to the normal, Gaussian distribution -// with zero mean and unit variance. -// This is done very efficiently using the Box-Muller algorithm -// that consumes two random 32 bit uint values. -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - let ur = RandUint32Vec2(counter, funcIndex, key); - var f = sincospi(Uint32ToFloat32Range11(ur.x)); - let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. - return f * r; -} - -// RandFloat32Norm returns a random float32 number -// distributed according to the normal, Gaussian distribution -// with zero mean and unit variance. -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { - return RandFloat32Vec2(counter, funcIndex, key).x; -} - -// RandUint32N returns a uint32 in the range [0,N). -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { - let v = RandFloat32(counter, funcIndex, key); - return u32(v * f32(n)); -} - -// Counter is used for storing the random counter using aligned 16 byte -// storage, with convenience functions for typical use cases. -// It retains a copy of the last Seed value, which is applied to -// the Hi uint32 value. -struct RandCounter { - Counter: su64, - HiSeed: u32, - pad: u32, -} - -// Reset resets counter to last set Seed state. -fn RandCounter_Reset(ct: ptr) { - (*ct).Counter.x = u32(0); - (*ct).Counter.y = (*ct).HiSeed; -} - -// Seed sets the Hi uint32 value from given seed, saving it in Seed field. -// Each increment in seed generates a unique sequence of over 4 billion numbers, -// so it is reasonable to just use incremental values there, but more widely -// spaced numbers will result in longer unique sequences. -// Resets Lo to 0. -// This same seed will be restored during Reset -fn RandCounter_Seed(ct: ptr, seed: u32) { - (*ct).HiSeed = seed; - RandCounter_Reset(ct); -} - -// Add increments the counter by given amount. -// Call this after completing a pass of computation -// where the value passed here is the max of funcIndex+1 -// used for any possible random calls during that pass. -fn RandCounter_Add(ct: ptr, inc: u32) { - (*ct).Counter = Uint64Add32((*ct).Counter, inc); -} - - -struct Rnds { - Uints: vec2, - pad: i32, - pad1: i32, - Floats: vec2, - pad2: i32, - pad3: i32, - Floats11: vec2, - pad4: i32, - pad5: i32, - Gauss: vec2, - pad6: i32, - pad7: i32, -} - -// RndGen calls random function calls to test generator. -// Note that the counter to the outer-most computation function -// is passed by *value*, so the same counter goes to each element -// as it is computed, but within this scope, counter is passed by -// reference (as a pointer) so subsequent calls get a new counter value. -// The counter should be incremented by the number of random calls -// outside of the overall update function. -fn Rnds_RndGen(r: ptr, counter: su64, idx: u32) { - (*r).Uints = RandUint32Vec2(counter, u32(0), idx); - (*r).Floats = RandFloat32Vec2(counter, u32(1), idx); - (*r).Floats11 = RandFloat32Range11Vec2(counter, u32(2), idx); - (*r).Gauss = RandFloat32NormVec2(counter, u32(3), idx); -} - -// from file: rand.wgsl - -@group(0) @binding(0) -var Counter: array; - -@group(0) @binding(1) -var Data: array; - -@compute -@workgroup_size(64) -fn main(@builtin(global_invocation_id) idx: vec3) { - var ctr = Counter[0]; - var data = Data[idx.x]; - Rnds_RndGen(&data, ctr, idx.x); - Data[idx.x] = data; -} - diff --git a/goal/gosl/examples/rand/shaders/slrand.wgsl b/goal/gosl/examples/rand/shaders/slrand.wgsl deleted file mode 100644 index 820e7bdf62..0000000000 --- a/goal/gosl/examples/rand/shaders/slrand.wgsl +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/slrand -// See README.md there for documentation. - -// These random number generation (RNG) functions are optimized for -// use on the GPU, with equivalent Go versions available in slrand.go. -// This is using the Philox2x32 counter-based RNG. - -#include "sltype.wgsl" - -// Philox2x32round does one round of updating of the counter. -fn Philox2x32round(counter: su64, key: u32) -> su64 { - let mul = Uint32Mul64(u32(0xD256D193), counter.x); - var ctr: su64; - ctr.x = mul.y ^ key ^ counter.y; - ctr.y = mul.x; - return ctr; -} - -// Philox2x32bumpkey does one round of updating of the key -fn Philox2x32bumpkey(key: u32) -> u32 { - return key + u32(0x9E3779B9); -} - -// Philox2x32 implements the stateless counter-based RNG algorithm -// returning a random number as two uint32 values, given a -// counter and key input that determine the result. -// The input counter is not modified. -fn Philox2x32(counter: su64, key: u32) -> vec2 { - // this is an unrolled loop of 10 updates based on initial counter and key, - // which produces the random deviation deterministically based on these inputs. - var ctr = Philox2x32round(counter, key); // 1 - var ky = Philox2x32bumpkey(key); - ctr = Philox2x32round(ctr, ky); // 2 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 3 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 4 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 5 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 6 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 7 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 8 - ky = Philox2x32bumpkey(ky); - ctr = Philox2x32round(ctr, ky); // 9 - ky = Philox2x32bumpkey(ky); - - return Philox2x32round(ctr, ky); // 10 -} - -//////////////////////////////////////////////////////////// -// Methods below provide a standard interface with more -// readable names, mapping onto the Go rand methods. -// -// They assume a global shared counter, which is then -// incremented by a function index, defined for each function -// consuming random numbers that _could_ be called within a parallel -// processing loop. At the end of the loop, the global counter should -// be incremented by the total possible number of such functions. -// This results in fully resproducible results, invariant to -// specific processing order, and invariant to whether any one function -// actually calls the random number generator. - -// RandUint32Vec2 returns two uniformly distributed 32 unsigned integers, -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - return Philox2x32(Uint64Add32(counter, funcIndex), key); -} - -// RandUint32 returns a uniformly distributed 32 unsigned integer, -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { - return Philox2x32(Uint64Add32(counter, funcIndex), key).x; -} - -// RandFloat32Vec2 returns two uniformly distributed float32 values in range (0,1), -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); -} - -// RandFloat32 returns a uniformly distributed float32 value in range (0,1), -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { - return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); -} - -// RandFloat32Range11Vec2 returns two uniformly distributed float32 values in range [-1,1], -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); -} - -// RandFloat32Range11 returns a uniformly distributed float32 value in range [-1,1], -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { - return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); -} - -// RandBoolP returns a bool true value with probability p -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { - return (RandFloat32(counter, funcIndex, key) < p); -} - -fn sincospi(x: f32) -> vec2 { - let PIf = 3.1415926535897932; - var r: vec2; - r.x = cos(PIf*x); - r.y = sin(PIf*x); - return r; -} - -// RandFloat32NormVec2 returns two random float32 numbers -// distributed according to the normal, Gaussian distribution -// with zero mean and unit variance. -// This is done very efficiently using the Box-Muller algorithm -// that consumes two random 32 bit uint values. -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { - let ur = RandUint32Vec2(counter, funcIndex, key); - var f = sincospi(Uint32ToFloat32Range11(ur.x)); - let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. - return f * r; -} - -// RandFloat32Norm returns a random float32 number -// distributed according to the normal, Gaussian distribution -// with zero mean and unit variance. -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { - return RandFloat32Vec2(counter, funcIndex, key).x; -} - -// RandUint32N returns a uint32 in the range [0,N). -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. -fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { - let v = RandFloat32(counter, funcIndex, key); - return u32(v * f32(n)); -} - -// Counter is used for storing the random counter using aligned 16 byte -// storage, with convenience functions for typical use cases. -// It retains a copy of the last Seed value, which is applied to -// the Hi uint32 value. -struct RandCounter { - Counter: su64, - HiSeed: u32, - pad: u32, -} - -// Reset resets counter to last set Seed state. -fn RandCounter_Reset(ct: ptr) { - (*ct).Counter.x = u32(0); - (*ct).Counter.y = (*ct).HiSeed; -} - -// Seed sets the Hi uint32 value from given seed, saving it in Seed field. -// Each increment in seed generates a unique sequence of over 4 billion numbers, -// so it is reasonable to just use incremental values there, but more widely -// spaced numbers will result in longer unique sequences. -// Resets Lo to 0. -// This same seed will be restored during Reset -fn RandCounter_Seed(ct: ptr, seed: u32) { - (*ct).HiSeed = seed; - RandCounter_Reset(ct); -} - -// Add increments the counter by given amount. -// Call this after completing a pass of computation -// where the value passed here is the max of funcIndex+1 -// used for any possible random calls during that pass. -fn RandCounter_Add(ct: ptr, inc: u32) { - (*ct).Counter = Uint64Add32((*ct).Counter, inc); -} diff --git a/goal/gosl/examples/rand/shaders/sltype.wgsl b/goal/gosl/examples/rand/shaders/sltype.wgsl deleted file mode 100644 index e3ffe9e8e6..0000000000 --- a/goal/gosl/examples/rand/shaders/sltype.wgsl +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/sltype -// See README.md there for documentation. - -// This file emulates uint64 (u64) using 2 uint32 integers. -// and defines conversions between uint and float. - -// define a u64 type as an alias. -// if / when u64 actually happens, will make it easier to update. -alias su64 = vec2; - -// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using vec2). -fn Uint32Mul64(a: u32, b: u32) -> su64 { - let LOMASK = (((u32(1))<<16)-1); - var r: su64; - r.x = a * b; /* full low multiply */ - let ahi = a >> 16; - let alo = a & LOMASK; - let bhi = b >> 16; - let blo = b & LOMASK; - - let ahbl = ahi * blo; - let albh = alo * bhi; - - let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); - var hit = ahi*bhi + (ahbl>>16) + (albh>>16); - hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ - /* carry from the sum with alo*blo */ - if ((r.x >> u32(16)) < (ahbl_albh&LOMASK)) { - hit += u32(1); - } - r.y = hit; - return r; -} - -/* -// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using su64). -fn Uint32Mul64(a: u32, b: u32) -> su64 { - return su64(a) * su64(b); -} -*/ - - -// Uint64Add32 adds given uint32 number to given uint64 (using vec2). -fn Uint64Add32(a: su64, b: u32) -> su64 { - if (b == 0) { - return a; - } - var s = a; - if (s.x > u32(0xffffffff) - b) { - s.y++; - s.x = (b - 1) - (u32(0xffffffff) - s.x); - } else { - s.x += b; - } - return s; -} - -// Uint64Incr returns increment of the given uint64 (using vec2). -fn Uint64Incr(a: su64) -> su64 { - var s = a; - if(s.x == 0xffffffff) { - s.y++; - s.x = u32(0); - } else { - s.x++; - } - return s; -} - -// Uint32ToFloat32 converts a uint32 integer into a float32 -// in the (0,1) interval (i.e., exclusive of 1). -// This differs from the Go standard by excluding 0, which is handy for passing -// directly to Log function, and from the reference Philox code by excluding 1 -// which is in the Go standard and most other standard RNGs. -fn Uint32ToFloat32(val: u32) -> f32 { - let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); - let halffactor = f32(0.5) * factor; - var f = f32(val) * factor + halffactor; - if (f == 1.0) { // exclude 1 - return bitcast(0x3F7FFFFF); - } - return f; -} - -// note: there is no overloading of user-defined functions -// https://github.com/gpuweb/gpuweb/issues/876 - -// Uint32ToFloat32Vec2 converts two uint 32 bit integers -// into two corresponding 32 bit f32 values -// in the (0,1) interval (i.e., exclusive of 1). -fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { - var r: vec2; - r.x = Uint32ToFloat32(val.x); - r.y = Uint32ToFloat32(val.y); - return r; -} - -// Uint32ToFloat32Range11 converts a uint32 integer into a float32 -// in the [-1..1] interval (inclusive of -1 and 1, never identically == 0). -fn Uint32ToFloat32Range11(val: u32) -> f32 { - let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); - let halffactor = f32(0.5) * factor; - return (f32(val) * factor + halffactor); -} - -// Uint32ToFloat32Range11Vec2 converts two uint32 integers into two float32 -// in the [-1,1] interval (inclusive of -1 and 1, never identically == 0). -fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { - var r: vec2; - r.x = Uint32ToFloat32Range11(val.x); - r.y = Uint32ToFloat32Range11(val.y); - return r; -} - - diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index 459ecd15f6..318bdce629 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -18,7 +18,7 @@ import ( func (st *State) ExtractFiles() { for fn, fl := range st.GoFiles { fl.Lines = st.ExtractGosl(fl.Lines) - st.WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) + WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) } } @@ -36,7 +36,7 @@ func (st *State) ExtractImports() { for _, im := range st.GoImports { for fn, fl := range im { fl.Lines = st.ExtractGosl(fl.Lines) - st.WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) + WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) } } } diff --git a/goal/gosl/gotosl/files.go b/goal/gosl/gotosl/files.go index d8885b4c5a..b9a55caeaa 100644 --- a/goal/gosl/gotosl/files.go +++ b/goal/gosl/gotosl/files.go @@ -7,7 +7,6 @@ package gotosl import ( "bytes" "fmt" - "io" "io/fs" "log" "os" @@ -41,7 +40,7 @@ func ReadFileLines(fn string) ([][]byte, error) { return lines, nil } -func (st *State) WriteFileLines(fn string, lines [][]byte) error { +func WriteFileLines(fn string, lines [][]byte) error { res := bytes.Join(lines, []byte("\n")) return os.WriteFile(fn, res, 0644) } @@ -182,21 +181,21 @@ func (st *State) CopyPackageFile(fnm, packagePath string) error { } dir, _ := filepath.Split(fn) fmfn := filepath.Join(dir, fnm) - CopyFile(fmfn, tofn) + lines, err := CopyFile(fmfn, tofn) + if err == nil { + st.SLImportFiles = append(st.SLImportFiles, &File{Name: fnm, Lines: lines}) + } return nil } -func CopyFile(src, dst string) error { - in, err := os.Open(src) +func CopyFile(src, dst string) ([][]byte, error) { + lines, err := ReadFileLines(src) if err != nil { - return err + return lines, err } - defer in.Close() - out, err := os.Create(dst) + err = WriteFileLines(dst, lines) if err != nil { - return err + return lines, err } - defer out.Close() - _, err = io.Copy(out, in) - return err + return lines, err } diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index 235746eb78..79681a0206 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -26,7 +26,7 @@ func (st *State) GenKernel(sy *System, kn *Kernel) { kfn := kn.Name + ".wgsl" fn := filepath.Join(st.Config.Output, kfn) kn.Filename = fn - st.WriteFileLines(fn, lines) + WriteFileLines(fn, lines) st.CompileFile(kfn) } diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index bba346480e..b5ca0d83e3 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -190,7 +190,13 @@ func (st *State) Run() error { RemoveGenFiles(st.Config.Output) RemoveGenFiles(st.ImportsDir) - st.ProjectFiles() // get list of all files, recursively gets imports etc. + st.ProjectFiles() // get list of all files, recursively gets imports etc. + if len(st.GoFiles) == 0 { + if st.Config.Debug { + fmt.Println("gosl: no gosl files in current directory") + } + return nil + } st.ExtractFiles() // get .go from project files st.ExtractImports() // get .go from imports st.TranslateDir("./" + st.ImportsDir) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index bb710a573c..152f905c67 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1644,7 +1644,7 @@ func (p *printer) globalVar(idx *ast.IndexExpr) (isGlobal bool, tmpVar, typName } // gosl: methodIndex processes an index expression as receiver type of method call -func (p *printer) methodIndex(idx *ast.IndexExpr) (recvPath, recvType string, pathType types.Type, err error) { +func (p *printer) methodIndex(idx *ast.IndexExpr) (recvPath, recvType string, pathType types.Type, isReadOnly bool, err error) { id, ok := idx.X.(*ast.Ident) if !ok { err = fmt.Errorf("gosl methodIndex ERROR: must have a recv variable identifier, not %#v:", idx.X) @@ -1656,7 +1656,6 @@ func (p *printer) methodIndex(idx *ast.IndexExpr) (recvPath, recvType string, pa recvPath = tmpVar recvType = typName pathType = vtyp - _ = isReadOnly } else { _ = id // do above @@ -1689,7 +1688,7 @@ func (p *printer) tensorMethod(x *ast.CallExpr, vr *Var, methName string) { } p.print(token.RPAREN, token.RBRACK) if methName == "Set" { - p.print(token.ASSIGN) + p.print(blank, token.ASSIGN, blank) p.expr(args[0]) } } @@ -1701,6 +1700,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { recvType := "" var err error pathIsPackage := false + var rwargs []rwArg var pathType types.Type if sl, ok := path.X.(*ast.SelectorExpr); ok { // path is itself a selector recvPath, recvType, pathType, err = p.methodPath(sl) @@ -1729,23 +1729,28 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { recvType = id.Name // is a package path } } else if idx, ok := path.X.(*ast.IndexExpr); ok { - recvPath, recvType, pathType, err = p.methodIndex(idx) + isReadOnly := false + recvPath, recvType, pathType, isReadOnly, err = p.methodIndex(idx) if err != nil { return } + if !isReadOnly { + rwargs = append(rwargs, rwArg{idx: idx, tmpVar: recvPath}) + } } else { err := fmt.Errorf("gosl methodExpr ERROR: path expression for method call must be simple list of fields, not %#v:", path.X) errors.Log(err) return } args := x.Args - var rwargs []rwArg if pathType != nil { meth, _, _ := types.LookupFieldOrMethod(pathType, true, p.pkg.Types, methName) if meth != nil { if ft, ok := meth.(*types.Func); ok { sig := ft.Type().(*types.Signature) - args, rwargs = p.goslFixArgs(x.Args, sig.Params()) + var rwa []rwArg + args, rwa = p.goslFixArgs(x.Args, sig.Params()) + rwargs = append(rwargs, rwa...) } } if len(rwargs) > 0 { diff --git a/goal/gosl/slrand/slrand.wgsl b/goal/gosl/slrand/slrand.wgsl index 820e7bdf62..372959fcb9 100644 --- a/goal/gosl/slrand/slrand.wgsl +++ b/goal/gosl/slrand/slrand.wgsl @@ -9,7 +9,7 @@ // use on the GPU, with equivalent Go versions available in slrand.go. // This is using the Philox2x32 counter-based RNG. -#include "sltype.wgsl" +// #include "sltype.wgsl" // Philox2x32round does one round of updating of the counter. fn Philox2x32round(counter: su64, key: u32) -> su64 { diff --git a/goal/gosl/sltensor/README.md b/goal/gosl/sltensor/README.md index da6db63506..4ba8e06ae6 100644 --- a/goal/gosl/sltensor/README.md +++ b/goal/gosl/sltensor/README.md @@ -1,7 +1,9 @@ # sltensor: tensor access in WGSL -sltensor provides tensor indexing functions used by gosl to translate tensor access functions into direct code, for `array` and `array` global variables, which encode the strides in their first NumDims values. The 1D case just uses direct indexing with no strides. - -Strides are always encoded for all dimensions to allow complete flexibility in memory organization. +sltensor has functions to set the shape of a [tensor](../../tensor) to encode the strides in their first NumDims values, which are used to index into the tensor values. +For example: +```Go +sltensor.SetShapeSizes(Data, n, 4) +``` diff --git a/goal/gosl/sltensor/sltensor.wgsl b/goal/gosl/sltensor/sltensor.wgsl deleted file mode 100644 index 5aab6a961f..0000000000 --- a/goal/gosl/sltensor/sltensor.wgsl +++ /dev/null @@ -1,18 +0,0 @@ -// sltensor indexing functions - -fn F32Index2D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { - return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; -} - -fn F32Index3D(s0: f32, s1: f32, s2: f32, i0: u32, i1: u32, i2: u32) -> u32 { - return u32(3) + bitcast(s0) * i0 + bitcast(s1) * i1 + bitcast(s2) * i2; -} - -fn U32Index2D(s0: u32, s1: u32, i0: u32, i1: u32) -> u32 { - return u32(2) + s0 * i0 + s1 * i1; -} - -fn U32Index3D(s0: u32, s1: u32, s2: u32, i0: u32, i1: u32, i2: u32) -> u32 { - return u32(3) + s0 * i0 + s1 * i1 + s2 * i2; -} - From 9109105e09662406c958690561afd2eed4c9b63a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 03:38:46 -0700 Subject: [PATCH 202/311] add wgsl --- goal/gosl/examples/rand/rand.wgsl | 16 - goal/gosl/examples/rand/shaders/Compute.wgsl | 388 +++++++++++++++++++ 2 files changed, 388 insertions(+), 16 deletions(-) delete mode 100644 goal/gosl/examples/rand/rand.wgsl create mode 100644 goal/gosl/examples/rand/shaders/Compute.wgsl diff --git a/goal/gosl/examples/rand/rand.wgsl b/goal/gosl/examples/rand/rand.wgsl deleted file mode 100644 index 780ae9ef26..0000000000 --- a/goal/gosl/examples/rand/rand.wgsl +++ /dev/null @@ -1,16 +0,0 @@ - -@group(0) @binding(0) -var Counter: array; - -@group(0) @binding(1) -var Data: array; - -@compute -@workgroup_size(64) -fn main(@builtin(global_invocation_id) idx: vec3) { - var ctr = Counter[0]; - var data = Data[idx.x]; - Rnds_RndGen(&data, ctr, idx.x); - Data[idx.x] = data; -} - diff --git a/goal/gosl/examples/rand/shaders/Compute.wgsl b/goal/gosl/examples/rand/shaders/Compute.wgsl new file mode 100644 index 0000000000..113f10fa95 --- /dev/null +++ b/goal/gosl/examples/rand/shaders/Compute.wgsl @@ -0,0 +1,388 @@ +// Code generated by "gosl"; DO NOT EDIT +// kernel: Compute + +@group(0) @binding(0) +var Seed: array; +@group(0) @binding(1) +var Data: array; + +@compute @workgroup_size(64, 1, 1) +fn main(@builtin(global_invocation_id) idx: vec3) { + Compute(idx.x); +} + + +///////////// import: "slrand.wgsl" +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/slrand +// See README.md there for documentation. + +// These random number generation (RNG) functions are optimized for +// use on the GPU, with equivalent Go versions available in slrand.go. +// This is using the Philox2x32 counter-based RNG. + +// #include "sltype.wgsl" + +// Philox2x32round does one round of updating of the counter. +fn Philox2x32round(counter: su64, key: u32) -> su64 { + let mul = Uint32Mul64(u32(0xD256D193), counter.x); + var ctr: su64; + ctr.x = mul.y ^ key ^ counter.y; + ctr.y = mul.x; + return ctr; +} + +// Philox2x32bumpkey does one round of updating of the key +fn Philox2x32bumpkey(key: u32) -> u32 { + return key + u32(0x9E3779B9); +} + +// Philox2x32 implements the stateless counter-based RNG algorithm +// returning a random number as two uint32 values, given a +// counter and key input that determine the result. +// The input counter is not modified. +fn Philox2x32(counter: su64, key: u32) -> vec2 { + // this is an unrolled loop of 10 updates based on initial counter and key, + // which produces the random deviation deterministically based on these inputs. + var ctr = Philox2x32round(counter, key); // 1 + var ky = Philox2x32bumpkey(key); + ctr = Philox2x32round(ctr, ky); // 2 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 3 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 4 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 5 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 6 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 7 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 8 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 9 + ky = Philox2x32bumpkey(ky); + + return Philox2x32round(ctr, ky); // 10 +} + +//////////////////////////////////////////////////////////// +// Methods below provide a standard interface with more +// readable names, mapping onto the Go rand methods. +// +// They assume a global shared counter, which is then +// incremented by a function index, defined for each function +// consuming random numbers that _could_ be called within a parallel +// processing loop. At the end of the loop, the global counter should +// be incremented by the total possible number of such functions. +// This results in fully resproducible results, invariant to +// specific processing order, and invariant to whether any one function +// actually calls the random number generator. + +// RandUint32Vec2 returns two uniformly distributed 32 unsigned integers, +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Philox2x32(Uint64Add32(counter, funcIndex), key); +} + +// RandUint32 returns a uniformly distributed 32 unsigned integer, +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { + return Philox2x32(Uint64Add32(counter, funcIndex), key).x; +} + +// RandFloat32Vec2 returns two uniformly distributed float32 values in range (0,1), +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} + +// RandFloat32 returns a uniformly distributed float32 value in range (0,1), +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); +} + +// RandFloat32Range11Vec2 returns two uniformly distributed float32 values in range [-1,1], +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} + +// RandFloat32Range11 returns a uniformly distributed float32 value in range [-1,1], +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); +} + +// RandBoolP returns a bool true value with probability p +// based on given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { + return (RandFloat32(counter, funcIndex, key) < p); +} + +fn sincospi(x: f32) -> vec2 { + let PIf = 3.1415926535897932; + var r: vec2; + r.x = cos(PIf*x); + r.y = sin(PIf*x); + return r; +} + +// RandFloat32NormVec2 returns two random float32 numbers +// distributed according to the normal, Gaussian distribution +// with zero mean and unit variance. +// This is done very efficiently using the Box-Muller algorithm +// that consumes two random 32 bit uint values. +// Uses given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + let ur = RandUint32Vec2(counter, funcIndex, key); + var f = sincospi(Uint32ToFloat32Range11(ur.x)); + let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. + return f * r; +} + +// RandFloat32Norm returns a random float32 number +// distributed according to the normal, Gaussian distribution +// with zero mean and unit variance. +// Uses given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { + return RandFloat32Vec2(counter, funcIndex, key).x; +} + +// RandUint32N returns a uint32 in the range [0,N). +// Uses given global shared counter, function index offset from that +// counter for this specific random number call, and key as unique +// index of the item being processed. +fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { + let v = RandFloat32(counter, funcIndex, key); + return u32(v * f32(n)); +} + +// Counter is used for storing the random counter using aligned 16 byte +// storage, with convenience functions for typical use cases. +// It retains a copy of the last Seed value, which is applied to +// the Hi uint32 value. +struct RandCounter { + Counter: su64, + HiSeed: u32, + pad: u32, +} + +// Reset resets counter to last set Seed state. +fn RandCounter_Reset(ct: ptr) { + (*ct).Counter.x = u32(0); + (*ct).Counter.y = (*ct).HiSeed; +} + +// Seed sets the Hi uint32 value from given seed, saving it in Seed field. +// Each increment in seed generates a unique sequence of over 4 billion numbers, +// so it is reasonable to just use incremental values there, but more widely +// spaced numbers will result in longer unique sequences. +// Resets Lo to 0. +// This same seed will be restored during Reset +fn RandCounter_Seed(ct: ptr, seed: u32) { + (*ct).HiSeed = seed; + RandCounter_Reset(ct); +} + +// Add increments the counter by given amount. +// Call this after completing a pass of computation +// where the value passed here is the max of funcIndex+1 +// used for any possible random calls during that pass. +fn RandCounter_Add(ct: ptr, inc: u32) { + (*ct).Counter = Uint64Add32((*ct).Counter, inc); +} + + +///////////// import: "sltype.wgsl" +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/sltype +// See README.md there for documentation. + +// This file emulates uint64 (u64) using 2 uint32 integers. +// and defines conversions between uint and float. + +// define a u64 type as an alias. +// if / when u64 actually happens, will make it easier to update. +alias su64 = vec2; + +// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using vec2). +fn Uint32Mul64(a: u32, b: u32) -> su64 { + let LOMASK = (((u32(1))<<16)-1); + var r: su64; + r.x = a * b; /* full low multiply */ + let ahi = a >> 16; + let alo = a & LOMASK; + let bhi = b >> 16; + let blo = b & LOMASK; + + let ahbl = ahi * blo; + let albh = alo * bhi; + + let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); + var hit = ahi*bhi + (ahbl>>16) + (albh>>16); + hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ + /* carry from the sum with alo*blo */ + if ((r.x >> u32(16)) < (ahbl_albh&LOMASK)) { + hit += u32(1); + } + r.y = hit; + return r; +} + +/* +// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using su64). +fn Uint32Mul64(a: u32, b: u32) -> su64 { + return su64(a) * su64(b); +} +*/ + + +// Uint64Add32 adds given uint32 number to given uint64 (using vec2). +fn Uint64Add32(a: su64, b: u32) -> su64 { + if (b == 0) { + return a; + } + var s = a; + if (s.x > u32(0xffffffff) - b) { + s.y++; + s.x = (b - 1) - (u32(0xffffffff) - s.x); + } else { + s.x += b; + } + return s; +} + +// Uint64Incr returns increment of the given uint64 (using vec2). +fn Uint64Incr(a: su64) -> su64 { + var s = a; + if(s.x == 0xffffffff) { + s.y++; + s.x = u32(0); + } else { + s.x++; + } + return s; +} + +// Uint32ToFloat32 converts a uint32 integer into a float32 +// in the (0,1) interval (i.e., exclusive of 1). +// This differs from the Go standard by excluding 0, which is handy for passing +// directly to Log function, and from the reference Philox code by excluding 1 +// which is in the Go standard and most other standard RNGs. +fn Uint32ToFloat32(val: u32) -> f32 { + let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + var f = f32(val) * factor + halffactor; + if (f == 1.0) { // exclude 1 + return bitcast(0x3F7FFFFF); + } + return f; +} + +// note: there is no overloading of user-defined functions +// https://github.com/gpuweb/gpuweb/issues/876 + +// Uint32ToFloat32Vec2 converts two uint 32 bit integers +// into two corresponding 32 bit f32 values +// in the (0,1) interval (i.e., exclusive of 1). +fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32(val.x); + r.y = Uint32ToFloat32(val.y); + return r; +} + +// Uint32ToFloat32Range11 converts a uint32 integer into a float32 +// in the [-1..1] interval (inclusive of -1 and 1, never identically == 0). +fn Uint32ToFloat32Range11(val: u32) -> f32 { + let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + return (f32(val) * factor + halffactor); +} + +// Uint32ToFloat32Range11Vec2 converts two uint32 integers into two float32 +// in the [-1,1] interval (inclusive of -1 and 1, never identically == 0). +fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32Range11(val.x); + r.y = Uint32ToFloat32Range11(val.y); + return r; +} + + + + +///////////// import: "rand.wgsl" + +//gosl:vars + +//gosl:read-only + +// Data + +struct Seeds { + Seed: su64, + pad: i32, + pad1: i32, +} + +struct Rnds { + Uints: vec2, + pad: i32, + pad1: i32, + Floats: vec2, + pad2: i32, + pad3: i32, + Floats11: vec2, + pad4: i32, + pad5: i32, + Gauss: vec2, + pad6: i32, + pad7: i32, +} + +// RndGen calls random function calls to test generator. +// Note that the counter to the outer-most computation function +// is passed by *value*, so the same counter goes to each element +// as it is computed, but within this scope, counter is passed by +// reference (as a pointer) so subsequent calls get a new counter value. +// The counter should be incremented by the number of random calls +// outside of the overall update function. +fn Rnds_RndGen(r: ptr, counter: su64, idx: u32) { + (*r).Uints = RandUint32Vec2(counter, u32(0), idx); + (*r).Floats = RandFloat32Vec2(counter, u32(1), idx); + (*r).Floats11 = RandFloat32Range11Vec2(counter, u32(2), idx); + (*r).Gauss = RandFloat32NormVec2(counter, u32(3), idx); +} + +fn Compute(i: u32) { //gosl:kernel + var data=Data[i]; + Rnds_RndGen(&data, Seed[0].Seed, i); + Data[i]=data; +} From 03ca3277c101bc2fe8daa0206881977296f28014 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 03:45:29 -0700 Subject: [PATCH 203/311] move gosl back up to main gosl dir -- don't need other things in gosl space b/c everything is generated. --- goal/gosl/{cmd/gosl => }/gosl.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename goal/gosl/{cmd/gosl => }/gosl.go (100%) diff --git a/goal/gosl/cmd/gosl/gosl.go b/goal/gosl/gosl.go similarity index 100% rename from goal/gosl/cmd/gosl/gosl.go rename to goal/gosl/gosl.go From f8e60835737e30f770815f7dc62cc30364826265 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 11:57:17 -0700 Subject: [PATCH 204/311] matrix.Inverse function, and return errors instead of panic in matrix functions. also return *Float64 directly as that makes more sense -- goal wraps in generic tensor automatically but ops should return the most specific value possible to facilitate further opts. --- goal/GPU.md | 4 +- goal/gosl/slrand/slrand_test.go | 2 +- tensor/matrix/matrix_test.go | 27 ++++-- tensor/matrix/ops.go | 93 +++++++++++++++++-- tensor/tmath/ops.go | 12 +-- .../symbols/cogentcore_org-core-tensor.go | 1 + 6 files changed, 115 insertions(+), 24 deletions(-) diff --git a/goal/GPU.md b/goal/GPU.md index e28155fd0f..aaf035ca63 100644 --- a/goal/GPU.md +++ b/goal/GPU.md @@ -2,7 +2,9 @@ The use of massively parallel _Graphical Processsing Unit_ (_GPU_) hardware has revolutionized machine learning and other fields, producing many factors of speedup relative to traditional _CPU_ (_Central Processing Unit_) computation. However, there are numerous challenges for supporting GPU-based computation, relative to the more flexible CPU coding. -Goal provides a solution to these challenges that enables the same Go-based code to work efficiently and reasonably naturally on both the GPU and CPU (i.e., standard Go execution), for maximum portability. The ability to run the same code on both types of hardware is also critical for debugging the otherwise difficult to debug GPU version, and for avoiding bugs in the first place by catching them first on the CPU, while providing known correct comparison results. +The Goal framework provides a solution to these challenges that enables the same Go-based code to work efficiently and reasonably naturally on both the GPU and CPU (i.e., standard Go execution), via the [gosl](gosl) package. Debugging code on the GPU is notoriously difficult because the usual tools are not directly available (not even print statements), so the ability to run exactly the same code on the CPU is invaluable, in addition to the benefits in portability across platforms without GPU hardware. + +See the [gosl](gosl) documentation for the details on how to write code that works on the GPU. The remainder of this document provides an overview of the overall approach in relation to other related tools. The two most important challenges are: diff --git a/goal/gosl/slrand/slrand_test.go b/goal/gosl/slrand/slrand_test.go index 82c3d976af..38114148f7 100644 --- a/goal/gosl/slrand/slrand_test.go +++ b/goal/gosl/slrand/slrand_test.go @@ -8,7 +8,7 @@ import ( "fmt" "testing" - "cogentcore.org/core/gpu/gosl/sltype" + "cogentcore.org/core/goal/gosl/sltype" "github.com/stretchr/testify/assert" ) diff --git a/tensor/matrix/matrix_test.go b/tensor/matrix/matrix_test.go index c97523a8f5..8304d3d6e6 100644 --- a/tensor/matrix/matrix_test.go +++ b/tensor/matrix/matrix_test.go @@ -7,6 +7,7 @@ package matrix import ( "testing" + "cogentcore.org/core/base/tolassert" "cogentcore.org/core/tensor" "github.com/stretchr/testify/assert" ) @@ -18,47 +19,53 @@ func TestMatrix(t *testing.T) { v := tensor.NewFloat64FromValues(2, 3) _ = v - o := Mul(a, a).(*tensor.Float64) + o := Mul(a, a) assert.Equal(t, []float64{7, 10, 15, 22}, o.Values) - o = Mul(a, v).(*tensor.Float64) + o = Mul(a, v) assert.Equal(t, []float64{8, 18}, o.Values) assert.Equal(t, []int{2}, o.Shape().Sizes) - o = Mul(v, a).(*tensor.Float64) + o = Mul(v, a) assert.Equal(t, []float64{11, 16}, o.Values) assert.Equal(t, []int{2}, o.Shape().Sizes) nr := 3 - b := tensor.NewFloat64(nr, 2, 2) + b := tensor.NewFloat64(nr, 1, 2, 2) for r := range nr { b.SetRowTensor(a, r) } // fmt.Println(b) - o = Mul(b, a).(*tensor.Float64) + o = Mul(b, a) assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) - o = Mul(a, b).(*tensor.Float64) + o = Mul(a, b) assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) - o = Mul(b, b).(*tensor.Float64) + o = Mul(b, b) assert.Equal(t, []float64{7, 10, 15, 22, 7, 10, 15, 22, 7, 10, 15, 22}, o.Values) assert.Equal(t, []int{3, 2, 2}, o.Shape().Sizes) - o = Mul(v, b).(*tensor.Float64) + o = Mul(v, b) assert.Equal(t, []float64{11, 16, 11, 16, 11, 16}, o.Values) assert.Equal(t, []int{3, 2}, o.Shape().Sizes) - o = Mul(b, v).(*tensor.Float64) + o = Mul(b, v) assert.Equal(t, []float64{8, 18, 8, 18, 8, 18}, o.Values) assert.Equal(t, []int{3, 2}, o.Shape().Sizes) - o = Mul(a, tensor.Transpose(a)).(*tensor.Float64) + o = Mul(a, tensor.Transpose(a)) assert.Equal(t, []float64{5, 11, 11, 25}, o.Values) d := Det(a) assert.Equal(t, -2.0, d.Float1D(0)) + + inv := Inverse(a) + tolassert.EqualTolSlice(t, []float64{-2, 1, 1.5, -0.5}, inv.Values, 1.0e-8) + + inv = Inverse(b) + tolassert.EqualTolSlice(t, []float64{-2, 1, 1.5, -0.5, -2, 1, 1.5, -0.5, -2, 1, 1.5, -0.5}, inv.Values, 1.0e-8) } diff --git a/tensor/matrix/ops.go b/tensor/matrix/ops.go index 3a7e5dfce0..c7df78bc1f 100644 --- a/tensor/matrix/ops.go +++ b/tensor/matrix/ops.go @@ -12,9 +12,17 @@ import ( "gonum.org/v1/gonum/mat" ) +// CallOut1 calls an Out function with 1 input arg. All matrix functions +// require *tensor.Float64 outputs. +func CallOut1(fun func(a tensor.Tensor, out *tensor.Float64) error, a tensor.Tensor) *tensor.Float64 { + out := tensor.NewFloat64() + errors.Log(fun(a, out)) + return out +} + // CallOut2 calls an Out function with 2 input args. All matrix functions // require *tensor.Float64 outputs. -func CallOut2(fun func(a, b tensor.Tensor, out *tensor.Float64) error, a, b tensor.Tensor) tensor.Values { +func CallOut2(fun func(a, b tensor.Tensor, out *tensor.Float64) error, a, b tensor.Tensor) *tensor.Float64 { out := tensor.NewFloat64() errors.Log(fun(a, b, out)) return out @@ -30,14 +38,14 @@ func CallOut2(fun func(a, b tensor.Tensor, out *tensor.Float64) error, a, b tens // a 1 to its dimensions. After matrix multiplication the prepended 1 is removed. // - If the second argument is 1-D, it is promoted to a matrix by appending // a 1 to its dimensions. After matrix multiplication the appended 1 is removed. -func Mul(a, b tensor.Tensor) tensor.Tensor { +func Mul(a, b tensor.Tensor) *tensor.Float64 { return CallOut2(MulOut, a, b) } // MulOut performs matrix multiplication, into the given output tensor, // using the following rules based // on the shapes of the relevant tensors. If the tensor shapes are not -// suitable, an error is returned. +// suitable, a [gonum] [mat.ErrShape] error is returned. // - If both arguments are 2-D they are multiplied like conventional matrices. // The result has shape a.Rows, b.Columns. // - If either argument is N-D, N > 2, it is treated as a stack of matrices @@ -77,6 +85,8 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { if asz[0] == 1 { ea = tensor.Reshape(a, asz[1:]...) na = 2 + } else { + ea = tensor.Reshape(a, asz...) } } if nb > 2 { @@ -84,16 +94,24 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { if bsz[0] == 1 { eb = tensor.Reshape(b, bsz[1:]...) nb = 2 + } else { + eb = tensor.Reshape(b, bsz...) } } switch { case na == nb && na == 2: + if ea.DimSize(1) != eb.DimSize(0) { + return mat.ErrShape + } ma, _ := NewMatrix(ea) mb, _ := NewMatrix(eb) out.SetShapeSizes(ea.DimSize(0), eb.DimSize(1)) do, _ := NewDense(out) do.Mul(ma, mb) case na > 2 && nb == 2: + if ea.DimSize(2) != eb.DimSize(0) { + return mat.ErrShape + } mb, _ := NewMatrix(eb) nr := ea.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(1), eb.DimSize(1)) @@ -104,6 +122,9 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { do.Mul(ma, mb) } case nb > 2 && na == 2: + if ea.DimSize(1) != eb.DimSize(1) { + return mat.ErrShape + } ma, _ := NewMatrix(ea) nr := eb.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(0), eb.DimSize(2)) @@ -117,6 +138,9 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { if ea.DimSize(0) != eb.DimSize(0) { return errors.New("matrix.Mul: a and b input matricies are > 2 dimensional; must have same outer dimension sizes") } + if ea.DimSize(2) != eb.DimSize(1) { + return mat.ErrShape + } nr := ea.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(1), eb.DimSize(2)) for r := range nr { @@ -128,7 +152,7 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { do.Mul(ma, mb) } default: - return errors.New("matrix.Mul: input dimensions do not align") + return mat.ErrShape } if collapse { nd := out.NumDims() @@ -146,7 +170,7 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { // For a 2D matrix [[a, b], [c, d]] it this is ad - bc. // See also [LogDet] for a version that is more numerically // stable for large matricies. -func Det(a tensor.Tensor) tensor.Tensor { +func Det(a tensor.Tensor) *tensor.Float64 { m, err := NewMatrix(a) if errors.Log(err) != nil { return tensor.NewFloat64Scalar(0) @@ -158,7 +182,7 @@ func Det(a tensor.Tensor) tensor.Tensor { // as the log and sign of the value, which is more // numerically stable. The return is a 1D vector of length 2, // with the first value being the log, and the second the sign. -func LogDet(a tensor.Tensor) tensor.Tensor { +func LogDet(a tensor.Tensor) *tensor.Float64 { m, err := NewMatrix(a) if errors.Log(err) != nil { return tensor.NewFloat64Scalar(0) @@ -166,3 +190,60 @@ func LogDet(a tensor.Tensor) tensor.Tensor { l, s := mat.LogDet(m) return tensor.NewFloat64FromValues(l, s) } + +// Inverse performs matrix inversion of a square matrix, +// logging an error for non-invertable cases. +// See [InverseOut] for a version that returns an error. +// If the input tensor is > 2D, it is treated as a list of 2D matricies +// which are each inverted. +func Inverse(a tensor.Tensor) *tensor.Float64 { + return CallOut1(InverseOut, a) +} + +// InverseOut performs matrix inversion of a square matrix, +// returning an error for non-invertable cases. If the input tensor +// is > 2D, it is treated as a list of 2D matricies which are each inverted. +func InverseOut(a tensor.Tensor, out *tensor.Float64) error { + if err := StringCheck(a); err != nil { + return err + } + na := a.NumDims() + if na == 1 { + return mat.ErrShape + } + var asz []int + ea := a + if na > 2 { + asz = tensor.SplitAtInnerDims(a, 2) + if asz[0] == 1 { + ea = tensor.Reshape(a, asz[1:]...) + na = 2 + } + } + if na == 2 { + if a.DimSize(0) != a.DimSize(1) { + return mat.ErrShape + } + ma, _ := NewMatrix(a) + out.SetShapeSizes(a.DimSize(0), a.DimSize(1)) + do, _ := NewDense(out) + return do.Inverse(ma) + } + ea = tensor.Reshape(a, asz...) + if ea.DimSize(1) != ea.DimSize(2) { + return mat.ErrShape + } + nr := ea.DimSize(0) + out.SetShapeSizes(nr, ea.DimSize(1), ea.DimSize(2)) + var errs []error + for r := range nr { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + err := do.Inverse(ma) + if err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} diff --git a/tensor/tmath/ops.go b/tensor/tmath/ops.go index 6b8a5bc924..9e400d6a94 100644 --- a/tensor/tmath/ops.go +++ b/tensor/tmath/ops.go @@ -64,7 +64,7 @@ func Dec(a tensor.Tensor) error { } // Add adds two tensors into output. -func Add(a, b tensor.Tensor) tensor.Tensor { +func Add(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(AddOut, a, b) } @@ -77,7 +77,7 @@ func AddOut(a, b tensor.Tensor, out tensor.Values) error { } // Sub subtracts tensors into output. -func Sub(a, b tensor.Tensor) tensor.Tensor { +func Sub(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(SubOut, a, b) } @@ -87,7 +87,7 @@ func SubOut(a, b tensor.Tensor, out tensor.Values) error { } // Mul multiplies tensors into output. -func Mul(a, b tensor.Tensor) tensor.Tensor { +func Mul(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(MulOut, a, b) } @@ -98,7 +98,7 @@ func MulOut(a, b tensor.Tensor, out tensor.Values) error { // Div divides tensors into output. always does floating point division, // even with integer operands. -func Div(a, b tensor.Tensor) tensor.Tensor { +func Div(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2Float64(DivOut, a, b) } @@ -108,7 +108,7 @@ func DivOut(a, b tensor.Tensor, out tensor.Values) error { } // Mod performs modulus a%b on tensors into output. -func Mod(a, b tensor.Tensor) tensor.Tensor { +func Mod(a, b tensor.Tensor) tensor.Values { return tensor.CallOut2(ModOut, a, b) } @@ -118,7 +118,7 @@ func ModOut(a, b tensor.Tensor, out tensor.Values) error { } // Negate stores in the output the bool value -a. -func Negate(a tensor.Tensor) tensor.Tensor { +func Negate(a tensor.Tensor) tensor.Values { return tensor.CallOut1(NegateOut, a) } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/symbols/cogentcore_org-core-tensor.go index 4d73caf4b2..26ce555dce 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor.go @@ -112,6 +112,7 @@ func init() { "NewStringFull": reflect.ValueOf(tensor.NewStringFull), "NewStringScalar": reflect.ValueOf(tensor.NewStringScalar), "NewStringShape": reflect.ValueOf(tensor.NewStringShape), + "NewUint32": reflect.ValueOf(tensor.NewUint32), "NumThreads": reflect.ValueOf(&tensor.NumThreads).Elem(), "OddColumn": reflect.ValueOf(tensor.OddColumn), "OddRow": reflect.ValueOf(tensor.OddRow), From 6891c5eb68900234c9cec4ccd70da75eb0d4def6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 13:55:14 -0700 Subject: [PATCH 205/311] gotosl test now working with new paradigm --- goal/gosl/gotosl/files.go | 12 +- goal/gosl/gotosl/gengpu.go | 4 +- goal/gosl/gotosl/gosl_test.go | 83 ++------- goal/gosl/gotosl/gotosl.go | 3 + goal/gosl/gotosl/nodes.go | 2 +- .../testdata/{basic.golden => Compute.golden} | 42 +++-- goal/gosl/gotosl/testdata/basic.go | 56 ++---- goal/gosl/gotosl/testdata/gosl.golden | 172 ++++++++++++++++++ goal/gosl/gotosl/typegen.go | 52 +++--- 9 files changed, 274 insertions(+), 152 deletions(-) rename goal/gosl/gotosl/testdata/{basic.golden => Compute.golden} (88%) create mode 100644 goal/gosl/gotosl/testdata/gosl.golden diff --git a/goal/gosl/gotosl/files.go b/goal/gosl/gotosl/files.go index b9a55caeaa..0c542c1dca 100644 --- a/goal/gosl/gotosl/files.go +++ b/goal/gosl/gotosl/files.go @@ -46,10 +46,16 @@ func WriteFileLines(fn string, lines [][]byte) error { } // HasGoslTag returns true if given file has a //gosl: tag -func HasGoslTag(lines [][]byte) bool { +func (st *State) HasGoslTag(lines [][]byte) bool { key := []byte("//gosl:") + pkg := []byte("package ") for _, ln := range lines { tln := bytes.TrimSpace(ln) + if st.Package == "" { + if bytes.HasPrefix(tln, pkg) { + st.Package = string(bytes.TrimPrefix(tln, pkg)) + } + } if bytes.HasPrefix(tln, key) { return true } @@ -78,7 +84,7 @@ func (st *State) ProjectFiles() { if err != nil { continue } - if !HasGoslTag(fl.Lines) { + if !st.HasGoslTag(fl.Lines) { continue } st.GoFiles[fn] = fl @@ -125,7 +131,7 @@ func (st *State) ImportFiles(lines [][]byte) { if err != nil { continue } - if !HasGoslTag(lns) { + if !st.HasGoslTag(lns) { continue } _, fo := filepath.Split(gf) diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index fc8587ff0a..c590a164fe 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -31,7 +31,7 @@ func (st *State) GenGPU() { header := `// Code generated by "gosl"; DO NOT EDIT -package main +package %s import ( "embed" @@ -50,7 +50,7 @@ var UseGPU bool ` - b.WriteString(fmt.Sprintf(header, st.Config.Output)) + b.WriteString(fmt.Sprintf(header, st.Package, st.Config.Output)) for _, sy := range st.Systems { b.WriteString(fmt.Sprintf("// %s is a GPU compute System with kernels operating on the\n// same set of data variables.\n", st.genSysVar(sy))) diff --git a/goal/gosl/gotosl/gosl_test.go b/goal/gosl/gotosl/gosl_test.go index 8e23328ba5..056b3e6e4b 100644 --- a/goal/gosl/gotosl/gosl_test.go +++ b/goal/gosl/gotosl/gosl_test.go @@ -5,94 +5,43 @@ package gotosl import ( - "bytes" - "flag" "os" - "path/filepath" - "strings" "testing" + "cogentcore.org/core/cli" "github.com/stretchr/testify/assert" ) -var update = flag.Bool("update", false, "update .golden files") +// TestTranslate +func TestTranslate(t *testing.T) { + os.Chdir("testdata") -func runTest(t *testing.T, in, out string) { - // process flags - _, err := os.Lstat(in) + opts := cli.DefaultOptions("gosl", "Go as a shader language converts Go code to WGSL WebGPU shader code, which can be run on the GPU through WebGPU.") + cfg := &Config{} + cli.Run(opts, cfg, Run) + + exSh, err := os.ReadFile("Compute.golden") if err != nil { t.Error(err) return } - - sls, err := ProcessFiles([]string{in}) + exGosl, err := os.ReadFile("gosl.golden") if err != nil { t.Error(err) return } - expected, err := os.ReadFile(out) + gotSh, err := os.ReadFile("shaders/Compute.wgsl") if err != nil { t.Error(err) return } - - var got []byte - for _, b := range sls { - got = b - break - } - - if !bytes.Equal(got, expected) { - if *update { - if in != out { - if err := os.WriteFile(out, got, 0666); err != nil { - t.Error(err) - } - return - } - // in == out: don't accidentally destroy input - t.Errorf("WARNING: -update did not rewrite input file %s", in) - } - - assert.Equal(t, string(expected), string(got)) - if err := os.WriteFile(in+".gosl", got, 0666); err != nil { - t.Error(err) - } - } -} - -// TestRewrite processes testdata/*.input files and compares them to the -// corresponding testdata/*.golden files. The gosl flags used to process -// a file must be provided via a comment of the form -// -// //gosl flags -// -// in the processed file within the first 20 lines, if any. -func TestRewrite(t *testing.T) { - if gomod := os.Getenv("GO111MODULE"); gomod == "off" { - t.Error("gosl only works in go modules mode, but GO111MODULE=off") - return - } - - // determine input files - match, err := filepath.Glob("testdata/*.go") + gotGosl, err := os.ReadFile("gosl.go") if err != nil { - t.Fatal(err) - } - - if *outDir != "" { - os.MkdirAll(*outDir, 0755) + t.Error(err) + return } - for _, in := range match { - name := filepath.Base(in) - t.Run(name, func(t *testing.T) { - out := in // for files where input and output are identical - if strings.HasSuffix(in, ".go") { - out = in[:len(in)-len(".go")] + ".golden" - } - runTest(t, in, out) - }) - } + assert.Equal(t, string(exSh), string(gotSh)) + assert.Equal(t, string(exGosl), string(gotGosl)) } diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index b5ca0d83e3..057502657e 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -139,6 +139,9 @@ type State struct { // path to shaders/imports directory. ImportsDir string + // name of the package + Package string + // GoFiles are all the files with gosl content in current directory. GoFiles map[string]*File diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 152f905c67..92c13e3732 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1722,7 +1722,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { recvType = id.Name // is a package path } else { pathType = typ - recvPath = "&" + recvPath + recvPath = recvPath } } else { pathIsPackage = true diff --git a/goal/gosl/gotosl/testdata/basic.golden b/goal/gosl/gotosl/testdata/Compute.golden similarity index 88% rename from goal/gosl/gotosl/testdata/basic.golden rename to goal/gosl/gotosl/testdata/Compute.golden index feabd4e16a..252e19440c 100644 --- a/goal/gosl/gotosl/testdata/basic.golden +++ b/goal/gosl/gotosl/testdata/Compute.golden @@ -1,13 +1,27 @@ +// Code generated by "gosl"; DO NOT EDIT +// kernel: Compute +// // Params are the parameters for the computation. +@group(0) @binding(0) +var Params: array; +@group(0) @binding(1) +var Data: array; -// note: here is the wgsl version, only included in wgsl - -// MyTrickyFun this is the GPU version of the tricky function -fn MyTrickyFun(x: f32) -> f32 { - return 16.0; // ok actually not tricky here, but whatever +@compute @workgroup_size(64, 1, 1) +fn main(@builtin(global_invocation_id) idx: vec3) { + Compute(idx.x); } +///////////// import: "basic.wgsl" + +//gosl:vars + +// Params are the parameters for the computation. +//gosl:read-only + +// Data is the data on which the computation operates. + // FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph // It does not have any of the sanity checking of a standard method -- returns // nonsense when arg is out of range. Runs in 2.23ns vs. 6.3ns for 64bit which is faster @@ -150,17 +164,9 @@ fn ParamStruct_AnotherMeth(ps: ptr, ds: ptr Params: array; - -@group(0) @binding(1) -var Data: array; - -@compute -@workgroup_size(64) -fn main(@builtin(global_invocation_id) idx: vec3) { - var pars = Params[0]; - var data = Data[idx.x]; - ParamStruct_IntegFromRaw(&pars, &data); - Data[idx.x] = data; +// Compute does the main computation +fn Compute(i: u32) { //gosl:kernel + var params=Params[0]; var data=Data[i]; + ParamStruct_IntegFromRaw(¶ms, &data); + Data[i]=data; } diff --git a/goal/gosl/gotosl/testdata/basic.go b/goal/gosl/gotosl/testdata/basic.go index 72a3610c95..1f0215e8e8 100644 --- a/goal/gosl/gotosl/testdata/basic.go +++ b/goal/gosl/gotosl/testdata/basic.go @@ -3,31 +3,21 @@ package test import ( "math" + "cogentcore.org/core/goal/gosl/slbool" "cogentcore.org/core/math32" - "cogentcore.org/core/vgpu/gosl/slbool" ) -// note: this code is included in the go pre-processing output but -// then removed from the final wgsl output. -// Use when you need different versions of the same function for CPU vs. GPU +//gosl:start -// MyTrickyFun this is the CPU version of the tricky function -func MyTrickyFun(x float32) float32 { - return 10 // ok actually not tricky here, but whatever -} - -//gosl:wgsl basic - -// // note: here is the wgsl version, only included in wgsl - -// // MyTrickyFun this is the GPU version of the tricky function -// fn MyTrickyFun(x: f32) -> f32 { -// return 16.0; // ok actually not tricky here, but whatever -// } - -//gosl:end basic +//gosl:vars +var ( + // Params are the parameters for the computation. + //gosl:read-only + Params []ParamStruct -//gosl:start basic + // Data is the data on which the computation operates. + Data []DataStruct +) // FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph // It does not have any of the sanity checking of a standard method -- returns @@ -166,7 +156,12 @@ func (ps *ParamStruct) AnotherMeth(ds *DataStruct, ptrarg *float32) { *ptrarg = -1 } -//gosl:end basic +// Compute does the main computation +func Compute(i uint32) { //gosl:kernel + Params[0].IntegFromRaw(&Data[i]) +} + +//gosl:end // note: only core compute code needs to be in shader -- all init is done CPU-side @@ -178,22 +173,3 @@ func (ps *ParamStruct) Defaults() { func (ps *ParamStruct) Update() { ps.Dt = 1.0 / ps.Tau } - -//gosl:wgsl basic -/* -@group(0) @binding(0) -var Params: array; - -@group(0) @binding(1) -var Data: array; - -@compute -@workgroup_size(64) -fn main(@builtin(global_invocation_id) idx: vec3) { - var pars = Params[0]; - var data = Data[idx.x]; - ParamStruct_IntegFromRaw(&pars, &data); - Data[idx.x] = data; -} -*/ -//gosl:end basic diff --git a/goal/gosl/gotosl/testdata/gosl.golden b/goal/gosl/gotosl/testdata/gosl.golden new file mode 100644 index 0000000000..70455d54fa --- /dev/null +++ b/goal/gosl/gotosl/testdata/gosl.golden @@ -0,0 +1,172 @@ +// Code generated by "gosl"; DO NOT EDIT + +package test + +import ( + "embed" + "unsafe" + "cogentcore.org/core/gpu" +) + +//go:embed shaders/*.wgsl +var shaders embed.FS + +// ComputeGPU is the compute gpu device +var ComputeGPU *gpu.GPU + +// UseGPU indicates whether to use GPU vs. CPU. +var UseGPU bool + +// GPUSystem is a GPU compute System with kernels operating on the +// same set of data variables. +var GPUSystem *gpu.ComputeSystem + +// GPUVars is an enum for GPU variables, for specifying what to sync. +type GPUVars int32 //enums:enum + +const ( + ParamsVar GPUVars = 0 + DataVar GPUVars = 1 +) + +// GPUInit initializes the GPU compute system, +// configuring system(s), variables and kernels. +func GPUInit() { + gp := gpu.NewComputeGPU() + ComputeGPU = gp + { + sy := gpu.NewComputeSystem(gp, "Default") + GPUSystem = sy + gpu.NewComputePipelineShaderFS(shaders, "shaders/Compute.wgsl", sy) + vars := sy.Vars() + { + sgp := vars.AddGroup(gpu.Storage) + var vr *gpu.Var + vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) + vr.ReadOnly = true + vr = sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), 1, gpu.ComputeShader) + sgp.SetNValues(1) + } + sy.Config() + } +} + +// GPURelease releases the GPU compute system resources. +// Call this at program exit. +func GPURelease() { + GPUSystem.Release() + ComputeGPU.Release() +} + +// RunCompute runs the Compute kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// Can call multiple Run* kernels in a row, which are then all launched +// in the same command submission on the GPU, which is by far the most efficient. +// MUST call RunDone (with optional vars to sync) after all Run calls. +// Alternatively, a single-shot RunOneCompute call does Run and Done for a +// single run-and-sync case. +func RunCompute(n int) { + if UseGPU { + RunComputeGPU(n) + } else { + RunComputeCPU(n) + } +} + +// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. +func RunComputeGPU(n int) { + sy := GPUSystem + pl := sy.ComputePipelines["Compute"] + ce, _ := sy.BeginComputePass() + pl.Dispatch1D(ce, n, 64) +} + +// RunComputeCPU runs the Compute kernel on the CPU. +func RunComputeCPU(n int) { + // todo: need threaded api -- not tensor + for i := range n { + Compute(uint32(i)) + } +} + +// RunOneCompute runs the Compute kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// This version then calls RunDone with the given variables to sync +// after the Run, for a single-shot Run-and-Done call. If multiple kernels +// can be run in sequence, it is much more efficient to do multiple Run* +// calls followed by a RunDone call. +func RunOneCompute(n int, syncVars ...GPUVars) { + if UseGPU { + RunComputeGPU(n) + RunDone(syncVars...) + } else { + RunComputeCPU(n) + } +} + +// RunDone must be called after Run* calls to start compute kernels. +// This actually submits the kernel jobs to the GPU, and adds commands +// to synchronize the given variables back from the GPU to the CPU. +// After this function completes, the GPU results will be available in +// the specified variables. +func RunDone(syncVars ...GPUVars) { + if !UseGPU { + return + } + sy := GPUSystem + sy.ComputeEncoder.End() + ReadFromGPU(syncVars...) + sy.EndComputePass() + SyncFromGPU(syncVars...) +} + + +// ToGPU copies given variables to the GPU for the system. +func ToGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, vr := range vars { + switch vr { + case ParamsVar: + v, _ := syVars.ValueByIndex(0, "Params", 0) + gpu.SetValueFrom(v, Params) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + gpu.SetValueFrom(v, Data) + } + } +} + +// ReadFromGPU starts the process of copying vars to the GPU. +func ReadFromGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, vr := range vars { + switch vr { + case ParamsVar: + v, _ := syVars.ValueByIndex(0, "Params", 0) + v.GPUToRead(sy.CommandEncoder) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + v.GPUToRead(sy.CommandEncoder) + } + } +} + +// SyncFromGPU synchronizes vars from the GPU to the actual variable. +func SyncFromGPU(vars ...GPUVars) { + sy := GPUSystem + syVars := sy.Vars() + for _, vr := range vars { + switch vr { + case ParamsVar: + v, _ := syVars.ValueByIndex(0, "Params", 0) + v.ReadSync() + gpu.ReadToBytes(v, Params) + case DataVar: + v, _ := syVars.ValueByIndex(0, "Data", 0) + v.ReadSync() + gpu.ReadToBytes(v, Data) + } + } +} diff --git a/goal/gosl/gotosl/typegen.go b/goal/gosl/gotosl/typegen.go index ea67c466fe..947b65642f 100644 --- a/goal/gosl/gotosl/typegen.go +++ b/goal/gosl/gotosl/typegen.go @@ -8,10 +8,24 @@ import ( var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system.", Fields: []types.Field{{Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.System", IDName: "system", Doc: "System represents a ComputeSystem, and its kernels and variables.", Fields: []types.Field{{Name: "Name"}, {Name: "Kernels", Doc: "Kernels are the kernels using this compute system."}, {Name: "Groups", Doc: "Groups are the variables for this compute system."}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Kernel", IDName: "kernel", Doc: "Kernel represents a kernel function, which is the basis for\neach wgsl generated code file.", Fields: []types.Field{{Name: "Name"}, {Name: "Args"}, {Name: "Filename", Doc: "Filename is the name of the kernel shader file, e.g., shaders/Compute.wgsl"}, {Name: "FuncCode", Doc: "function code"}, {Name: "Lines", Doc: "Lines is full shader code"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Var", IDName: "var", Doc: "Var represents one global system buffer variable.", Fields: []types.Field{{Name: "Name"}, {Name: "Doc", Doc: "comment docs about this var."}, {Name: "Type", Doc: "Type of variable: either []Type or F32, U32 for tensors"}, {Name: "ReadOnly", Doc: "ReadOnly indicates that this variable is never read back from GPU,\nspecified by the gosl:read-only property in the variable comments.\nIt is important to optimize GPU memory usage to indicate this."}, {Name: "Tensor", Doc: "True if a tensor type"}, {Name: "TensorDims", Doc: "Number of dimensions"}, {Name: "TensorKind", Doc: "data kind of the tensor"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Group", IDName: "group", Doc: "Group represents one variable group.", Fields: []types.Field{{Name: "Name"}, {Name: "Doc", Doc: "comment docs about this group"}, {Name: "Uniform", Doc: "Uniform indicates a uniform group; else default is Storage"}, {Name: "Vars"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.File", IDName: "file", Doc: "File has contents of a file as lines of bytes.", Fields: []types.Field{{Name: "Name"}, {Name: "Lines"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current Go -> WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}}}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.exprListMode", IDName: "expr-list-mode"}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.paramMode", IDName: "param-mode"}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.rwArg", IDName: "rw-arg", Fields: []types.Field{{Name: "idx"}, {Name: "tmpVar"}}}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.sizeCounter", IDName: "size-counter", Doc: "sizeCounter is an io.Writer which counts the number of bytes written,\nas well as whether a newline character was seen.", Fields: []types.Field{{Name: "hasNewline"}, {Name: "size"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.whiteSpace", IDName: "white-space"}) @@ -26,24 +40,12 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.tr var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Mode", IDName: "mode", Doc: "A Mode value is a set of flags (or 0). They control printing."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.PrintConfig", IDName: "print-config", Doc: "A PrintConfig node controls the output of Fprint.", Fields: []types.Field{{Name: "Mode"}, {Name: "Tabwidth"}, {Name: "Indent"}, {Name: "ExcludeFunctions"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.PrintConfig", IDName: "print-config", Doc: "A PrintConfig node controls the output of Fprint.", Fields: []types.Field{{Name: "Mode"}, {Name: "Tabwidth"}, {Name: "Indent"}, {Name: "GoToSL"}, {Name: "ExcludeFunctions"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.CommentedNode", IDName: "commented-node", Doc: "A CommentedNode bundles an AST node and corresponding comments.\nIt may be provided as argument to any of the [Fprint] functions.", Fields: []types.Field{{Name: "Node"}, {Name: "Comments"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Replace", IDName: "replace", Fields: []types.Field{{Name: "From"}, {Name: "To"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Var", IDName: "var", Doc: "Var represents one variable", Fields: []types.Field{{Name: "Name"}, {Name: "Type"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Group", IDName: "group", Doc: "Group represents one variable group.", Fields: []types.Field{{Name: "Vars"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.System", IDName: "system", Doc: "System represents a ComputeSystem, and its variables.", Fields: []types.Field{{Name: "Name"}, {Name: "Groups"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Kernel", IDName: "kernel", Doc: "Kernel represents a kernel function, which is the basis for\neach wgsl generated code file.", Fields: []types.Field{{Name: "Name"}, {Name: "FileLines", Doc: "accumulating lines of code for the wgsl file."}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.File", IDName: "file", Doc: "File has info for a file being processed", Fields: []types.Field{{Name: "Name"}, {Name: "Lines"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current processing state", Fields: []types.Field{{Name: "Config", Doc: "Config options"}, {Name: "Files", Doc: "files with gosl content in current directory"}, {Name: "Imports"}, {Name: "Kernels"}, {Name: "Systems"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude."}}}) - var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.formatDocComment", Doc: "formatDocComment reformats the doc comment list,\nreturning the canonical formatting.", Args: []string{"list"}, Returns: []string{"Comment"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isDirective", Doc: "isDirective reports whether c is a comment directive.\nSee go.dev/issue/37974.\nThis code is also in go/ast.", Args: []string{"c"}, Returns: []string{"bool"}}) @@ -52,22 +54,32 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.al var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.Run", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.wgslFile", Doc: "wgslFile returns the file with a \".wgsl\" extension", Args: []string{"fn"}, Returns: []string{"string"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.bareFile", Doc: "bareFile returns the file with no extention", Args: []string{"fn"}, Returns: []string{"string"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.ReadFileLines", Args: []string{"fn"}, Returns: []string{"[][]byte", "error"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.HasGoslTag", Doc: "HasGoslTag returns true if given file has a //gosl: tag", Args: []string{"lines"}, Returns: []string{"bool"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.WriteFileLines", Args: []string{"fn", "lines"}, Returns: []string{"error"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.IsGoFile", Args: []string{"f"}, Returns: []string{"bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.IsWGSLFile", Args: []string{"f"}, Returns: []string{"bool"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.IsSPVFile", Args: []string{"f"}, Returns: []string{"bool"}}) - var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.RemoveGenFiles", Doc: "RemoveGenFiles removes .go, .wgsl, .spv files in shader generated dir", Args: []string{"dir"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.CopyFile", Args: []string{"src", "dst"}, Returns: []string{"[][]byte", "error"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.appendLines", Doc: "appendLines is like append(x, y...)\nbut it avoids creating doubled blank lines,\nwhich would not be gofmt-standard output.\nIt assumes that only whole blocks of lines are being appended,\nnot line fragments.", Args: []string{"x", "y"}, Returns: []string{"[]byte"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isNL", Args: []string{"b"}, Returns: []string{"bool"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.NewSystem", Args: []string{"name"}, Returns: []string{"System"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.hasDirective", Doc: "gosl: hasDirective returns whether directive(s) contains string", Args: []string{"dirs", "dir"}, Returns: []string{"bool"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.directiveAfter", Doc: "gosl: directiveAfter returns the directive after given leading text,\nand a bool indicating if the string was found.", Args: []string{"dirs", "dir"}, Returns: []string{"string", "bool"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.combinesWithName", Doc: "combinesWithName reports whether a name followed by the expression x\nsyntactically combines to another valid (value) expression. For instance\nusing *T for x, \"name *T\" syntactically appears as the expression x*T.\nOn the other hand, using P|Q or *P|~Q for x, \"name P|Q\" or name *P|~Q\"\ncannot be combined into a valid (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isTypeElem", Doc: "isTypeElem reports whether x is a (possibly parenthesized) type element expression.\nThe result is false if x could be a type element OR an ordinary (value) expression.", Args: []string{"x"}, Returns: []string{"bool"}}) @@ -92,6 +104,8 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.ge var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getStructType", Args: []string{"typ"}, Returns: []string{"Struct", "error"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getNamedType", Args: []string{"typ"}, Returns: []string{"Named", "error"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isTypeName", Args: []string{"x"}, Returns: []string{"bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.stripParens", Args: []string{"x"}, Returns: []string{"Expr"}}) @@ -124,13 +138,9 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.ne var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.Fprint", Doc: "Fprint \"pretty-prints\" an AST node to output.\nIt calls [PrintConfig.Fprint] with default settings.\nNote that gofmt uses tabs for indentation but spaces for alignment;\nuse format.Node (package go/format) for output that matches gofmt.", Args: []string{"output", "pkg", "node"}, Returns: []string{"error"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.wgslFile", Doc: "wgslFile returns the file with a \".wgsl\" extension", Args: []string{"fn"}, Returns: []string{"string"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.bareFile", Doc: "bareFile returns the file with no extention", Args: []string{"fn"}, Returns: []string{"string"}}) - var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.MoveLines", Doc: "MoveLines moves the st,ed region to 'to' line", Args: []string{"lines", "to", "st", "ed"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlEdits", Doc: "SlEdits performs post-generation edits for wgsl\n* moves wgsl segments around, e.g., methods\ninto their proper classes\n* fixes printf, slice other common code\nreturns true if a slrand. or sltype. prefix was found,\ndriveing copying of those files.", Args: []string{"src"}, Returns: []string{"[]byte", "bool", "bool"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlEdits", Doc: "SlEdits performs post-generation edits for wgsl,\nreplacing type names, slbool, function calls, etc.\nreturns true if a slrand. or sltype. prefix was found,\ndriveing copying of those files.", Args: []string{"src"}, Returns: []string{"lines", "hasSlrand", "hasSltype"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.MathReplaceAll", Args: []string{"mat", "ln"}, Returns: []string{"[]byte"}}) From 90c1ff5e92e756c0f1a7914a745eced66cd2a1ce Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 16:11:02 -0700 Subject: [PATCH 206/311] updated docs for gosl and GPU --- goal/GPU.md | 65 ++---------- goal/README.md | 10 +- goal/gosl/README.md | 147 +++++++++++++++++++++++----- goal/gosl/examples/basic/compute.go | 2 +- 4 files changed, 134 insertions(+), 90 deletions(-) diff --git a/goal/GPU.md b/goal/GPU.md index aaf035ca63..bdbb3e734f 100644 --- a/goal/GPU.md +++ b/goal/GPU.md @@ -32,31 +32,20 @@ In addition to the critical differences between Go and C++ as languages, Goal ta The bottom line is that the fantasy of being able to write CPU-native code and have it magically "just work" on the GPU with high levels of efficiency is just that: a fantasy. The reality is that code must be specifically structured and organized to work efficiently on the GPU. Goal just makes this process relatively clean and efficient and easy to read, with a minimum of extra boilerplate. The resulting code should be easily understood by anyone familiar with the Go language, even if that isn't the way you would have written it in the first place. The reward is that you can get highly efficient results with significant GPU-accelerated speedups that works on _any platform_, including the web and mobile phones, all with a single easy-to-read codebase. -## Gosl: go shader language - -The [gosl](gosl) (_Go shader language_) package within Goal does the heavy lifting of translating Go code into WGSL shader language code that can run on the WebGPU, and generally manages most of the gpu-specific functionality. It has various important functions defined in the `gosl.` package space, and a number of `//gosl:` comment directives described below, that make everything work. - -Meanwhile, the `goal build` command provides an outer-loop of orchestration and math code transpiling prior to handing off to gosl to run on the relevant files. - -For example, `gosl.UseCPU()` causes subsequent execution to use CPU code, while `gosl.UseGPU()` causes it to use GPU kernels. Other `gosl` calls configure and activate the GPU during an initialization step. - -## Overall Code Organization +# Kernel functions First, we assume the scope is a single Go package that implements a set of computations on some number of associated data representations. The package will likely contain a lot of CPU-only Go code that manages all the surrounding infrastructure for the computations, in terms of creating and configuring the data in memory, visualization, i/o, etc. -The GPU-specific computation is organized into some (hopefully small) number of **kernel** functions, that are called using a special `parallel` version of a `for range` loop: - +The GPU-specific computation is organized into some (hopefully small) number of **kernel** functions, that are conceptually called using a **parallel for loop**, e.g., something like this: ```Go for i := range parallel(data) { Compute(i) } ``` -Where the `parallel` function is a special Goal keyword that triggers GPU vs. CPU execution, depending on prior configuration setup, and `MyCompute` is a kernel function. The `i` index effectively iterates over the range of the values of the `tensor` variable `data` (using specific dimension(s) with optional parameter(s)), with the GPU version launching the kernel on the GPU. - -We assume that multiple kernels will in general be required, and that there is likely to be a significant amount of shared code infrastructure across these kernels. +The `i` index effectively iterates over the range of the values of the `data` variable, with the GPU version launching kernels on the GPU for each different index value. The CPU version actually runs in parallel as well, using goroutines. -> We support multiple CPU-based kernels within a single Go package directory. +We assume that multiple kernels will in general be required, and that there is likely to be a significant amount of shared code infrastructure across these kernels. Thus, the kernel functions are typically relatively short, and call into a large body of code that is likely shared among the different kernel functions. Even though the GPU kernels must each be compiled separately into a single distinct WGSL _shader_ file that is run under WebGPU, they can `import` a shared codebase of files, and thus replicate the same overall shared code structure as the CPU versions. @@ -71,8 +60,6 @@ func Compute(i uint32) { //gosl:kernel } ``` -For CPU (regular Go) mode, the parallel `for range` loop as shown above translates into a `tensor.VectorizeThreaded` call of the named Go function. For GPU mode, it launches the kernel on the GPU. - ## Memory Organization Perhaps the strongest constraints for GPU programming stem from the need to organize and synchronize all the memory buffers holding the data that the GPU kernel operates on. Furthermore, within a GPU kernel, the variables representing this data are _global variables_, which is sensible given the standalone nature of each kernel. @@ -85,38 +72,10 @@ Within the [gpu](../gpu) framework, each `ComputeSystem` defines a specific orga > Kernels and variables both must be defined within a specific system context. -The following comment directive can be used in any kernel file to specify which system it uses, and there is an initial `default` system that is used if none is ever specified. -```Go -//gosl:system -``` - -To define the global variables for each system, use a standard Go `var` block declaration (with optional system name qualifier): -```Go -//gosl:vars [system name] -var ( - // Layer-level parameters - //gosl:group -uniform Params - Layers []LayerParam // note: struct with appropriate memory alignment - - // Path-level parameters - Paths []PathParam - - - // Unit state values - //gosl:group Units - Units tensor.Float32 - - // Synapse weight state values - Weights tensor.Float32 -) -``` - -The `//gosl:vars` directive flags this block of variables as GPU-accessible variables, which will drive the automatic generation of [gpu](../gpu) code to define these variables for the current (named) system, and to declare them in each kernel associated with that system. - -The `//gosl:group` directives specify groups of variables, which generally should have similar memory syncing behavior, as documented in the [gpu](../gpu) system. - ### datafs mapping +TODO: + The grouped global variables can be mapped directly to a corresponding [datafs](../tensor/datafs) directory, which provides direct accessibility to this data within interactive Goal usage. Further, different sets of variable values can be easily managed by saving and loading different such directories. ```Go @@ -127,18 +86,6 @@ The grouped global variables can be mapped directly to a corresponding [datafs]( These and all such `gosl` functions use the current system if none is explicitly specified, which is settable using the `gosl.SetSystem` call. Any given variable can use the `get` or `set` Goal math mode functions directly. -## Memory syncing - -It is up to the programmer to manage the syncing of memory between the CPU and the GPU, using simple `gosl` wrapper functions that manage all the details: - -```Go - gosl.ToGPU(varName...) // sync current contents of given GPU variable(s) from CPU to GPU - - gosl.FromGPU(varName...) // sync back from GPU to CPU -``` - -These are no-ops if operating in CPU mode. - ## Memory access In general, all global GPU variables will be arrays (slices) or tensors, which are exposed to the GPU as an array of floats. diff --git a/goal/README.md b/goal/README.md index def641ee54..1e042ab517 100644 --- a/goal/README.md +++ b/goal/README.md @@ -2,13 +2,13 @@ Goal is an augmented version of the Go language, which combines the best parts of Go, `bash`, and Python, to provide and integrated shell and numerical expression processing experience, which can be combined with the [yaegi](https://github.com/traefik/yaegi) interpreter to provide an interactive "REPL" (read, evaluate, print loop). -Goal transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: +Goal transpiles directly into Go, so it automatically leverages all the great features of Go, and remains fully compatible with it. The augmentation is designed to overcome some of the limitations of Go in specific domains: * Shell scripting, where you want to be able to directly call other executable programs with arguments, without having to navigate all the complexity of the standard [os.exec](https://pkg.go.dev/os/exec) package. -* Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about type conversions and need extended indexing and slicing expressions. Python is the dominant language here precisely because it lets you ignore type information and write such expressions. +* Numerical / math / data processing, where you want to be able to write simple mathematical expressions operating on vectors, matricies and other more powerful data types, without having to constantly worry about numerical type conversions, and advanced n-dimensional indexing and slicing expressions are critical. Python is the dominant language here precisely because it lets you ignore type information and write such expressions, using operator overloading. -* GPU-based parallel computation, which can greatly speed up some types of parallelizable computations significantly by effectively running many instances of the same code in parallel across a large array of data. Goal (and [gosl](gosl)) allow you to run the same Go-based code on a GPU or CPU (using parallel goroutines). See the [GPU](GPU.md) docs for full info. +* GPU-based parallel computation, which can greatly speed up some types of parallelizable computations by effectively running many instances of the same code in parallel across a large array of data. The [gosl](gosl) package (automatically run in `goal build` mode) allows you to run the same Go-based code on a GPU or CPU (using parallel goroutines). See the [GPU](GPU.md) docs for an overview and comparison to other approaches to GPU computation. The main goal of Goal is to achieve a "best of both worlds" solution that retains all the type safety and explicitness of Go for all the surrounding control flow and large-scale application logic, while also allowing for a more relaxed syntax in specific, well-defined domains where the Go language has been a barrier. Thus, unlike Python where there are various weak attempts to try to encourage better coding habits, Goal retains in its Go foundation a fundamentally scalable, "industrial strength" language that has already proven its worth in countless real-world applications. @@ -39,11 +39,11 @@ for _, x := range #[1,2,3]# { } ``` -In general, the math mode syntax in Goal is designed to be as compatible with Python NumPy / scipy syntax as possible, while also adding a few Go-specific additions as well -- see [Math mode](#math-mode) for details. All elements of a Goal math expression are [tensors](../tensor), which can represent everything from a scalar to an n-dimenstional tensor. These are called an "ndarray" in NumPy terms. +In general, the math mode syntax in Goal is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well: see the [Math mode](#math-mode) section for details. All elements of a Goal math expression are [tensors](../tensor), which can represent everything from a scalar to an n-dimenstional tensor. These are called "ndarray" in NumPy terms. The rationale and mnemonics for using `$` and `#` are as follows: -* These are two of the three symbols that are not part of standard Go syntax (`@` being the other). +* These are two of the three common ASCII keyboard symbols that are not part of standard Go syntax (`@` being the other). * `$` can be thought of as "S" in _S_hell, and is often used for a `bash` prompt, and many bash examples use it as a prefix. Furthermore, in bash, `$( )` is used to wrap shell expressions. diff --git a/goal/gosl/README.md b/goal/gosl/README.md index 3dc0b3695b..3a2b5585f6 100644 --- a/goal/gosl/README.md +++ b/goal/gosl/README.md @@ -2,55 +2,152 @@ `gosl` implements _Go as a shader language_ for GPU compute shaders (using [WebGPU](https://www.w3.org/TR/webgpu/)), **enabling standard Go code to run on the GPU**. -`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go langauge. See the [GPU](../GPU.md) documentation for an overview. The `goal` command processes more compact math-mode expressions that +`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader, using the [gpu](../../gpu) GPU compute shader system. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go language. See the [GPU](../GPU.md) documentation for an overview of issues in GPU computation. The `goal build` command automatically runs `gosl`, which has no effect if there are no `//gosl:` tags in a given directory. -The relevant subsets of Go code are specifically marked using `//gosl:` comment directives, and this code must only use basic expressions and concrete types that will compile correctly in a shader (see [Restrictions](#restrictions) below). Method functions and pass-by-reference pointer arguments to `struct` types are supported and incur no additional compute cost due to inlining (see notes below for more detail). +The relevant regions of Go code to be run on the GPU are tagged using the `//gosl:start` and `//gosl:end` comment directives, and this code must only use basic expressions and concrete types that will compile correctly in a GPU shader (see [Restrictions](#restrictions) below). Method functions and pass-by-reference pointer arguments to `struct` types are supported and incur no additional compute cost due to inlining (see notes below for more detail). -See [examples/basic](examples/basic) and [rand](examples/rand) for examples, using the [gpu](../../gpu) GPU compute shader system. It is also possible in principle to use gosl to generate shader files for any other WebGPU application, but this has not been tested. +See [examples/basic](examples/basic) and [rand](examples/rand) for complete working examples. -You must also install `goimports` which is used on the extracted subset of Go code, to get the imports right: +Although `gosl` is typically run via the `goal build` command, you can also run `gosl` directly. Here's how to install the standalone `gosl` command: ```bash -$ go install golang.org/x/tools/cmd/goimports@latest +$ go install cogentcore.org/core/gpu/gosl@latest ``` -To install the `gosl` command, do: -```bash -$ go install cogentcore.org/core/gpu/gosl@latest +# Usage + +There are two critical elements for GPU-enabled code: + +1. One or more [Kernel](#kernels) compute functions that take an _index_ argument and perform computations for that specific index of data, _in parallel_. On the GPU, each such kernel is implemented by its own separate compute shader code, and one of the main functions of `gosl` is to generate this code from the Go sources, in the automatically-created `shaders/` directory. + +2. [Global variables](#global-variables) on which the kernel functions _exclusively_ operate: all relevant data must be specifically copied from the CPU to the GPU and back. As explained in the [GPU](../GPU.md) docs, each GPU compute shader is effectively a _standalone_ program operating on these global variables. To replicate this environment on the CPU, so the code is transferrable, we need to make these variables global in the CPU (Go) environment as well. + +`gosl` generates a file named `gosl.go` in your package directory that initializes the GPU with all of the global variables, and functions for running the kernels and syncing the gobal variable data back and forth between the CPu and GPU. + +## Kernels + +Each distinct compute kernel must be tagged with a `//gosl:kernel` comment directive, as in this example: +```Go +// Compute does the main computation. +func Compute(i uint32) { //gosl:kernel + Params[0].IntegFromRaw(int(i)) +} ``` -In your Go code, use these comment directives: +The kernel functions receive a `uint32` index argument, and use this to index into the global variables containing the relevant data. Typically the kernel code itself just calls other relevant function(s) using the index, as in the above example. Critically, _all_ of the data that a kernel function ultimately depends on must be contained with the global variables, and these variables must have been sync'd up to the GPU from the CPU prior to running the kernel (more on this below). + +In the CPU mode, the kernel is effectively run in a `for` loop like this: +```Go + for i := range n { + Compute(uint32(i)) + } ``` -//gosl:start +A parallel goroutine-based mechanism is actually used, but conceptually this is what it does, on both the CPU and the GPU: **GPU computation is effectively just a parallel for loop**. -< Go code to be translated > +## Global variables -//gosl:end +The global variables on which the kernels operate are declared in the usual Go manner, as a single `var` block, which is marked at the top using the `//gosl:vars` comment directive: + +```Go +//gosl:vars +var ( + // Params are the parameters for the computation. + //gosl:read-only + Params []ParamStruct + + // Data is the data on which the computation operates. + // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + //gosl:dims 2 + Data tensor.Float32 +) +``` + +All such variables must be either: +1. A `slice` of basic GPU-compatible data values such as `float32`, or GPU-alignment compatible `struct` types, such as `ParamStruct` in the above example. +2. A `tensor` of a GPU-compatible data type (`float32`, `uint32`, or `int32`), with the number of dimensions indicated by the `//gosl:dims ` tag as shown above. + +### Tensor data + +On the GPU, the tensor data is represented using a simple flat array of the basic data type, with the _strides_ for each dimension encoded in the first `n` elements. `gosl` automatically generates the appropriate indexing code using these strides (which is why the number of dimensions is needed). + +The tensor must be initialized using this special [sltensor](sltensor) function to encode the stride values in the "header" section of the tensor data: +```Go + sltensor.SetShapeSizes(&Data, n, 3) // critically, makes GPU compatible Header with strides +``` + +### Systems and Groups + +Each kernel belongs to a `gpu.ComputeSystem`, and each such system has one specific configuration of memory variables. In general, it is best to use a single set of global variables, and perform as much of the computation as possible on this set of variables, to minimize the number of memory transfers. However, if necessary, multiple systems can be defined, using an optional additional system name argument for the `args` and `kernel` tags. + +In addition, the vars can be organized into _groups_, which generally should have similar memory syncing behavior, as documented in the [gpu](../gpu) system. + +Here's an example with multiple groups: +```Go +//gosl:vars [system name] +var ( + // Layer-level parameters + //gosl:group -uniform Params + Layers []LayerParam // note: struct with appropriate memory alignment + + // Path-level parameters + Paths []PathParam + + + // Unit state values + //gosl:group Units + Units tensor.Float32 + + // Synapse weight state values + Weights tensor.Float32 +) ``` -to bracket code to be processed for GPU. The resulting converted code is copied into a `shaders` subdirectory created under the current directory where the `gosl` command is run, using the filenames specified in the comment directives, or the name of the current file if not specified. +## Memory syncing + +Each global variable gets an automatically-generated `*Var` enum (e.g., `DataVar` for global variable named `Data`), that used for the memory syncing functions, to make it easy to specify any number of such variables to sync, which is by far the most efficient. All of this is in the generated `gosl.go` file. For example: + +```Go + ToGPU(ParamsVar, DataVar) +``` -Use the `//gosl:import package` directive to include files from other packages, similar to the standard Go import directive. It is assumed that many other Go imports are not GPU relevant, so this separate directive is required. +Specifies that the current contents of `Params` and `Data` are to be copied up to the GPU, which is guaranteed to complete by the time the next kernel run starts, within a given system. -Each such filename should correspond to a complete shader program (i.e., a "kernel"), or a file that can be included into other shader programs. Code is appended to the target file names in the order of the source .go files on the command line, so multiple .go files can be combined into one resulting WGSL file. +## Kernel running -WGSL specific code, e.g., for the `main` compute function or to specify `#include` files, can be included either by specifying files with a `.wgsl` extension as arguments to the `gosl` command, or by using a `//gosl:wgsl` comment directive as follows: +As with memory transfers, it is much more efficient to run multiple kernels in sequence, all operating on the current data variables, followed by a single sync of the updated global variable data that has been computed. Thus, there are separate functions for specifying the kernels to run, followed by a single "Done" function that actually submits the entire batch of kernels, along with memory sync commands to get the data back from the GPU. For example: + +```Go + RunCompute1(n) + RunCompute2(n) + ... + RunDone(Data1Var, Data2Var) // launch all kernels and get data back to given vars +``` + +For CPU mode, `RunDone` is a no-op, and it just runs each kernel during each `Run` command. + +It is absolutely essential to understand that _all data must already be on the GPU_ at the start of the first Run command, and that any CPU-based computation between these calls is completely irrelevant for the GPU. Thus, it typically makes sense to just have a sequence of Run commands grouped together into a logical unit, with the relevant `ToGPU` calls at the start and the final `RunDone` grabs everything of relevance back from the GPU. + +## GPU relevant code taggng + +In a large GPU-based application, you should organize your code as you normally would in any standard Go application, distributing it across different files and packages. The GPU-relevant parts of each of those files can be tagged with the gosl tags: ``` -//gosl:wgsl +//gosl:start -// +< Go code to be translated > -//gosl:end +//gosl:end ``` -where the WGSL shader code is commented out in the .go file -- it will be copied into the target filename and uncommented. The WGSL code can be surrounded by `/*` `*/` comment blocks (each on a separate line) for multi-line code (though using a separate `.wgsl` file is generally preferable in this case). +to make this code available to all of the shaders that are generated. + +Use the `//gosl:import "package/path"` directive to import GPU-relevant code from other packages, similar to the standard Go import directive. It is assumed that many other Go imports are not GPU relevant, so this separate directive is required. -For `.wgsl` files, their filename is used to determine the `shaders` destination file name, and they are automatically appended to the end of the corresponding `.wgsl` file generated from the `Go` files -- this is where the `main` function and associated global variables should be specified. +`gosl` automatically includes _all_ tagged code with each shader, and lets the compiler sort out the subset of code that is actually relevant to each specific kernel. Ideally, we could do this with a pre-processing step that performs dead code elimination, but that does not appear to be functional yet. **IMPORTANT:** all `.go` and `.wgsl` files are removed from the `shaders` directory prior to processing to ensure everything there is current -- always specify a different source location for any custom `.wgsl` files that are included. -# Usage +# Command line usage ``` -gosl [flags] [path ...] +gosl [flags] ``` The flags are: @@ -58,14 +155,14 @@ The flags are: -debug enable debugging messages while running -exclude string - comma-separated list of names of functions to exclude from exporting to HLSL (default "Update,Defaults") + comma-separated list of names of functions to exclude from exporting to WGSL (default "Update,Defaults") -keep keep temporary converted versions of the source files, for debugging -out string output directory for shader code, relative to where gosl is invoked -- must not be an empty string (default "shaders") ``` -`gosl` path args can include filenames, directory names, or Go package paths (e.g., `cogentcore.org/core/math32/fastexp.go` loads just that file from the given package) -- files without any `//gosl:` comment directives will be skipped up front before any expensive processing, so it is not a problem to specify entire directories where only some files are relevant. Also, you can specify a particular file from a directory, then the entire directory, to ensure that a particular file from that directory appears first -- otherwise alphabetical order is used. `gosl` ensures that only one copy of each file is included. +`gosl` always operates on the current directory, looking for all files with `//gosl:` tags, and accumulating all the `import` files that they include, etc. Any `struct` types encountered will be checked for 16-byte alignment of sub-types and overall sizes as an even multiple of 16 bytes (4 `float32` or `int32` values), which is the alignment used in WGSL and glsl shader languages, and the underlying GPU hardware presumably. Look for error messages on the output from the gosl run. This ensures that direct byte-wise copies of data between CPU and GPU will be successful. The fact that `gosl` operates directly on the original CPU-side Go code uniquely enables it to perform these alignment checks, which are otherwise a major source of difficult-to-diagnose bugs. diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index c7f71b5a5e..eb6f6e3dc7 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -51,7 +51,7 @@ func (ps *ParamStruct) IntegFromRaw(idx int) { Data.Set(math32.FastExp(-integ), idx, Exp) } -// Compute does the main computation +// Compute does the main computation. func Compute(i uint32) { //gosl:kernel Params[0].IntegFromRaw(int(i)) } From b1eb7d26be5b2c07aba22b5fd78674d802dfee16 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 6 Oct 2024 17:25:53 -0700 Subject: [PATCH 207/311] tensor n-dim indexing in goal mode; translates to Value, Set* functions --- goal/transpile/transpile.go | 85 +++++++++++++++++++++++++++++++- goal/transpile/transpile_test.go | 9 +++- tensor/number.go | 13 +++++ 3 files changed, 104 insertions(+), 3 deletions(-) diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 1bb6f12d19..c5d502baf0 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -166,7 +166,8 @@ func (st *State) TranspileGo(toks Tokens, code string) Tokens { gtoks := make(Tokens, 0, len(toks)) // return tokens for i := 0; i < n; i++ { tok := toks[i] - if tok.Tok == token.ILLEGAL { + switch { + case tok.Tok == token.ILLEGAL: et := toks[i:].ModeEnd() if et > 0 { if tok.Str == "#" { @@ -179,7 +180,20 @@ func (st *State) TranspileGo(toks Tokens, code string) Tokens { } else { gtoks = append(gtoks, tok) } - } else { + case tok.Tok == token.LBRACK && i > 0 && toks[i-1].Tok == token.IDENT: // index expr + ixtoks := toks[i:] + rm := ixtoks.RightMatching() + if rm < 3 { + gtoks = append(gtoks, tok) + continue + } + idx := st.TranspileGoNDimIndex(toks, >oks, i-1, rm+i) + if idx > 0 { + i = idx + } else { + gtoks = append(gtoks, tok) + } + default: gtoks = append(gtoks, tok) } } @@ -298,3 +312,70 @@ func (st *State) TranspileExec(ewords []string, output bool) Tokens { endExec() return etoks } + +// TranspileGoNDimIndex processes an ident[*] sequence of tokens, +// translating it into a corresponding tensor Value or Set expression, +// if it is a multi-dimensional indexing expression which is not valid in Go, +// to support simple n-dimensional tensor indexing in Go (not math) mode. +// Gets the current sequence of toks tokens, where the ident starts at idIdx +// and the ] is at rbIdx. It puts the results in gtoks generated tokens. +// Returns a positive index to resume processing at, if it is actually an +// n-dimensional expr, and -1 if not, in which case the normal process resumes. +func (st *State) TranspileGoNDimIndex(toks Tokens, gtoks *Tokens, idIdx, rbIdx int) int { + nc := 0 + for i := idIdx + 2; i < rbIdx; i++ { + tk := toks[i] + if tk.Tok == token.COMMA { + nc++ + break + } + if tk.Tok == token.LPAREN || tk.Tok == token.LBRACE || tk.Tok == token.LBRACK { + rp := toks[i:rbIdx].RightMatching() + if rp > 0 { + i += rp + } + } + } + if nc == 0 { // not multidim + return -1 + } + // now we need to determine if it is a Set based on what happens after rb + isSet := false + stok := token.ILLEGAL + n := len(toks) + if n-rbIdx > 1 { + ntk := toks[rbIdx+1].Tok + if ntk == token.ASSIGN || (ntk >= token.ADD_ASSIGN && ntk <= token.QUO_ASSIGN) { + isSet = true + stok = ntk + } + } + fun := "Value" + if isSet { + fun = "Set" + switch stok { + case token.ADD_ASSIGN: + fun += "Add" + case token.SUB_ASSIGN: + fun += "Sub" + case token.MUL_ASSIGN: + fun += "Mul" + case token.QUO_ASSIGN: + fun += "Div" + } + } + gtoks.Add(token.PERIOD) + gtoks.Add(token.IDENT, fun) + gtoks.Add(token.LPAREN) + if isSet { + gtoks.AddTokens(toks[rbIdx+2:]...) + gtoks.Add(token.COMMA) + } + gtoks.AddTokens(toks[idIdx+2 : rbIdx]...) + gtoks.Add(token.RPAREN) + if isSet { + return n + } else { + return rbIdx + } +} diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 99bad2a1eb..d9151824d8 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -170,6 +170,13 @@ func TestTranspile(t *testing.T) { {"stru.ctr++", "stru.ctr++"}, {"meta += ln", "meta += ln"}, {"var data map[string]any", "var data map[string]any"}, + // non-math-mode tensor indexing: + {"x = a[1,f(2,3)]", `x = a.Value(1, f(2, 3))`}, + {"x = a[1]", `x = a[1]`}, + {"x = a[f(2,3)]", `x = a[f(2, 3)]`}, + {"x = a[1,2] = 55", `x = a.Set(55, 1, 2)`}, + {"x = a[1,2] += f(2,55)", `x = a.SetAdd(f(2, 55), 1, 2)`}, + {"x = a[1,2] *= f(2,55)", `x = a.SetMul(f(2, 55), 1, 2)`}, } st := NewState() @@ -204,7 +211,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"# a := [[1,2], [3,4]]", `a := tensor.Tensor(tensor.Reshape(tensor.NewIntFromValues([]int { 1, 2, 3, 4 } ...), 2, 2))`}, + {"x = a[1,2]", `x = a.Value(1, 2)`}, } st := NewState() st.MathRecord = false diff --git a/tensor/number.go b/tensor/number.go index 965e3ccc3e..0edd5250c7 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -110,6 +110,19 @@ func (tsr *Number[T]) IsString() bool { return false } func (tsr *Number[T]) AsValues() Values { return tsr } +func (tsr *Number[T]) SetAdd(val T, i ...int) { + tsr.Values[tsr.shape.IndexTo1D(i...)] += val +} +func (tsr *Number[T]) SetSub(val T, i ...int) { + tsr.Values[tsr.shape.IndexTo1D(i...)] -= val +} +func (tsr *Number[T]) SetMul(val T, i ...int) { + tsr.Values[tsr.shape.IndexTo1D(i...)] *= val +} +func (tsr *Number[T]) SetDiv(val T, i ...int) { + tsr.Values[tsr.shape.IndexTo1D(i...)] /= val +} + ///////////////////// Strings func (tsr *Number[T]) SetString(val string, i ...int) { From e176319dac92df15820395a422d24fdfdaff773c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 02:19:42 -0700 Subject: [PATCH 208/311] split out yaegicore into nogui and gui components, goal only includes nogui; fixes for tensor, table io to not use core.Filename; finally don't have gui mode activated for goal command. --- base/fsx/fsx.go | 4 ++++ goal/interpreter/interpreter.go | 5 ++--- goal/interpreter/typegen.go | 2 +- goal/transpile/addfuncs.go | 4 ++-- plot/plotcore/ploteditor.go | 5 +++-- tensor/databrowser/filetree.go | 4 ++-- tensor/io.go | 6 +++--- tensor/table/io.go | 6 +++--- tensor/tensorcore/tensoreditor.go | 5 +++-- .../cogentcore_org-core-base-errors.go | 2 +- .../cogentcore_org-core-base-fileinfo.go | 2 +- .../cogentcore_org-core-base-fsx.go | 5 ++++- .../cogentcore_org-core-base-labels.go | 2 +- .../cogentcore_org-core-base-num.go | 2 +- .../cogentcore_org-core-base-reflectx.go | 2 +- .../cogentcore_org-core-goal-goalib.go | 2 +- .../{symbols => nogui}/cogentcore_org-core-math32.go | 2 +- .../cogentcore_org-core-tensor-datafs.go | 2 +- .../cogentcore_org-core-tensor-matrix.go | 5 ++++- .../cogentcore_org-core-tensor-stats-metric.go | 2 +- .../cogentcore_org-core-tensor-stats-stats.go | 2 +- .../cogentcore_org-core-tensor-table.go | 2 +- .../cogentcore_org-core-tensor-tmath.go | 2 +- .../cogentcore_org-core-tensor-vector.go | 2 +- .../{symbols => nogui}/cogentcore_org-core-tensor.go | 2 +- yaegicore/{symbols => nogui}/fmt.go | 2 +- yaegicore/{symbols => nogui}/log-slog.go | 2 +- yaegicore/nogui/make | 12 ++++++++++++ yaegicore/{symbols => nogui}/math.go | 2 +- yaegicore/{symbols => nogui}/reflect.go | 2 +- yaegicore/{symbols => nogui}/strconv.go | 2 +- yaegicore/{symbols => nogui}/strings.go | 2 +- yaegicore/nogui/symbols.go | 12 ++++++++++++ yaegicore/{symbols => nogui}/time.go | 2 +- yaegicore/symbols/make | 7 ++++--- yaegicore/yaegicore.go | 2 ++ 36 files changed, 82 insertions(+), 44 deletions(-) rename yaegicore/{symbols => nogui}/cogentcore_org-core-base-errors.go (98%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-base-fileinfo.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-base-fsx.go (92%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-base-labels.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-base-num.go (92%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-base-reflectx.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-goal-goalib.go (97%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-math32.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-datafs.go (98%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-matrix.go (90%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-stats-metric.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-stats-stats.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-table.go (98%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-tmath.go (99%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor-vector.go (97%) rename yaegicore/{symbols => nogui}/cogentcore_org-core-tensor.go (99%) rename yaegicore/{symbols => nogui}/fmt.go (99%) rename yaegicore/{symbols => nogui}/log-slog.go (99%) create mode 100755 yaegicore/nogui/make rename yaegicore/{symbols => nogui}/math.go (99%) rename yaegicore/{symbols => nogui}/reflect.go (99%) rename yaegicore/{symbols => nogui}/strconv.go (99%) rename yaegicore/{symbols => nogui}/strings.go (99%) create mode 100644 yaegicore/nogui/symbols.go rename yaegicore/{symbols => nogui}/time.go (99%) diff --git a/base/fsx/fsx.go b/base/fsx/fsx.go index b757ee5f05..9fb0d865fa 100644 --- a/base/fsx/fsx.go +++ b/base/fsx/fsx.go @@ -17,6 +17,10 @@ import ( "time" ) +// Filename is used to open a file picker dialog when used as an argument +// type in a function, or as a field value. +type Filename string + // GoSrcDir tries to locate dir in GOPATH/src/ or GOROOT/src/pkg/ and returns its // full path. GOPATH may contain a list of paths. From Robin Elkind github.com/mewkiz/pkg. func GoSrcDir(dir string) (absDir string, err error) { diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index f234e1fbb9..9cca117257 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -22,7 +22,7 @@ import ( _ "cogentcore.org/core/tensor/stats/metric" _ "cogentcore.org/core/tensor/stats/stats" _ "cogentcore.org/core/tensor/tmath" - "cogentcore.org/core/yaegicore/symbols" + "cogentcore.org/core/yaegicore/nogui" "github.com/cogentcore/yaegi/interp" "github.com/cogentcore/yaegi/stdlib" "github.com/ergochat/readline" @@ -67,8 +67,7 @@ func NewInterpreter(options interp.Options) *Interpreter { options.Stderr = in.Goal.StdIOWrappers.Err options.Stdin = in.Goal.StdIOWrappers.In in.Interp = interp.New(options) - // errors.Log(in.Interp.Use(stdlib.Symbols)) // note: yaegicore/symbols gets just a few key things - errors.Log(in.Interp.Use(symbols.Symbols)) + errors.Log(in.Interp.Use(nogui.Symbols)) in.ImportGoal() go in.MonitorSignals() return in diff --git a/goal/interpreter/typegen.go b/goal/interpreter/typegen.go index c0a0961344..b8836bd6be 100644 --- a/goal/interpreter/typegen.go +++ b/goal/interpreter/typegen.go @@ -6,7 +6,7 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/interpreter.Config", IDName: "config", Doc: "Config is the configuration information for the goal cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "Input", Doc: "Input is the input file to run/compile.\nIf this is provided as the first argument,\nthen the program will exit after running,\nunless the Interactive mode is flagged."}, {Name: "Expr", Doc: "Expr is an optional expression to evaluate, which can be used\nin addition to the Input file to run, to execute commands\ndefined within that file for example, or as a command to run\nprior to starting interactive mode if no Input is specified."}, {Name: "Args", Doc: "Args is an optional list of arguments to pass in the run command.\nThese arguments will be turned into an \"args\" local variable in the goal.\nThese are automatically processed from any leftover arguments passed, so\nyou should not need to specify this flag manually."}, {Name: "Interactive", Doc: "Interactive runs the interactive command line after processing any input file.\nInteractive mode is the default mode for the run command unless an input file\nis specified."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/interpreter.Config", IDName: "config", Doc: "Config is the configuration information for the goal cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "Input", Doc: "Input is the input file to run/compile.\nIf this is provided as the first argument,\nthen the program will exit after running,\nunless the Interactive mode is flagged."}, {Name: "Expr", Doc: "Expr is an optional expression to evaluate, which can be used\nin addition to the Input file to run, to execute commands\ndefined within that file for example, or as a command to run\nprior to starting interactive mode if no Input is specified."}, {Name: "Args", Doc: "Args is an optional list of arguments to pass in the run command.\nThese arguments will be turned into an \"args\" local variable in the goal.\nThese are automatically processed from any leftover arguments passed, so\nyou should not need to specify this flag manually."}, {Name: "Interactive", Doc: "Interactive runs the interactive command line after processing any input file.\nInteractive mode is the default mode for the run command unless an input file\nis specified."}, {Name: "InteractiveFunc", Doc: "InteractiveFunc is the function to run in interactive mode.\nset it to your own function as needed."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/interpreter.Interpreter", IDName: "interpreter", Doc: "Interpreter represents one running shell context", Fields: []types.Field{{Name: "Goal", Doc: "the goal shell"}, {Name: "HistFile", Doc: "HistFile is the name of the history file to open / save.\nDefaults to ~/.goal-history for the default goal shell.\nUpdate this prior to running Config() to take effect."}, {Name: "Interp", Doc: "the yaegi interpreter"}}}) diff --git a/goal/transpile/addfuncs.go b/goal/transpile/addfuncs.go index ef64a0d49b..a608fbba76 100644 --- a/goal/transpile/addfuncs.go +++ b/goal/transpile/addfuncs.go @@ -10,7 +10,7 @@ import ( "strings" "cogentcore.org/core/tensor" - "cogentcore.org/core/yaegicore/symbols" + "cogentcore.org/core/yaegicore/nogui" ) func init() { @@ -22,7 +22,7 @@ func init() { // properly convert symbols to either tensors or basic literals, // depending on the arg types for the current function. func AddYaegiTensorFuncs() { - for pth, symap := range symbols.Symbols { + for pth, symap := range nogui.Symbols { if !strings.Contains(pth, "/core/tensor/") { continue } diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 91a4cd9d6a..e3f44462c6 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -16,6 +16,7 @@ import ( "time" "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/iox/imagex" "cogentcore.org/core/base/metadata" "cogentcore.org/core/colors" @@ -216,7 +217,7 @@ func (pl *PlotEditor) SavePNG(fname core.Filename) { //types:add // SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim) func (pl *PlotEditor) SaveCSV(fname core.Filename, delim tensor.Delims) { //types:add - pl.table.SaveCSV(fname, delim, table.Headers) + pl.table.SaveCSV(fsx.Filename(fname), delim, table.Headers) pl.dataFile = fname } @@ -232,7 +233,7 @@ func (pl *PlotEditor) SaveAll(fname core.Filename) { //types:add // OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim) func (pl *PlotEditor) OpenCSV(filename core.Filename, delim tensor.Delims) { //types:add - pl.table.OpenCSV(filename, delim) + pl.table.OpenCSV(fsx.Filename(filename), delim) pl.dataFile = filename pl.UpdatePlot() } diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index 7c74d78250..ff1b31957b 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -141,7 +141,7 @@ func (fn *FileNode) OpenFile() error { default: dt := table.New() - err := dt.OpenCSV(fn.Filepath, tensor.Tab) // todo: need more flexible data handling mode + err := dt.OpenCSV(fsx.Filename(fn.Filepath), tensor.Tab) // todo: need more flexible data handling mode if err != nil { core.ErrorSnackbar(br, err) } else { @@ -235,7 +235,7 @@ func (fn *FileNode) PlotFile() { // dt = d.AsTable() default: dt = table.New(df) - err := dt.OpenCSV(fn.Filepath, tensor.Tab) // todo: need more flexible data handling mode + err := dt.OpenCSV(fsx.Filename(fn.Filepath), tensor.Tab) // todo: need more flexible data handling mode if err != nil { core.ErrorSnackbar(br, err) dt = nil diff --git a/tensor/io.go b/tensor/io.go index b4d0aba067..dad3d5268b 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -13,9 +13,9 @@ import ( "strconv" "strings" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" - "cogentcore.org/core/core" "cogentcore.org/core/math32" ) @@ -65,7 +65,7 @@ func Precision(md metadata.Data) (int, error) { // (where comma = any delimiter, specified in the delim arg). // Outer-most dims are rows in the file, and inner-most is column -- // Reading just grabs all values and doesn't care about shape. -func SaveCSV(tsr Tensor, filename core.Filename, delim Delims) error { +func SaveCSV(tsr Tensor, filename fsx.Filename, delim Delims) error { fp, err := os.Create(string(filename)) defer fp.Close() if err != nil { @@ -81,7 +81,7 @@ func SaveCSV(tsr Tensor, filename core.Filename, delim Delims) error { // using the Go standard encoding/csv reader conforming // to the official CSV standard. // Reads all values and assigns as many as fit. -func OpenCSV(tsr Tensor, filename core.Filename, delim Delims) error { +func OpenCSV(tsr Tensor, filename fsx.Filename, delim Delims) error { fp, err := os.Open(string(filename)) defer fp.Close() if err != nil { diff --git a/tensor/table/io.go b/tensor/table/io.go index 2de6619889..d74a14e6ba 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -18,7 +18,7 @@ import ( "strings" "cogentcore.org/core/base/errors" - "cogentcore.org/core/core" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/tensor" ) @@ -37,7 +37,7 @@ const ( // and tensor cell geometry of the columns, enabling full reloading // of exactly the same table format and data (recommended). // Otherwise, only the data is written. -func (dt *Table) SaveCSV(filename core.Filename, delim tensor.Delims, headers bool) error { //types:add +func (dt *Table) SaveCSV(filename fsx.Filename, delim tensor.Delims, headers bool) error { //types:add fp, err := os.Create(string(filename)) defer fp.Close() if err != nil { @@ -59,7 +59,7 @@ func (dt *Table) SaveCSV(filename core.Filename, delim tensor.Delims, headers bo // information for tensor type and dimensionality. // If the table DOES have existing columns, then those are used robustly // for whatever information fits from each row of the file. -func (dt *Table) OpenCSV(filename core.Filename, delim tensor.Delims) error { //types:add +func (dt *Table) OpenCSV(filename fsx.Filename, delim tensor.Delims) error { //types:add fp, err := os.Open(string(filename)) if err != nil { return errors.Log(err) diff --git a/tensor/tensorcore/tensoreditor.go b/tensor/tensorcore/tensoreditor.go index aac5b70a15..cf9ce068b9 100644 --- a/tensor/tensorcore/tensoreditor.go +++ b/tensor/tensorcore/tensoreditor.go @@ -12,6 +12,7 @@ import ( "cogentcore.org/core/base/fileinfo" "cogentcore.org/core/base/fileinfo/mimedata" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/core" "cogentcore.org/core/events" "cogentcore.org/core/icons" @@ -375,7 +376,7 @@ func (tb *TensorEditor) SizeFinal() { // Outer-most dims are rows in the file, and inner-most is column -- // Reading just grabs all values and doesn't care about shape. func (tb *TensorEditor) SaveCSV(filename core.Filename) error { //types:add - return tensor.SaveCSV(tb.Tensor, filename, tensor.Tab) + return tensor.SaveCSV(tb.Tensor, fsx.Filename(filename), tensor.Tab) } // OpenTSV reads a tensor from a tab-separated-values (TSV) file. @@ -383,7 +384,7 @@ func (tb *TensorEditor) SaveCSV(filename core.Filename) error { //types:add // to the official CSV standard. // Reads all values and assigns as many as fit. func (tb *TensorEditor) OpenCSV(filename core.Filename) error { //types:add - return tensor.OpenCSV(tb.Tensor, filename, tensor.Tab) + return tensor.OpenCSV(tb.Tensor, fsx.Filename(filename), tensor.Tab) } func (tb *TensorEditor) MakeToolbar(p *tree.Plan) { diff --git a/yaegicore/symbols/cogentcore_org-core-base-errors.go b/yaegicore/nogui/cogentcore_org-core-base-errors.go similarity index 98% rename from yaegicore/symbols/cogentcore_org-core-base-errors.go rename to yaegicore/nogui/cogentcore_org-core-base-errors.go index 6e34726b01..fd0777ae5d 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-errors.go +++ b/yaegicore/nogui/cogentcore_org-core-base-errors.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/base/errors'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/base/errors" diff --git a/yaegicore/symbols/cogentcore_org-core-base-fileinfo.go b/yaegicore/nogui/cogentcore_org-core-base-fileinfo.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-base-fileinfo.go rename to yaegicore/nogui/cogentcore_org-core-base-fileinfo.go index 0f3a5a442f..e55fb93a22 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-fileinfo.go +++ b/yaegicore/nogui/cogentcore_org-core-base-fileinfo.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/base/fileinfo'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/base/fileinfo" diff --git a/yaegicore/symbols/cogentcore_org-core-base-fsx.go b/yaegicore/nogui/cogentcore_org-core-base-fsx.go similarity index 92% rename from yaegicore/symbols/cogentcore_org-core-base-fsx.go rename to yaegicore/nogui/cogentcore_org-core-base-fsx.go index 4083d45fd0..7dde22275d 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-fsx.go +++ b/yaegicore/nogui/cogentcore_org-core-base-fsx.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/base/fsx'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/base/fsx" @@ -25,5 +25,8 @@ func init() { "RelativeFilePath": reflect.ValueOf(fsx.RelativeFilePath), "SplitRootPathFS": reflect.ValueOf(fsx.SplitRootPathFS), "Sub": reflect.ValueOf(fsx.Sub), + + // type definitions + "Filename": reflect.ValueOf((*fsx.Filename)(nil)), } } diff --git a/yaegicore/symbols/cogentcore_org-core-base-labels.go b/yaegicore/nogui/cogentcore_org-core-base-labels.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-base-labels.go rename to yaegicore/nogui/cogentcore_org-core-base-labels.go index 8e95a74de5..b00adf46fe 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-labels.go +++ b/yaegicore/nogui/cogentcore_org-core-base-labels.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/base/labels'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/base/labels" diff --git a/yaegicore/symbols/cogentcore_org-core-base-num.go b/yaegicore/nogui/cogentcore_org-core-base-num.go similarity index 92% rename from yaegicore/symbols/cogentcore_org-core-base-num.go rename to yaegicore/nogui/cogentcore_org-core-base-num.go index 765ec2c6c0..7896ff8d08 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-num.go +++ b/yaegicore/nogui/cogentcore_org-core-base-num.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/base/num'. DO NOT EDIT. -package symbols +package nogui import ( "reflect" diff --git a/yaegicore/symbols/cogentcore_org-core-base-reflectx.go b/yaegicore/nogui/cogentcore_org-core-base-reflectx.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-base-reflectx.go rename to yaegicore/nogui/cogentcore_org-core-base-reflectx.go index 3d9d493c96..c7ea91a582 100644 --- a/yaegicore/symbols/cogentcore_org-core-base-reflectx.go +++ b/yaegicore/nogui/cogentcore_org-core-base-reflectx.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/base/reflectx'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/base/reflectx" diff --git a/yaegicore/symbols/cogentcore_org-core-goal-goalib.go b/yaegicore/nogui/cogentcore_org-core-goal-goalib.go similarity index 97% rename from yaegicore/symbols/cogentcore_org-core-goal-goalib.go rename to yaegicore/nogui/cogentcore_org-core-goal-goalib.go index 7f398c94a1..2333b7c009 100644 --- a/yaegicore/symbols/cogentcore_org-core-goal-goalib.go +++ b/yaegicore/nogui/cogentcore_org-core-goal-goalib.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/goal/goalib'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/goal/goalib" diff --git a/yaegicore/symbols/cogentcore_org-core-math32.go b/yaegicore/nogui/cogentcore_org-core-math32.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-math32.go rename to yaegicore/nogui/cogentcore_org-core-math32.go index 757eaa5d39..93cd2e2bf8 100644 --- a/yaegicore/symbols/cogentcore_org-core-math32.go +++ b/yaegicore/nogui/cogentcore_org-core-math32.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/math32'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/math32" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go b/yaegicore/nogui/cogentcore_org-core-tensor-datafs.go similarity index 98% rename from yaegicore/symbols/cogentcore_org-core-tensor-datafs.go rename to yaegicore/nogui/cogentcore_org-core-tensor-datafs.go index b5be53dfb9..daa001e667 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-datafs.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-datafs.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/datafs'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/datafs" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go b/yaegicore/nogui/cogentcore_org-core-tensor-matrix.go similarity index 90% rename from yaegicore/symbols/cogentcore_org-core-tensor-matrix.go rename to yaegicore/nogui/cogentcore_org-core-tensor-matrix.go index ce4a4583b6..55bc38bdc0 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-matrix.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-matrix.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/matrix'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/matrix" @@ -10,6 +10,7 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/matrix/matrix"] = map[string]reflect.Value{ // function, constant and variable definitions + "CallOut1": reflect.ValueOf(matrix.CallOut1), "CallOut2": reflect.ValueOf(matrix.CallOut2), "CopyFromDense": reflect.ValueOf(matrix.CopyFromDense), "Det": reflect.ValueOf(matrix.Det), @@ -17,6 +18,8 @@ func init() { "DiagonalIndices": reflect.ValueOf(matrix.DiagonalIndices), "DiagonalN": reflect.ValueOf(matrix.DiagonalN), "Identity": reflect.ValueOf(matrix.Identity), + "Inverse": reflect.ValueOf(matrix.Inverse), + "InverseOut": reflect.ValueOf(matrix.InverseOut), "LogDet": reflect.ValueOf(matrix.LogDet), "Mul": reflect.ValueOf(matrix.Mul), "MulOut": reflect.ValueOf(matrix.MulOut), diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go rename to yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go index 1e926da0de..5d7074a9f8 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/stats/metric'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/stats/metric" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go b/yaegicore/nogui/cogentcore_org-core-tensor-stats-stats.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go rename to yaegicore/nogui/cogentcore_org-core-tensor-stats-stats.go index 95b716c7a4..fe09bc122a 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-stats-stats.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-stats-stats.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/stats/stats'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/stats/stats" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-table.go b/yaegicore/nogui/cogentcore_org-core-tensor-table.go similarity index 98% rename from yaegicore/symbols/cogentcore_org-core-tensor-table.go rename to yaegicore/nogui/cogentcore_org-core-tensor-table.go index 566f186d42..9b605ac86b 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-table.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-table.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/table'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/table" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go b/yaegicore/nogui/cogentcore_org-core-tensor-tmath.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-tensor-tmath.go rename to yaegicore/nogui/cogentcore_org-core-tensor-tmath.go index d97bba9505..a6e8242258 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-tmath.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-tmath.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/tmath'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/tmath" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go b/yaegicore/nogui/cogentcore_org-core-tensor-vector.go similarity index 97% rename from yaegicore/symbols/cogentcore_org-core-tensor-vector.go rename to yaegicore/nogui/cogentcore_org-core-tensor-vector.go index a5d83b1fc9..bddb196bf5 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-vector.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-vector.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor/vector'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/tensor/vector" diff --git a/yaegicore/symbols/cogentcore_org-core-tensor.go b/yaegicore/nogui/cogentcore_org-core-tensor.go similarity index 99% rename from yaegicore/symbols/cogentcore_org-core-tensor.go rename to yaegicore/nogui/cogentcore_org-core-tensor.go index 26ce555dce..42fb865722 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor.go @@ -1,6 +1,6 @@ // Code generated by 'yaegi extract cogentcore.org/core/tensor'. DO NOT EDIT. -package symbols +package nogui import ( "cogentcore.org/core/base/metadata" diff --git a/yaegicore/symbols/fmt.go b/yaegicore/nogui/fmt.go similarity index 99% rename from yaegicore/symbols/fmt.go rename to yaegicore/nogui/fmt.go index 4a2e90c6a0..b9f4ee7322 100644 --- a/yaegicore/symbols/fmt.go +++ b/yaegicore/nogui/fmt.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "fmt" diff --git a/yaegicore/symbols/log-slog.go b/yaegicore/nogui/log-slog.go similarity index 99% rename from yaegicore/symbols/log-slog.go rename to yaegicore/nogui/log-slog.go index 267e584759..6c7c118253 100644 --- a/yaegicore/symbols/log-slog.go +++ b/yaegicore/nogui/log-slog.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "context" diff --git a/yaegicore/nogui/make b/yaegicore/nogui/make new file mode 100755 index 0000000000..443900af86 --- /dev/null +++ b/yaegicore/nogui/make @@ -0,0 +1,12 @@ +#!/usr/bin/env cosh + +command extract { + for _, pkg := range args { + yaegi extract {"cogentcore.org/core/"+pkg} + } +} + +yaegi extract fmt strconv strings math time log/slog reflect + +extract math32 tensor tensor/tmath tensor/table tensor/vector tensor/matrix tensor/stats/stats tensor/stats/metric tensor/datafs base/errors base/fsx base/reflectx base/labels base/fileinfo base/num goal/goalib + diff --git a/yaegicore/symbols/math.go b/yaegicore/nogui/math.go similarity index 99% rename from yaegicore/symbols/math.go rename to yaegicore/nogui/math.go index 5b846c5773..33a942261e 100644 --- a/yaegicore/symbols/math.go +++ b/yaegicore/nogui/math.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "go/constant" diff --git a/yaegicore/symbols/reflect.go b/yaegicore/nogui/reflect.go similarity index 99% rename from yaegicore/symbols/reflect.go rename to yaegicore/nogui/reflect.go index d88982d619..d1868b1649 100644 --- a/yaegicore/symbols/reflect.go +++ b/yaegicore/nogui/reflect.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "reflect" diff --git a/yaegicore/symbols/strconv.go b/yaegicore/nogui/strconv.go similarity index 99% rename from yaegicore/symbols/strconv.go rename to yaegicore/nogui/strconv.go index 99a1e3a547..3c1471e8ea 100644 --- a/yaegicore/symbols/strconv.go +++ b/yaegicore/nogui/strconv.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "go/constant" diff --git a/yaegicore/symbols/strings.go b/yaegicore/nogui/strings.go similarity index 99% rename from yaegicore/symbols/strings.go rename to yaegicore/nogui/strings.go index 6d0eba0d58..17c6738310 100644 --- a/yaegicore/symbols/strings.go +++ b/yaegicore/nogui/strings.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "reflect" diff --git a/yaegicore/nogui/symbols.go b/yaegicore/nogui/symbols.go new file mode 100644 index 0000000000..5a47f34e8e --- /dev/null +++ b/yaegicore/nogui/symbols.go @@ -0,0 +1,12 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package symbols contains yaegi symbols for core packages. +package nogui + +//go:generate ./make + +import "reflect" + +var Symbols = map[string]map[string]reflect.Value{} diff --git a/yaegicore/symbols/time.go b/yaegicore/nogui/time.go similarity index 99% rename from yaegicore/symbols/time.go rename to yaegicore/nogui/time.go index 9bba72045c..7c50adc202 100644 --- a/yaegicore/symbols/time.go +++ b/yaegicore/nogui/time.go @@ -3,7 +3,7 @@ //go:build go1.22 // +build go1.22 -package symbols +package nogui import ( "go/constant" diff --git a/yaegicore/symbols/make b/yaegicore/symbols/make index 572582e352..ab362cdac4 100755 --- a/yaegicore/symbols/make +++ b/yaegicore/symbols/make @@ -1,11 +1,12 @@ #!/usr/bin/env cosh -yaegi extract fmt strconv strings math image image/color image/draw time log/slog reflect - command extract { for _, pkg := range args { yaegi extract {"cogentcore.org/core/"+pkg} } } -extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor goal/goalib htmlcore pages paint math32 plot plot/plots plot/plotcore tensor tensor/tmath tensor/table tensor/vector tensor/matrix tensor/stats/stats tensor/stats/metric tensor/datafs tensor/databrowser base/errors base/fsx base/reflectx base/labels base/fileinfo base/num +yaegi extract image image/color image/draw + +extract core icons events styles styles/states styles/abilities styles/units tree keymap colors colors/gradient filetree texteditor htmlcore pages paint plot plot/plots plot/plotcore tensor/databrowser + diff --git a/yaegicore/yaegicore.go b/yaegicore/yaegicore.go index 24178dbba9..154c806d7d 100644 --- a/yaegicore/yaegicore.go +++ b/yaegicore/yaegicore.go @@ -17,6 +17,7 @@ import ( "cogentcore.org/core/events" "cogentcore.org/core/htmlcore" "cogentcore.org/core/texteditor" + "cogentcore.org/core/yaegicore/nogui" "cogentcore.org/core/yaegicore/symbols" "github.com/cogentcore/yaegi/interp" ) @@ -26,6 +27,7 @@ var autoPlanNameCounter uint64 func init() { htmlcore.BindTextEditor = BindTextEditor symbols.Symbols["."] = map[string]reflect.Value{} // make "." available for use + nogui.Symbols["."] = map[string]reflect.Value{} // make "." available for use } // BindTextEditor binds the given text editor to a yaegi interpreter From d520baa6e25a584253dc49b10e53206a8a09f735 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 02:50:43 -0700 Subject: [PATCH 209/311] goal build runs gosl and go build --- goal/gosl/examples/basic/shaders/Compute.wgsl | 2 +- goal/interpreter/config.go | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 333b3a221a..89beed0bf9 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -56,7 +56,7 @@ fn ParamStruct_IntegFromRaw(ps: ptr, idx: i32) { Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Exp))] = FastExp(-integ); } -// Compute does the main computation +// Compute does the main computation. fn Compute(i: u32) { //gosl:kernel var params=Params[0]; ParamStruct_IntegFromRaw(¶ms, i32(i)); } diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go index 563b3bd6f8..2d8f1a3ca4 100644 --- a/goal/interpreter/config.go +++ b/goal/interpreter/config.go @@ -6,13 +6,17 @@ package interpreter import ( "fmt" + "log/slog" "os" "path/filepath" "strings" "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/exec" "cogentcore.org/core/base/fsx" + "cogentcore.org/core/base/logx" "cogentcore.org/core/goal" + "cogentcore.org/core/goal/gosl/gotosl" "github.com/cogentcore/yaegi/interp" ) @@ -100,6 +104,7 @@ func Interactive(c *Config, in *Interpreter) error { // If the file does not already contain a "package" specification, then // "package main; func main()..." wrappers are added, which allows the same // code to be used in interactive and Go compiled modes. +// go build is run after this. func Build(c *Config) error { var fns []string if c.Input != "" { @@ -115,5 +120,26 @@ func Build(c *Config) error { errs = append(errs, err) } } + + verbose := logx.UserLevel <= slog.LevelInfo + + cfg := &gotosl.Config{} + cfg.Debug = verbose + fmt.Println("running gosl...") + err := gotosl.Run(cfg) + if err != nil { + errs = append(errs, err) + } + + fmt.Println("running go build...") + args := []string{"build"} + if verbose { + args = append(args, "-v") + } + err = exec.Major().Run("go", args...) + if err != nil { + errs = append(errs, err) + } + return errors.Join(errs...) } From 293bb2c56897630876537537211b099e9baa77f2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 03:16:33 -0700 Subject: [PATCH 210/311] verbose fixes etc --- goal/gosl/examples/basic/compute.go | 4 ++ goal/gosl/examples/basic/compute.goal | 70 +++++++++++++++++++ goal/gosl/examples/basic/shaders/Compute.wgsl | 6 +- goal/interpreter/config.go | 6 +- 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 goal/gosl/examples/basic/compute.goal diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index eb6f6e3dc7..cb5749fdf4 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -15,11 +15,13 @@ import ( //gosl:vars var ( // Params are the parameters for the computation. + // //gosl:read-only Params []ParamStruct // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + // //gosl:dims 2 Data tensor.Float32 ) @@ -30,6 +32,8 @@ const ( Exp ) +// Code generated by "goal build"; DO NOT EDIT. + // ParamStruct has the test params type ParamStruct struct { diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal new file mode 100644 index 0000000000..178b12048f --- /dev/null +++ b/goal/gosl/examples/basic/compute.goal @@ -0,0 +1,70 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" +) + +//gosl:start +//gosl:import "cogentcore.org/core/math32" + +//gosl:vars +var ( + // Params are the parameters for the computation. + //gosl:read-only + Params []ParamStruct + + // Data is the data on which the computation operates. + // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + //gosl:dims 2 + Data tensor.Float32 +) + +const ( + Raw int = iota + Integ + Exp +) + +// ParamStruct has the test params +type ParamStruct struct { + + // rate constant in msec + Tau float32 + + // 1/Tau + Dt float32 + + pad float32 + pad1 float32 +} + +// IntegFromRaw computes integrated value from current raw value +func (ps *ParamStruct) IntegFromRaw(idx int) { + integ := Data[idx, Integ] + integ += ps.Dt * (Data[idx, Raw] - integ) + {Data[idx, Integ] = integ} + {Data[idx, Exp] = math32.FastExp(-integ)} +} + +// Compute does the main computation. +func Compute(i uint32) { //gosl:kernel + Params[0].IntegFromRaw(int(i)) +} + +//gosl:end + +// note: only core compute code needs to be in shader -- all init is done CPU-side + +func (ps *ParamStruct) Defaults() { + ps.Tau = 5 + ps.Update() +} + +func (ps *ParamStruct) Update() { + ps.Dt = 1.0 / ps.Tau +} diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 89beed0bf9..ba379bf7ae 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -1,7 +1,7 @@ // Code generated by "gosl"; DO NOT EDIT // kernel: Compute -// // Params are the parameters for the computation. +// // Params are the parameters for the computation. // @group(0) @binding(0) var Params: array; @group(0) @binding(1) @@ -24,10 +24,12 @@ fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { //gosl:vars // Params are the parameters for the computation. +// //gosl:read-only // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. +// //gosl:dims 2 const Raw: i32 = 0; @@ -35,6 +37,8 @@ const Integ: i32 = 1; const Exp: i32 = 2; +// Code generated by "goal build"; DO NOT EDIT. + // ParamStruct has the test params struct ParamStruct { diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go index 2d8f1a3ca4..e74afd7b53 100644 --- a/goal/interpreter/config.go +++ b/goal/interpreter/config.go @@ -107,6 +107,7 @@ func Interactive(c *Config, in *Interpreter) error { // go build is run after this. func Build(c *Config) error { var fns []string + verbose := logx.UserLevel <= slog.LevelInfo if c.Input != "" { fns = []string{c.Input} } else { @@ -114,6 +115,9 @@ func Build(c *Config) error { } var errs []error for _, fn := range fns { + if verbose { + fmt.Println(fn) + } ofn := strings.TrimSuffix(fn, filepath.Ext(fn)) + ".go" err := goal.NewGoal().TranspileFile(fn, ofn) if err != nil { @@ -121,8 +125,6 @@ func Build(c *Config) error { } } - verbose := logx.UserLevel <= slog.LevelInfo - cfg := &gotosl.Config{} cfg.Debug = verbose fmt.Println("running gosl...") From 087225cbdbbaaaa1321333a47d06803791351da1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 12:33:32 -0700 Subject: [PATCH 211/311] goal: fix parsing of Go assignment expressions, including now getting chmod +x right. --- goal/README.md | 3 +++ goal/transpile/token.go | 22 ++++++++++++++++++++++ goal/transpile/transpile.go | 3 +-- goal/transpile/transpile_test.go | 10 ++++++---- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/goal/README.md b/goal/README.md index 1e042ab517..00fb78819e 100644 --- a/goal/README.md +++ b/goal/README.md @@ -38,9 +38,12 @@ for _, x := range #[1,2,3]# { fmt.Println(#x^2#) } ``` +Note that you cannot enter math mode directly from shell mode, which is unlikely to be useful anyway (you can wrap in go mode `{ }` if really needed). In general, the math mode syntax in Goal is designed to be as compatible with Python NumPy / SciPy syntax as possible, while also adding a few Go-specific additions as well: see the [Math mode](#math-mode) section for details. All elements of a Goal math expression are [tensors](../tensor), which can represent everything from a scalar to an n-dimenstional tensor. These are called "ndarray" in NumPy terms. +The one special form of tensor processing that is available in regular Go code is _n dimensional indexing_, e.g., `tsr[1,2]`. This kind of expression with square brackets `[ ]` and a comma is illegal according to standard Go syntax, so when we detect it, we know that it is being used on a tensor object, and can transpile it into the corresponding `tensor.Value` or `tensor.Set*` expression. This is particularly convenient for [gosl](gosl) GPU code that has special support for tensor data. Note that for this GPU use-case, you actually do _not_ want to use math mode, because that engages a different, more complex form of indexing that does _not_ work on the GPU. + The rationale and mnemonics for using `$` and `#` are as follows: * These are two of the three common ASCII keyboard symbols that are not part of standard Go syntax (`@` being the other). diff --git a/goal/transpile/token.go b/goal/transpile/token.go index 126adbd144..0ba82b3c23 100644 --- a/goal/transpile/token.go +++ b/goal/transpile/token.go @@ -358,3 +358,25 @@ func (tk Tokens) ModeEnd() int { } return -1 } + +// IsAssignExpr checks if there are any Go assignment or define tokens +// outside of { } Go code. +func (tk Tokens) IsAssignExpr() bool { + n := len(tk) + if n == 0 { + return false + } + for i := 1; i < n; i++ { + tok := tk[i].Tok + if tok == token.ASSIGN || tok == token.DEFINE || (tok >= token.ADD_ASSIGN && tok <= token.AND_NOT_ASSIGN) { + return true + } + if tok == token.LBRACE { // skip Go mode + rp := tk[i:n].RightMatching() + if rp > 0 { + i += rp + } + } + } + return false +} diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index c5d502baf0..cabfddf3cc 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -135,7 +135,7 @@ func (st *State) TranspileLineTokens(code string) Tokens { case !f0exec: // exec must be IDENT logx.PrintlnDebug("go: not ident") return st.TranspileGo(toks, code) - case f0exec && en > 1 && (ewords[1][0] == '=' || ewords[1][0] == ':' || ewords[1][0] == '+' || toks[1].Tok == token.COMMA): + case f0exec && en > 1 && ewords[0] != "set" && toks.IsAssignExpr(): logx.PrintlnDebug("go: assignment or defn") return st.TranspileGo(toks, code) case f0exec: // now any ident @@ -270,7 +270,6 @@ func (st *State) TranspileExec(ewords []string, output bool) Tokens { for i := 0; i < n; i++ { f := ewords[i] switch { - // case f == "#": // embedded math TODO case f == "{": // embedded go if n < i+3 { st.AddError(fmt.Errorf("goal: no matching right brace } found in exec command line")) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index d9151824d8..955c8429fa 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -91,6 +91,7 @@ func TestTranspile(t *testing.T) { {"ls", `goal.Run("ls")`}, {"$ls -la$", `goal.Run("ls", "-la")`}, {"ls -la", `goal.Run("ls", "-la")`}, + {"chmod +x file", `goal.Run("chmod", "+x", "file")`}, {"ls --help", `goal.Run("ls", "--help")`}, {"ls go", `goal.Run("ls", "go")`}, {"cd go", `goal.Run("cd", "go")`}, @@ -174,9 +175,10 @@ func TestTranspile(t *testing.T) { {"x = a[1,f(2,3)]", `x = a.Value(1, f(2, 3))`}, {"x = a[1]", `x = a[1]`}, {"x = a[f(2,3)]", `x = a[f(2, 3)]`}, - {"x = a[1,2] = 55", `x = a.Set(55, 1, 2)`}, - {"x = a[1,2] += f(2,55)", `x = a.SetAdd(f(2, 55), 1, 2)`}, - {"x = a[1,2] *= f(2,55)", `x = a.SetMul(f(2, 55), 1, 2)`}, + {"a[1,2] = 55", `a.Set(55, 1, 2)`}, + {"a[1,2] += f(2,55)", `a.SetAdd(f(2, 55), 1, 2)`}, + {"a[1,2] *= f(2,55)", `a.SetMul(f(2, 55), 1, 2)`}, + {"Data[idx, Integ] = integ", `Data.Set(integ, idx, Integ)`}, } st := NewState() @@ -211,7 +213,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"x = a[1,2]", `x = a.Value(1, 2)`}, + {"Data[idx, Integ] = integ", `Data.Set(integ, idx, Integ)`}, } st := NewState() st.MathRecord = false From 562463c809694e1083476b3d8afa979b0a000aa2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 12:39:51 -0700 Subject: [PATCH 212/311] core.Filename is alias for fs.Filename --- core/filepicker.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/filepicker.go b/core/filepicker.go index f8ba257e5f..a46a4dd133 100644 --- a/core/filepicker.go +++ b/core/filepicker.go @@ -21,6 +21,7 @@ import ( "cogentcore.org/core/base/elide" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fileinfo" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/colors" "cogentcore.org/core/cursors" "cogentcore.org/core/events" @@ -705,7 +706,7 @@ func (fp *FilePicker) editRecentPaths() { // Filename is used to specify an file path. // It results in a [FileButton] [Value]. -type Filename string +type Filename = fsx.Filename // FileButton represents a [Filename] value with a button // that opens a [FilePicker]. From fb30eee847a587aeddda6dcd5a86606927ec3e2a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 12:59:02 -0700 Subject: [PATCH 213/311] fix goal build final code composition -- generate code first then wrap -- chunks.. --- goal/gosl/examples/basic/compute.go | 4 +- goal/gosl/examples/basic/compute.goal | 4 +- goal/gosl/examples/basic/shaders/Compute.wgsl | 2 - goal/transpile/state.go | 38 +++++++++++-------- 4 files changed, 27 insertions(+), 21 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index cb5749fdf4..b2291ec0b1 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -1,3 +1,5 @@ +// Code generated by "goal build"; DO NOT EDIT. + // Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -32,8 +34,6 @@ const ( Exp ) -// Code generated by "goal build"; DO NOT EDIT. - // ParamStruct has the test params type ParamStruct struct { diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal index 178b12048f..89a42c956f 100644 --- a/goal/gosl/examples/basic/compute.goal +++ b/goal/gosl/examples/basic/compute.goal @@ -47,8 +47,8 @@ type ParamStruct struct { func (ps *ParamStruct) IntegFromRaw(idx int) { integ := Data[idx, Integ] integ += ps.Dt * (Data[idx, Raw] - integ) - {Data[idx, Integ] = integ} - {Data[idx, Exp] = math32.FastExp(-integ)} + Data[idx, Integ] = integ + Data[idx, Exp] = math32.FastExp(-integ) } // Compute does the main computation. diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index ba379bf7ae..ddee8d11e0 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -37,8 +37,6 @@ const Integ: i32 = 1; const Exp: i32 = 2; -// Code generated by "goal build"; DO NOT EDIT. - // ParamStruct has the test params struct ParamStruct { diff --git a/goal/transpile/state.go b/goal/transpile/state.go index e7a77d9a53..66d46e2070 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -9,7 +9,6 @@ import ( "fmt" "log/slog" "os" - "slices" "strings" "cogentcore.org/core/base/logx" @@ -110,24 +109,33 @@ func (st *State) TranspileFile(in string, out string) error { if err != nil { return err } + + hdr := `package main +import ( + "cogentcore.org/core/goal" + "cogentcore.org/core/goal/goalib" + "cogentcore.org/core/tensor" + _ "cogentcore.org/core/tensor/tmath" + _ "cogentcore.org/core/tensor/stats/stats" + _ "cogentcore.org/core/tensor/stats/metric" +) + +func main() { + goal := goal.NewGoal() + _ = goal +` + + src := st.Code() gen := "// Code generated by \"goal build\"; DO NOT EDIT.\n\n" if hasPackage { - st.Lines = slices.Insert(st.Lines, 0, gen) + src = gen + src } else { - st.Lines = slices.Insert(st.Lines, 0, gen, "package main", "import (", - ` "cogentcore.org/core/goal"`, - ` "cogentcore.org/core/goal/goalib"`, - ` "cogentcore.org/core/tensor"`, - ` _ "cogentcore.org/core/tensor/tmath"`, - ` _ "cogentcore.org/core/tensor/stats/stats"`, - ` _ "cogentcore.org/core/tensor/stats/metric"`, - ")", "func main() {", "goal := goal.NewGoal()", "_ = goal") - st.Lines = append(st.Lines, "}") - } - src := []byte(st.Code()) - res, err := imports.Process(out, src, nil) + src = gen + hdr + src + "\n}" + } + bsrc := []byte(src) + res, err := imports.Process(out, bsrc, nil) if err != nil { - res = src + res = bsrc slog.Error(err.Error()) } else { err = st.DepthError() From 636a2d6b7fc887769eff67829265028d919ea43c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 14:03:15 -0700 Subject: [PATCH 214/311] get the scripts functionality and helper functions from numbers/databrowser. --- tensor/databrowser/browser.go | 63 +++++++- tensor/databrowser/scripts.go | 149 ++++++++++++++++++ tensor/databrowser/typegen.go | 21 ++- tensor/stats/histogram/histogram.go | 11 +- tensor/table/io.go | 43 +++++ .../cogentcore_org-core-tensor-databrowser.go | 6 + 6 files changed, 280 insertions(+), 13 deletions(-) create mode 100644 tensor/databrowser/scripts.go diff --git a/tensor/databrowser/browser.go b/tensor/databrowser/browser.go index 21d7cb268a..a1103eefc3 100644 --- a/tensor/databrowser/browser.go +++ b/tensor/databrowser/browser.go @@ -8,30 +8,56 @@ package databrowser import ( "io/fs" + "os" "path/filepath" + "slices" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" "cogentcore.org/core/core" "cogentcore.org/core/events" "cogentcore.org/core/filetree" + "cogentcore.org/core/goal/interpreter" "cogentcore.org/core/icons" "cogentcore.org/core/styles" "cogentcore.org/core/tree" "cogentcore.org/core/types" + "golang.org/x/exp/maps" ) -// Browser is a data browser, for browsing data either on an os filesystem +// TheBrowser is the current browser, +// which is valid immediately after NewBrowserWindow +// where it is used to get a local variable for subsequent use. +var TheBrowser *Browser + +// Browser is a data browser, for browsing data either on an OS filesystem // or as a datafs virtual data filesystem. +// It supports the automatic loading of [goal] scripts as toolbar actions to +// perform pre-programmed tasks on the data, to create app-like functionality. +// Scripts are ordered alphabetically and any leading #- prefix is automatically +// removed from the label, so you can use numbers to specify a custom order. type Browser struct { core.Frame // FS is the filesystem, if browsing an FS FS fs.FS - // DataRoot is the path to the root of the data to browse + // DataRoot is the path to the root of the data to browse. DataRoot string + // StartDir is the starting directory, where the app was originally started. + StartDir string + + // ScriptsDir is the directory containing scripts for toolbar actions. + // It defaults to DataRoot/dbscripts + ScriptsDir string + + // Scripts + Scripts map[string]string `set:"-"` + + // Interpreter is the interpreter to use for running Browser scripts + Interpreter *interpreter.Interpreter `set:"-"` + toolbar *core.Toolbar splits *core.Splits files *filetree.Tree @@ -44,6 +70,7 @@ func (br *Browser) Init() { br.Styler(func(s *styles.Style) { s.Grow.Set(1, 1) }) + br.InitInterp() br.OnShow(func(e events.Event) { br.UpdateFiles() @@ -80,7 +107,9 @@ func (br *Browser) Init() { // NewBrowserWindow opens a new data Browser for given // file system (nil for os files) and data directory. func NewBrowserWindow(fsys fs.FS, dataDir string) *Browser { - b := core.NewBody("Cogent Data Browser: " + fsx.DirAndFile(dataDir)) + startDir, _ := os.Getwd() + startDir = errors.Log1(filepath.Abs(startDir)) + b := core.NewBody("Cogent Data Browser: " + fsx.DirAndFile(startDir)) br := NewBrowser(b) br.FS = fsys ddr := dataDir @@ -93,6 +122,10 @@ func NewBrowserWindow(fsys fs.FS, dataDir string) *Browser { tb.Maker(br.MakeToolbar) }) br.SetDataRoot(ddr) + br.SetScriptsDir(filepath.Join(ddr, "dbscripts")) + TheBrowser = br + br.Interpreter.Eval("br := databrowser.TheBrowser") // grab it + br.UpdateScripts() b.RunWindow() return br } @@ -122,8 +155,32 @@ func (br *Browser) UpdateFiles() { //types:add br.Update() } +func (br *Browser) GetDataRoot() string { + return br.DataRoot +} + func (br *Browser) MakeToolbar(p *tree.Plan) { tree.Add(p, func(w *core.FuncButton) { w.SetFunc(br.UpdateFiles).SetText("").SetIcon(icons.Refresh).SetShortcut("Command+U") }) + tree.Add(p, func(w *core.FuncButton) { + w.SetFunc(br.UpdateScripts).SetText("").SetIcon(icons.Code) + }) + scr := maps.Keys(br.Scripts) + slices.Sort(scr) + for _, s := range scr { + lbl := TrimOrderPrefix(s) + tree.AddAt(p, lbl, func(w *core.Button) { + w.SetText(lbl).SetIcon(icons.RunCircle). + OnClick(func(e events.Event) { + br.RunScript(s) + }) + sc := br.Scripts[s] + tt := FirstComment(sc) + if tt == "" { + tt = "Run Script (add a comment to top of script to provide more useful info here)" + } + w.SetTooltip(tt) + }) + } } diff --git a/tensor/databrowser/scripts.go b/tensor/databrowser/scripts.go new file mode 100644 index 0000000000..c52b553846 --- /dev/null +++ b/tensor/databrowser/scripts.go @@ -0,0 +1,149 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package databrowser + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "unicode" + + "cogentcore.org/core/base/fsx" + "cogentcore.org/core/base/logx" + "cogentcore.org/core/core" + "cogentcore.org/core/events" + "cogentcore.org/core/goal/goalib" + "cogentcore.org/core/goal/interpreter" + "cogentcore.org/core/styles" + "github.com/cogentcore/yaegi/interp" +) + +func (br *Browser) InitInterp() { + br.Interpreter = interpreter.NewInterpreter(interp.Options{}) + br.Interpreter.Config() + // logx.UserLevel = slog.LevelDebug // for debugging of init loading +} + +func (br *Browser) RunScript(snm string) { + sc, ok := br.Scripts[snm] + if !ok { + slog.Error("script not found:", "Script:", snm) + return + } + logx.PrintlnDebug("\n################\nrunning script:\n", sc, "\n") + _, _, err := br.Interpreter.Eval(sc) + if err == nil { + err = br.Interpreter.Goal.TrState.DepthError() + } + br.Interpreter.Goal.TrState.ResetDepth() +} + +// UpdateScripts updates the Scripts and updates the toolbar. +func (br *Browser) UpdateScripts() { //types:add + redo := (br.Scripts != nil) + scr := fsx.Filenames(br.ScriptsDir, ".goal") + br.Scripts = make(map[string]string) + for _, s := range scr { + snm := strings.TrimSuffix(s, ".goal") + sc, err := os.ReadFile(filepath.Join(br.ScriptsDir, s)) + if err == nil { + if unicode.IsLower(rune(snm[0])) { + if !redo { + fmt.Println("run init script:", snm) + br.Interpreter.Eval(string(sc)) + } + } else { + ssc := string(sc) + br.Scripts[snm] = ssc + } + } else { + slog.Error(err.Error()) + } + } + if br.toolbar != nil { + br.toolbar.Update() + } +} + +// TrimOrderPrefix trims any optional #- prefix from given string, +// used for ordering items by name. +func TrimOrderPrefix(s string) string { + i := strings.Index(s, "-") + if i < 0 { + return s + } + ds := s[:i] + if _, err := strconv.Atoi(ds); err != nil { + return s + } + return s[i+1:] +} + +// PromptOKCancel prompts the user for whether to do something, +// calling the given function if the user clicks OK. +func PromptOKCancel(ctx core.Widget, prompt string, fun func()) { + d := core.NewBody(prompt) + d.AddBottomBar(func(bar *core.Frame) { + d.AddCancel(bar) + d.AddOK(bar).OnClick(func(e events.Event) { + if fun != nil { + fun() + } + }) + }) + d.RunDialog(ctx) +} + +// PromptString prompts the user for a string value (initial value given), +// calling the given function if the user clicks OK. +func PromptString(ctx core.Widget, str string, prompt string, fun func(s string)) { + d := core.NewBody(prompt) + tf := core.NewTextField(d).SetText(str) + tf.Styler(func(s *styles.Style) { + s.Min.X.Ch(60) + }) + d.AddBottomBar(func(bar *core.Frame) { + d.AddCancel(bar) + d.AddOK(bar).OnClick(func(e events.Event) { + if fun != nil { + fun(tf.Text()) + } + }) + }) + d.RunDialog(ctx) +} + +// PromptStruct prompts the user for the values in given struct (pass a pointer), +// calling the given function if the user clicks OK. +func PromptStruct(ctx core.Widget, str any, prompt string, fun func()) { + d := core.NewBody(prompt) + core.NewForm(d).SetStruct(str) + d.AddBottomBar(func(bar *core.Frame) { + d.AddCancel(bar) + d.AddOK(bar).OnClick(func(e events.Event) { + if fun != nil { + fun() + } + }) + }) + d.RunDialog(ctx) +} + +// FirstComment returns the first comment lines from given .goal file, +// which is used to set the tooltip for scripts. +func FirstComment(sc string) string { + sl := goalib.SplitLines(sc) + cmt := "" + for _, l := range sl { + if !strings.HasPrefix(l, "// ") { + return cmt + } + cmt += strings.TrimSpace(l[3:]) + " " + } + return cmt +} diff --git a/tensor/databrowser/typegen.go b/tensor/databrowser/typegen.go index 7ffdb629a5..b376948271 100644 --- a/tensor/databrowser/typegen.go +++ b/tensor/databrowser/typegen.go @@ -9,21 +9,34 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Browser", IDName: "browser", Doc: "Browser is a data browser, for browsing data either on an os filesystem\nor as a datafs virtual data filesystem.", Methods: []types.Method{{Name: "UpdateFiles", Doc: "UpdateFiles Updates the file picker with current files in DataRoot,", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FS", Doc: "Filesystem, if browsing an FS"}, {Name: "DataRoot", Doc: "DataRoot is the path to the root of the data to browse"}, {Name: "toolbar"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Browser", IDName: "browser", Doc: "Browser is a data browser, for browsing data either on an OS filesystem\nor as a datafs virtual data filesystem.\nIt supports the automatic loading of [goal] scripts as toolbar actions to\nperform pre-programmed tasks on the data, to create app-like functionality.\nScripts are ordered alphabetically and any leading #- prefix is automatically\nremoved from the label, so you can use numbers to specify a custom order.", Methods: []types.Method{{Name: "UpdateFiles", Doc: "UpdateFiles Updates the files list.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "UpdateScripts", Doc: "UpdateScripts updates the Scripts and updates the toolbar.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FS", Doc: "FS is the filesystem, if browsing an FS"}, {Name: "DataRoot", Doc: "DataRoot is the path to the root of the data to browse."}, {Name: "StartDir", Doc: "StartDir is the starting directory, where the app was originally started."}, {Name: "ScriptsDir", Doc: "ScriptsDir is the directory containing scripts for toolbar actions.\nIt defaults to DataRoot/dbscripts"}, {Name: "Scripts", Doc: "Scripts"}, {Name: "Interpreter", Doc: "Interpreter is the interpreter to use for running Browser scripts"}, {Name: "toolbar"}, {Name: "splits"}, {Name: "files"}, {Name: "tabs"}}}) // NewBrowser returns a new [Browser] with the given optional parent: -// Browser is a data browser, for browsing data either on an os filesystem +// Browser is a data browser, for browsing data either on an OS filesystem // or as a datafs virtual data filesystem. +// It supports the automatic loading of [goal] scripts as toolbar actions to +// perform pre-programmed tasks on the data, to create app-like functionality. +// Scripts are ordered alphabetically and any leading #- prefix is automatically +// removed from the label, so you can use numbers to specify a custom order. func NewBrowser(parent ...tree.Node) *Browser { return tree.New[Browser](parent...) } // SetFS sets the [Browser.FS]: -// Filesystem, if browsing an FS +// FS is the filesystem, if browsing an FS func (t *Browser) SetFS(v fs.FS) *Browser { t.FS = v; return t } // SetDataRoot sets the [Browser.DataRoot]: -// DataRoot is the path to the root of the data to browse +// DataRoot is the path to the root of the data to browse. func (t *Browser) SetDataRoot(v string) *Browser { t.DataRoot = v; return t } +// SetStartDir sets the [Browser.StartDir]: +// StartDir is the starting directory, where the app was originally started. +func (t *Browser) SetStartDir(v string) *Browser { t.StartDir = v; return t } + +// SetScriptsDir sets the [Browser.ScriptsDir]: +// ScriptsDir is the directory containing scripts for toolbar actions. +// It defaults to DataRoot/dbscripts +func (t *Browser) SetScriptsDir(v string) *Browser { t.ScriptsDir = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.FileNode", IDName: "file-node", Doc: "FileNode is databrowser version of FileNode for FileTree", Methods: []types.Method{{Name: "EditFiles", Doc: "EditFiles calls EditFile on selected files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "PlotFiles", Doc: "PlotFiles calls PlotFile on selected files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "DiffDirs", Doc: "DiffDirs displays a browser with differences between two selected directories", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Node"}}}) // NewFileNode returns a new [FileNode] with the given optional parent: diff --git a/tensor/stats/histogram/histogram.go b/tensor/stats/histogram/histogram.go index e8bee794ed..7f2d093c57 100644 --- a/tensor/stats/histogram/histogram.go +++ b/tensor/stats/histogram/histogram.go @@ -8,6 +8,9 @@ package histogram import ( "cogentcore.org/core/base/slicesx" + "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" ) // F64 generates a histogram of counts of values within given @@ -33,8 +36,6 @@ func F64(hist *[]float64, vals []float64, nBins int, min, max float64) { } } -/* - // F64Table generates an table with a histogram of counts of values within given // number of bins and min / max range. The table has columns: Value, Count // if value is < min or > max it is ignored. @@ -45,13 +46,11 @@ func F64Table(dt *table.Table, vals []float64, nBins int, min, max float64) { dt.AddFloat64Column("Value") dt.AddFloat64Column("Count") dt.SetNumRows(nBins) - ct := dt.Columns[1].(*tensor.Float64) + ct := dt.Columns.Values[1].(*tensor.Float64) F64(&ct.Values, vals, nBins, min, max) inc := (max - min) / float64(nBins) - vls := dt.Columns[0].(*tensor.Float64).Values + vls := dt.Columns.Values[0].(*tensor.Float64).Values for i := 0; i < nBins; i++ { vls[i] = math32.Truncate64(min+float64(i)*inc, 4) } } - -*/ diff --git a/tensor/table/io.go b/tensor/table/io.go index d74a14e6ba..a36866f531 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -11,6 +11,7 @@ import ( "io" "io/fs" "log" + "log/slog" "math" "os" "reflect" @@ -19,6 +20,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" + "cogentcore.org/core/core" "cogentcore.org/core/tensor" ) @@ -465,3 +467,44 @@ func (dt *Table) TableHeaders() []string { } return hdrs } + +// CleanCatTSV cleans a TSV file formed by concatenating multiple files together. +// Removes redundant headers and then sorts by given set of columns. +func CleanCatTSV(filename string, sorts ...string) error { + str, err := os.ReadFile(filename) + if err != nil { + slog.Error(err.Error()) + return err + } + lns := strings.Split(string(str), "\n") + if len(lns) == 0 { + return nil + } + hdr := lns[0] + f, err := os.Create(filename) + if err != nil { + slog.Error(err.Error()) + return err + } + for i, ln := range lns { + if i > 0 && ln == hdr { + continue + } + io.WriteString(f, ln) + io.WriteString(f, "\n") + } + f.Close() + dt := New() + err = dt.OpenCSV(core.Filename(filename), tensor.Detect) + if err != nil { + slog.Error(err.Error()) + return err + } + dt.SortColumns(tensor.Ascending, tensor.StableSort, sorts...) + st := dt.New() + err = st.SaveCSV(core.Filename(filename), tensor.Tab, true) + if err != nil { + slog.Error(err.Error()) + } + return err +} diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go b/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go index 51e341d16a..c25a7f9fe7 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go @@ -11,12 +11,18 @@ func init() { Symbols["cogentcore.org/core/tensor/databrowser/databrowser"] = map[string]reflect.Value{ // function, constant and variable definitions "DataFS": reflect.ValueOf(databrowser.DataFS), + "FirstComment": reflect.ValueOf(databrowser.FirstComment), "IsTableFile": reflect.ValueOf(databrowser.IsTableFile), "NewBrowser": reflect.ValueOf(databrowser.NewBrowser), "NewBrowserWindow": reflect.ValueOf(databrowser.NewBrowserWindow), "NewDiffBrowserDirs": reflect.ValueOf(databrowser.NewDiffBrowserDirs), "NewFileNode": reflect.ValueOf(databrowser.NewFileNode), "ParentBrowser": reflect.ValueOf(databrowser.ParentBrowser), + "PromptOKCancel": reflect.ValueOf(databrowser.PromptOKCancel), + "PromptString": reflect.ValueOf(databrowser.PromptString), + "PromptStruct": reflect.ValueOf(databrowser.PromptStruct), + "TheBrowser": reflect.ValueOf(&databrowser.TheBrowser).Elem(), + "TrimOrderPrefix": reflect.ValueOf(databrowser.TrimOrderPrefix), // type definitions "Browser": reflect.ValueOf((*databrowser.Browser)(nil)), From 6e816011e076f5b0b4cbafe5dff9c7f3b3aa4e6c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 16:51:27 -0700 Subject: [PATCH 215/311] fileinfo detects generated files using standard Go regex, and filetree colors such files with distincitve blue-grey color. Fixes #1238 Also finally fixed dark / light mode switching for texteditor highlighting -- uses distinct generated theme colors instead of changing the original color, which remains intact. --- base/fileinfo/fileinfo.go | 5 + base/fileinfo/mimetype.go | 16 ++++ filetree/node.go | 3 + texteditor/highlighting/defaults.highlighting | 8 +- texteditor/highlighting/style.go | 94 +++++++++++-------- 5 files changed, 81 insertions(+), 45 deletions(-) diff --git a/base/fileinfo/fileinfo.go b/base/fileinfo/fileinfo.go index 28caffa376..aecb6c52a7 100644 --- a/base/fileinfo/fileinfo.go +++ b/base/fileinfo/fileinfo.go @@ -75,6 +75,10 @@ type FileInfo struct { //types:add // version control system status, when enabled VCS vcs.FileStatus `table:"-"` + // Generated indicates that the file is generated and should not be edited. + // For Go files, this regex: `^// Code generated .* DO NOT EDIT\.$` is used. + Generated bool + // full path to file, including name; for file functions Path string `table:"-"` } @@ -143,6 +147,7 @@ func (fi *FileInfo) SetMimeInfo() error { } fi.Cat = UnknownCategory fi.Known = Unknown + fi.Generated = IsGeneratedFile(fi.Path) fi.Kind = "" mtyp, _, err := MimeFromFile(fi.Path) if err != nil { diff --git a/base/fileinfo/mimetype.go b/base/fileinfo/mimetype.go index f1e6c03891..4d44d2bc7e 100644 --- a/base/fileinfo/mimetype.go +++ b/base/fileinfo/mimetype.go @@ -7,9 +7,12 @@ package fileinfo import ( "fmt" "mime" + "os" "path/filepath" + "regexp" "strings" + "cogentcore.org/core/base/errors" "github.com/h2non/filetype" ) @@ -99,6 +102,19 @@ func MimeFromFile(fname string) (mtype, ext string, err error) { return "", ext, fmt.Errorf("fileinfo.MimeFromFile could not find mime type for ext: %v file: %v", ext, fn) } +var generatedRe = regexp.MustCompile(`^// Code generated .* DO NOT EDIT`) + +func IsGeneratedFile(fname string) bool { + file, err := os.Open(fname) + if err != nil { + errors.Log(err) + return false + } + head := make([]byte, 2048) + file.Read(head) + return generatedRe.Match(head) +} + // todo: use this to check against mime types! // MimeToKindMapInit makes sure the MimeToKindMap is initialized from diff --git a/filetree/node.go b/filetree/node.go index 4cd44609ae..2f26abff3d 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -94,6 +94,9 @@ func (fn *Node) Init() { case status == vcs.Stored: s.Color = colors.Scheme.OnSurface } + if fn.Info.Generated { + s.Color = errors.Must1(gradient.FromString("#8080C0")) + } }) fn.On(events.KeyChord, func(e events.Event) { if core.DebugSettings.KeyEventTrace { diff --git a/texteditor/highlighting/defaults.highlighting b/texteditor/highlighting/defaults.highlighting index 3e42a6b9a9..cb7dd0aff8 100644 --- a/texteditor/highlighting/defaults.highlighting +++ b/texteditor/highlighting/defaults.highlighting @@ -6458,10 +6458,10 @@ "A": 0 }, "Background": { - "R": 225, - "G": 225, - "B": 225, - "A": 255 + "R": 0, + "G": 0, + "B": 0, + "A": 0 }, "Border": { "R": 0, diff --git a/texteditor/highlighting/style.go b/texteditor/highlighting/style.go index 634dc76e83..ec38c876ca 100644 --- a/texteditor/highlighting/style.go +++ b/texteditor/highlighting/style.go @@ -46,26 +46,34 @@ func (t Trilean) Prefix(s string) string { // StyleEntry is one value in the map of highlight style values type StyleEntry struct { - // text color + // Color is the text color. Color color.RGBA - // background color + // Background color. + // In general it is not good to use this because it obscures highlighting. Background color.RGBA - // border color? not sure what this is -- not really used + // Border color? not sure what this is -- not really used. Border color.RGBA `display:"-"` - // bold font + // Bold font. Bold Trilean - // italic font + // Italic font. Italic Trilean - // underline + // Underline. Underline Trilean - // don't inherit these settings from sub-category or category levels -- otherwise everything with a Pass is inherited + // NoInherit indicates to not inherit these settings from sub-category or category levels. + // Otherwise everything with a Pass is inherited. NoInherit bool + + // themeColor is the theme-adjusted text color. + themeColor color.RGBA + + // themeBackground is the theme-adjusted background color. + themeBackground color.RGBA } // // FromChroma copies styles from chroma @@ -108,7 +116,7 @@ func (se *StyleEntry) UpdateFromTheme() { if matcolor.SchemeIsDark { ctone = 80 } - se.Color = hc.WithChroma(max(hc.Chroma, 48)).WithTone(ctone).AsRGBA() + se.themeColor = hc.WithChroma(max(hc.Chroma, 48)).WithTone(ctone).AsRGBA() if !colors.IsNil(se.Background) { hb := hct.FromColor(se.Background) @@ -116,7 +124,7 @@ func (se *StyleEntry) UpdateFromTheme() { if matcolor.SchemeIsDark { btone = min(hb.Tone, 17) } - se.Background = hb.WithChroma(max(hb.Chroma, 6)).WithTone(btone).AsRGBA() + se.themeBackground = hb.WithChroma(max(hb.Chroma, 6)).WithTone(btone).AsRGBA() } } @@ -134,11 +142,11 @@ func (se StyleEntry) String() string { if se.NoInherit { out = append(out, "noinherit") } - if !colors.IsNil(se.Color) { - out = append(out, colors.AsString(se.Color)) + if !colors.IsNil(se.themeColor) { + out = append(out, colors.AsString(se.themeColor)) } - if !colors.IsNil(se.Background) { - out = append(out, "bg:"+colors.AsString(se.Background)) + if !colors.IsNil(se.themeBackground) { + out = append(out, "bg:"+colors.AsString(se.themeBackground)) } if !colors.IsNil(se.Border) { out = append(out, "border:"+colors.AsString(se.Border)) @@ -149,11 +157,11 @@ func (se StyleEntry) String() string { // ToCSS converts StyleEntry to CSS attributes. func (se StyleEntry) ToCSS() string { styles := []string{} - if !colors.IsNil(se.Color) { - styles = append(styles, "color: "+colors.AsString(se.Color)) + if !colors.IsNil(se.themeColor) { + styles = append(styles, "color: "+colors.AsString(se.themeColor)) } - if !colors.IsNil(se.Background) { - styles = append(styles, "background-color: "+colors.AsString(se.Background)) + if !colors.IsNil(se.themeBackground) { + styles = append(styles, "background-color: "+colors.AsString(se.themeBackground)) } if se.Bold == Yes { styles = append(styles, "font-weight: bold") @@ -170,11 +178,11 @@ func (se StyleEntry) ToCSS() string { // ToProperties converts the StyleEntry to key-value properties. func (se StyleEntry) ToProperties() map[string]any { pr := map[string]any{} - if !colors.IsNil(se.Color) { - pr["color"] = se.Color + if !colors.IsNil(se.themeColor) { + pr["color"] = se.themeColor } - if !colors.IsNil(se.Background) { - pr["background-color"] = se.Background + if !colors.IsNil(se.themeBackground) { + pr["background-color"] = se.themeBackground } if se.Bold == Yes { pr["font-weight"] = styles.WeightBold @@ -189,25 +197,27 @@ func (se StyleEntry) ToProperties() map[string]any { } // Sub subtracts two style entries, returning an entry with only the differences set -func (s StyleEntry) Sub(e StyleEntry) StyleEntry { +func (se StyleEntry) Sub(e StyleEntry) StyleEntry { out := StyleEntry{} - if e.Color != s.Color { - out.Color = s.Color + if e.Color != se.Color { + out.Color = se.Color + out.themeColor = se.themeColor } - if e.Background != s.Background { - out.Background = s.Background + if e.Background != se.Background { + out.Background = se.Background + out.themeBackground = se.themeBackground } - if e.Border != s.Border { - out.Border = s.Border + if e.Border != se.Border { + out.Border = se.Border } - if e.Bold != s.Bold { - out.Bold = s.Bold + if e.Bold != se.Bold { + out.Bold = se.Bold } - if e.Italic != s.Italic { - out.Italic = s.Italic + if e.Italic != se.Italic { + out.Italic = se.Italic } - if e.Underline != s.Underline { - out.Underline = s.Underline + if e.Underline != se.Underline { + out.Underline = se.Underline } return out } @@ -215,18 +225,20 @@ func (s StyleEntry) Sub(e StyleEntry) StyleEntry { // Inherit styles from ancestors. // // Ancestors should be provided from oldest, furthest away to newest, closest. -func (s StyleEntry) Inherit(ancestors ...StyleEntry) StyleEntry { - out := s +func (se StyleEntry) Inherit(ancestors ...StyleEntry) StyleEntry { + out := se for i := len(ancestors) - 1; i >= 0; i-- { if out.NoInherit { return out } ancestor := ancestors[i] - if colors.IsNil(out.Color) { + if colors.IsNil(out.themeColor) { out.Color = ancestor.Color + out.themeColor = ancestor.themeColor } - if colors.IsNil(out.Background) { + if colors.IsNil(out.themeBackground) { out.Background = ancestor.Background + out.themeBackground = ancestor.themeBackground } if colors.IsNil(out.Border) { out.Border = ancestor.Border @@ -244,9 +256,9 @@ func (s StyleEntry) Inherit(ancestors ...StyleEntry) StyleEntry { return out } -func (s StyleEntry) IsZero() bool { - return colors.IsNil(s.Color) && colors.IsNil(s.Background) && colors.IsNil(s.Border) && s.Bold == Pass && s.Italic == Pass && - s.Underline == Pass && !s.NoInherit +func (se StyleEntry) IsZero() bool { + return colors.IsNil(se.Color) && colors.IsNil(se.Background) && colors.IsNil(se.Border) && se.Bold == Pass && se.Italic == Pass && + se.Underline == Pass && !se.NoInherit } /////////////////////////////////////////////////////////////////////////////////// From 73e866314abb131b89e0cb80904660312a80c875 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 17:27:24 -0700 Subject: [PATCH 216/311] use go format.Source on generated full goal build files (but not import -- too slow). --- base/fileinfo/fileinfo.go | 2 +- goal/gosl/gotosl/gotosl.go | 6 +++--- goal/interpreter/config.go | 3 --- goal/transpile/state.go | 11 +++++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/base/fileinfo/fileinfo.go b/base/fileinfo/fileinfo.go index aecb6c52a7..0ef6bf11f4 100644 --- a/base/fileinfo/fileinfo.go +++ b/base/fileinfo/fileinfo.go @@ -77,7 +77,7 @@ type FileInfo struct { //types:add // Generated indicates that the file is generated and should not be edited. // For Go files, this regex: `^// Code generated .* DO NOT EDIT\.$` is used. - Generated bool + Generated bool `table:"-"` // full path to file, including name; for file functions Path string `table:"-"` diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 057502657e..704e971d12 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -195,9 +195,9 @@ func (st *State) Run() error { st.ProjectFiles() // get list of all files, recursively gets imports etc. if len(st.GoFiles) == 0 { - if st.Config.Debug { - fmt.Println("gosl: no gosl files in current directory") - } + // if st.Config.Debug { + // fmt.Println("gosl: no gosl files in current directory") + // } return nil } st.ExtractFiles() // get .go from project files diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go index e74afd7b53..c770a29468 100644 --- a/goal/interpreter/config.go +++ b/goal/interpreter/config.go @@ -127,13 +127,10 @@ func Build(c *Config) error { cfg := &gotosl.Config{} cfg.Debug = verbose - fmt.Println("running gosl...") err := gotosl.Run(cfg) if err != nil { errs = append(errs, err) } - - fmt.Println("running go build...") args := []string{"build"} if verbose { args = append(args, "-v") diff --git a/goal/transpile/state.go b/goal/transpile/state.go index 66d46e2070..9317158e5c 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -7,6 +7,7 @@ package transpile import ( "errors" "fmt" + "go/format" "log/slog" "os" "strings" @@ -126,14 +127,16 @@ func main() { ` src := st.Code() + res := []byte(src) + bsrc := res gen := "// Code generated by \"goal build\"; DO NOT EDIT.\n\n" if hasPackage { - src = gen + src + bsrc = []byte(gen + src) + res, err = format.Source(bsrc) } else { - src = gen + hdr + src + "\n}" + bsrc = []byte(gen + hdr + src + "\n}") + res, err = imports.Process(out, bsrc, nil) } - bsrc := []byte(src) - res, err := imports.Process(out, bsrc, nil) if err != nil { res = bsrc slog.Error(err.Error()) From 35825cc26e2a2de8d7c7008e532ae1c9fe0684d6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 17:36:58 -0700 Subject: [PATCH 217/311] textedtitor readonly uses SurfaceContainer (not Low) background for read only; does NOT set the buffer read only status based on styling (only effect of buffer is undo -- need that for output buffers that don't need to be recording undos). --- texteditor/editor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/texteditor/editor.go b/texteditor/editor.go index 180832babc..10dcb9a19d 100644 --- a/texteditor/editor.go +++ b/texteditor/editor.go @@ -248,6 +248,9 @@ func (ed *Editor) Init() { s.MaxBorder.Width.Set(units.Dp(2)) s.Background = colors.Scheme.SurfaceContainerLow + if s.IsReadOnly() { + s.Background = colors.Scheme.SurfaceContainer + } // note: a blank background does NOT work for depth color rendering if s.Is(states.Focused) { s.StateLayer = 0 @@ -317,9 +320,6 @@ func (ed *Editor) resetState() { if ed.Buffer == nil || ed.lastFilename != ed.Buffer.Filename { // don't reset if reopening.. ed.CursorPos = lexer.Pos{} } - if ed.Buffer != nil { - ed.Buffer.SetReadOnly(ed.IsReadOnly()) - } } // SetBuffer sets the [Buffer] that this is an editor of, and interconnects their events. From a7ed4e8f9c754326fe97d04b2de9f1a5cca56d8c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 21:41:56 -0700 Subject: [PATCH 218/311] exclude generated files from search results in filetree --- filetree/search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filetree/search.go b/filetree/search.go index 911b646139..80ae08b165 100644 --- a/filetree/search.go +++ b/filetree/search.go @@ -77,7 +77,7 @@ func Search(start *Node, find string, ignoreCase, regExp bool, loc FindLocation, // fmt.Printf("dir: %v closed\n", sfn.FPath) return tree.Break // don't go down into closed directories! } - if sfn.IsDir() || sfn.IsExec() || sfn.Info.Kind == "octet-stream" || sfn.isAutoSave() { + if sfn.IsDir() || sfn.IsExec() || sfn.Info.Kind == "octet-stream" || sfn.isAutoSave() || sfn.Info.Generated { // fmt.Printf("dir: %v opened\n", sfn.Nm) return tree.Continue } From a9780190b4d3014691b027ce483c34a6443058b0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 7 Oct 2024 23:30:15 -0700 Subject: [PATCH 219/311] moved PCA (EigSym) over to matrix, along with SVD, where they belong.. --- goal/gosl/gotosl/gotosl.go | 13 +- tensor/matrix/eigen.go | 309 ++++++++++++++++++ tensor/matrix/eigen_test.go | 89 +++++ tensor/matrix/testdata/iris-covar.tsv | 4 + tensor/matrix/testdata/iris.data | 150 +++++++++ tensor/stats/metric/README.md | 4 +- tensor/stats/metric/matrix.go | 111 ------- tensor/stats/metric/matrix_test.go | 40 +++ tensor/stats/metric/metric_test.go | 82 ----- tensor/table/io.go | 5 +- .../cogentcore_org-core-base-fileinfo.go | 1 + .../cogentcore_org-core-tensor-matrix.go | 64 ++-- ...cogentcore_org-core-tensor-stats-metric.go | 4 - .../nogui/cogentcore_org-core-tensor-table.go | 1 + 14 files changed, 640 insertions(+), 237 deletions(-) create mode 100644 tensor/matrix/eigen.go create mode 100644 tensor/matrix/eigen_test.go create mode 100644 tensor/matrix/testdata/iris-covar.tsv create mode 100644 tensor/matrix/testdata/iris.data create mode 100644 tensor/stats/metric/matrix_test.go diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 704e971d12..ccbd55d327 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -187,19 +187,18 @@ func (st *State) Run() error { if st.Config.Output == "" { st.Config.Output = "shaders" } + + st.ProjectFiles() // get list of all files, recursively gets imports etc. + if len(st.GoFiles) == 0 { + return nil + } + st.ImportsDir = filepath.Join(st.Config.Output, "imports") os.MkdirAll(st.Config.Output, 0755) os.MkdirAll(st.ImportsDir, 0755) RemoveGenFiles(st.Config.Output) RemoveGenFiles(st.ImportsDir) - st.ProjectFiles() // get list of all files, recursively gets imports etc. - if len(st.GoFiles) == 0 { - // if st.Config.Debug { - // fmt.Println("gosl: no gosl files in current directory") - // } - return nil - } st.ExtractFiles() // get .go from project files st.ExtractImports() // get .go from imports st.TranslateDir("./" + st.ImportsDir) diff --git a/tensor/matrix/eigen.go b/tensor/matrix/eigen.go new file mode 100644 index 0000000000..3b272e24e1 --- /dev/null +++ b/tensor/matrix/eigen.go @@ -0,0 +1,309 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/stats/stats" + "cogentcore.org/core/tensor/tmath" + "gonum.org/v1/gonum/mat" +) + +// Eig performs the eigen decomposition of the given square matrix, +// which is not symmetric. See EigSym for a symmetric square matrix. +// In this non-symmetric case, the results are typically complex valued, +// so the outputs are complex tensors. TODO: need complex support! +// The vectors are same size as the input. Each vector is a column +// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// i.e., maximum vector is the last column. +// The values are the size of one row, ordered *lowest* to *highest*. +// If the input tensor is > 2D, it is treated as a list of 2D matricies. +func Eig(a tensor.Tensor) (vecs, vals *tensor.Float64) { + vecs = tensor.NewFloat64() + vals = tensor.NewFloat64() + errors.Log(EigOut(a, vecs, vals)) + return +} + +// EigOut performs the eigen decomposition of the given square matrix, +// which is not symmetric. See EigSym for a symmetric square matrix. +// In this non-symmetric case, the results are typically complex valued, +// so the outputs are complex tensors. TODO: need complex support! +// The vectors are same size as the input. Each vector is a column +// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// i.e., maximum vector is the last column. +// The values are the size of one row, ordered *lowest* to *highest*. +// If the input tensor is > 2D, it is treated as a list of 2D matricies. +func EigOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { + if err := StringCheck(a); err != nil { + return err + } + na := a.NumDims() + if na == 1 { + return mat.ErrShape + } + var asz []int + ea := a + if na > 2 { + asz = tensor.SplitAtInnerDims(a, 2) + if asz[0] == 1 { + ea = tensor.Reshape(a, asz[1:]...) + na = 2 + } + } + if na == 2 { + if a.DimSize(0) != a.DimSize(1) { + return mat.ErrShape + } + ma, _ := NewMatrix(a) + vecs.SetShapeSizes(a.DimSize(0), a.DimSize(1)) + vals.SetShapeSizes(a.DimSize(0)) + do, _ := NewDense(vecs) + var eig mat.Eigen + ok := eig.Factorize(ma, mat.EigenRight) + if !ok { + return errors.New("gonum mat.Eigen Factorize failed") + } + _ = do + // eig.VectorsTo(do) // todo: requires complex! + // eig.Values(vals.Values) + return nil + } + ea = tensor.Reshape(a, asz...) + if ea.DimSize(1) != ea.DimSize(2) { + return mat.ErrShape + } + nr := ea.DimSize(0) + sz := ea.DimSize(1) + vecs.SetShapeSizes(nr, sz, sz) + vals.SetShapeSizes(nr, sz) + var errs []error + for r := range nr { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) + var eig mat.Eigen + ok := eig.Factorize(ma, mat.EigenRight) + if !ok { + errs = append(errs, errors.New("gonum mat.Eigen Factorize failed")) + } + _ = do + // eig.VectorsTo(do) // todo: requires complex! + // eig.Values(vals.Values[r*sz : (r+1)*sz]) + } + return errors.Join(errs...) +} + +// EigSym performs the eigen decomposition of the given symmetric square matrix, +// which produces real-valued results. When input is the [metric.CovarianceMatrix], +// this is known as Principal Components Analysis (PCA). +// The vectors are same size as the input. Each vector is a column +// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// i.e., maximum vector is the last column. +// The values are the size of one row, ordered *lowest* to *highest*. +// Note that Eig produces results in the *opposite* order of [SVD] (which is much faster). +// If the input tensor is > 2D, it is treated as a list of 2D matricies. +func EigSym(a tensor.Tensor) (vecs, vals *tensor.Float64) { + vecs = tensor.NewFloat64() + vals = tensor.NewFloat64() + errors.Log(EigSymOut(a, vecs, vals)) + return +} + +// EigSymOut performs the eigen decomposition of the given symmetric square matrix, +// which produces real-valued results. When input is the [metric.CovarianceMatrix], +// this is known as Principal Components Analysis (PCA). +// The vectors are same size as the input. Each vector is a column +// in this 2D square matrix, ordered *lowest* to *highest* across the columns, +// i.e., maximum vector is the last column. +// The values are the size of one row, ordered *lowest* to *highest*. +// Note that Eig produces results in the *opposite* order of [SVD] (which is much faster). +// If the input tensor is > 2D, it is treated as a list of 2D matricies. +func EigSymOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { + if err := StringCheck(a); err != nil { + return err + } + na := a.NumDims() + if na == 1 { + return mat.ErrShape + } + var asz []int + ea := a + if na > 2 { + asz = tensor.SplitAtInnerDims(a, 2) + if asz[0] == 1 { + ea = tensor.Reshape(a, asz[1:]...) + na = 2 + } + } + if na == 2 { + if a.DimSize(0) != a.DimSize(1) { + return mat.ErrShape + } + ma, _ := NewSymmetric(a) + vecs.SetShapeSizes(a.DimSize(0), a.DimSize(1)) + vals.SetShapeSizes(a.DimSize(0)) + do, _ := NewDense(vecs) + var eig mat.EigenSym + ok := eig.Factorize(ma, true) + if !ok { + return errors.New("gonum mat.EigenSym Factorize failed") + } + eig.VectorsTo(do) + eig.Values(vals.Values) + return nil + } + ea = tensor.Reshape(a, asz...) + if ea.DimSize(1) != ea.DimSize(2) { + return mat.ErrShape + } + nr := ea.DimSize(0) + sz := ea.DimSize(1) + vecs.SetShapeSizes(nr, sz, sz) + vals.SetShapeSizes(nr, sz) + var errs []error + for r := range nr { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewSymmetric(sa) + do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) + var eig mat.EigenSym + ok := eig.Factorize(ma, true) + if !ok { + errs = append(errs, errors.New("gonum mat.Eigen Factorize failed")) + } + eig.VectorsTo(do) + eig.Values(vals.Values[r*sz : (r+1)*sz]) + } + return errors.Join(errs...) +} + +// SVD performs the singular value decomposition of the given symmetric square matrix, +// which produces real-valued results, and is generally much faster than [EigSym], +// while producing the same results. +// The vectors are same size as the input. Each vector is a column +// in this 2D square matrix, ordered *highest* to *lowest* across the columns, +// i.e., maximum vector is the first column. +// The values are the size of one row ordered in alignment with the vectors. +// Note that SVD produces results in the *opposite* order of [EigSym]. +// If the input tensor is > 2D, it is treated as a list of 2D matricies. +func SVD(a tensor.Tensor) (vecs, vals *tensor.Float64) { + vecs = tensor.NewFloat64() + vals = tensor.NewFloat64() + errors.Log(SVDOut(a, vecs, vals)) + return +} + +// SVDOut performs the singular value decomposition of the given symmetric square matrix, +// which produces real-valued results, and is generally much faster than [EigSym], +// while producing the same results. +// The vectors are same size as the input. Each vector is a column +// in this 2D square matrix, ordered *highest* to *lowest* across the columns, +// i.e., maximum vector is the first column. +// The values are the size of one row ordered in alignment with the vectors. +// Note that SVD produces results in the *opposite* order of [EigSym]. +// If the input tensor is > 2D, it is treated as a list of 2D matricies. +func SVDOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { + if err := StringCheck(a); err != nil { + return err + } + na := a.NumDims() + if na == 1 { + return mat.ErrShape + } + var asz []int + ea := a + if na > 2 { + asz = tensor.SplitAtInnerDims(a, 2) + if asz[0] == 1 { + ea = tensor.Reshape(a, asz[1:]...) + na = 2 + } + } + if na == 2 { + if a.DimSize(0) != a.DimSize(1) { + return mat.ErrShape + } + ma, _ := NewSymmetric(a) + vecs.SetShapeSizes(a.DimSize(0), a.DimSize(1)) + vals.SetShapeSizes(a.DimSize(0)) + do, _ := NewDense(vecs) + var eig mat.SVD + ok := eig.Factorize(ma, mat.SVDFull) + if !ok { + return errors.New("gonum mat.SVD Factorize failed") + } + eig.UTo(do) + eig.Values(vals.Values) + return nil + } + ea = tensor.Reshape(a, asz...) + if ea.DimSize(1) != ea.DimSize(2) { + return mat.ErrShape + } + nr := ea.DimSize(0) + sz := ea.DimSize(1) + vecs.SetShapeSizes(nr, sz, sz) + vals.SetShapeSizes(nr, sz) + var errs []error + for r := range nr { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewSymmetric(sa) + do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) + var eig mat.SVD + ok := eig.Factorize(ma, mat.SVDFull) + if !ok { + errs = append(errs, errors.New("gonum mat.SVD Factorize failed")) + } + eig.UTo(do) + eig.Values(vals.Values[r*sz : (r+1)*sz]) + } + return errors.Join(errs...) +} + +// ProjectOnMatrixColumn is a convenience function for projecting given vector +// of values along a specific column (2nd dimension) of the given 2D matrix, +// specified by the scalar colindex, putting results into out. +// If the vec is more than 1 dimensional, then it is treated as rows x cells, +// and each row of cells is projected through the matrix column, producing a +// 1D output with the number of rows. Otherwise a single number is produced. +// This is typically done with results from SVD or EigSym (PCA). +func ProjectOnMatrixColumn(mtx, vec, colindex tensor.Tensor) tensor.Values { + out := tensor.NewOfType(vec.DataType()) + errors.Log(ProjectOnMatrixColumnOut(mtx, vec, colindex, out)) + return out +} + +// ProjectOnMatrixColumnOut is a convenience function for projecting given vector +// of values along a specific column (2nd dimension) of the given 2D matrix, +// specified by the scalar colindex, putting results into out. +// If the vec is more than 1 dimensional, then it is treated as rows x cells, +// and each row of cells is projected through the matrix column, producing a +// 1D output with the number of rows. Otherwise a single number is produced. +// This is typically done with results from SVD or EigSym (PCA). +func ProjectOnMatrixColumnOut(mtx, vec, colindex tensor.Tensor, out tensor.Values) error { + ci := int(colindex.Float1D(0)) + col := tensor.As1D(tensor.Reslice(mtx, tensor.Slice{}, ci)) + // fmt.Println(mtx.String(), col.String()) + rows, cells := vec.Shape().RowCellSize() + if rows > 0 && cells > 0 { + msum := tensor.NewFloat64Scalar(0) + out.SetShapeSizes(rows) + mout := tensor.NewFloat64(cells) + for i := range rows { + err := tmath.MulOut(tensor.Cells1D(vec, i), col, mout) + if err != nil { + return err + } + stats.SumOut(mout, msum) + out.SetFloat1D(msum.Float1D(0), i) + } + } else { + mout := tensor.NewFloat64(1) + tmath.MulOut(vec, col, mout) + stats.SumOut(mout, out) + } + return nil +} diff --git a/tensor/matrix/eigen_test.go b/tensor/matrix/eigen_test.go new file mode 100644 index 0000000000..e8d1258a39 --- /dev/null +++ b/tensor/matrix/eigen_test.go @@ -0,0 +1,89 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package matrix + +import ( + "testing" + + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" + "github.com/stretchr/testify/assert" +) + +func TestPCAIris(t *testing.T) { + dt := table.New() + dt.AddFloat64Column("data", 4) + dt.AddStringColumn("class") + err := dt.OpenCSV("testdata/iris.data", tensor.Comma) + if err != nil { + t.Error(err) + } + data := dt.Column("data") + + covar := tensor.NewFloat64(4, 4) + tensor.OpenCSV(covar, "testdata/iris-covar.tsv", tensor.Tab) + + vecs, vals := EigSym(covar) + + // fmt.Printf("correl vec: %v\n", vecs) + // fmt.Printf("correl val: %v\n", vals) + errtol := 1.0e-9 + corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} + for i, v := range vals.Values { + assert.InDelta(t, corvals[i], v, errtol) + } + + colidx := tensor.NewFloat64Scalar(3) // strongest at end + prjns := tensor.NewFloat64() + err = ProjectOnMatrixColumnOut(vecs, data, colidx, prjns) + assert.NoError(t, err) + // tensor.SaveCSV(prjns, "testdata/pca_projection.csv", tensor.Comma) + trgprjns := []float64{ + 2.6692308782935146, + 2.696434011868953, + 2.4811633041648684, + 2.5715124347750256, + 2.5906582247213543, + 3.0080988099460613, + 2.490941664609344, + 2.7014546083439073, + 2.4615836931965167, + 2.6716628159090594, + } + for i, v := range prjns.Values[:10] { + assert.InDelta(t, trgprjns[i], v, errtol) + } + + //////////////////////////////////////////////////////////// + // SVD + + err = SVDOut(covar, vecs, vals) + assert.NoError(t, err) + // fmt.Printf("correl vec: %v\n", vecs) + // fmt.Printf("correl val: %v\n", vals) + for i, v := range vals.Values { + assert.InDelta(t, corvals[3-i], v, errtol) // opposite order + } + + colidx.SetFloat1D(0, 0) // strongest at start + err = ProjectOnMatrixColumnOut(vecs, data, colidx, prjns) + assert.NoError(t, err) + // tensor.SaveCSV(prjns, "testdata/svd_projection.csv", tensor.Comma) + trgprjns = []float64{ + -2.6692308782935172, + -2.696434011868955, + -2.48116330416487, + -2.5715124347750273, + -2.590658224721357, + -3.008098809946064, + -2.4909416646093456, + -2.70145460834391, + -2.4615836931965185, + -2.671662815909061, + } + for i, v := range prjns.Values[:10] { + assert.InDelta(t, trgprjns[i], v, errtol) + } +} diff --git a/tensor/matrix/testdata/iris-covar.tsv b/tensor/matrix/testdata/iris-covar.tsv new file mode 100644 index 0000000000..d1493e2ec7 --- /dev/null +++ b/tensor/matrix/testdata/iris-covar.tsv @@ -0,0 +1,4 @@ +1 -0.10936924995064935 0.8717541573048719 0.8179536333691635 +-0.10936924995064935 1 -0.4205160964011548 -0.3565440896138057 +0.8717541573048719 -0.4205160964011548 1 0.9627570970509667 +0.8179536333691635 -0.3565440896138057 0.9627570970509667 1 diff --git a/tensor/matrix/testdata/iris.data b/tensor/matrix/testdata/iris.data new file mode 100644 index 0000000000..a3490e0e07 --- /dev/null +++ b/tensor/matrix/testdata/iris.data @@ -0,0 +1,150 @@ +5.1,3.5,1.4,0.2,Iris-setosa +4.9,3.0,1.4,0.2,Iris-setosa +4.7,3.2,1.3,0.2,Iris-setosa +4.6,3.1,1.5,0.2,Iris-setosa +5.0,3.6,1.4,0.2,Iris-setosa +5.4,3.9,1.7,0.4,Iris-setosa +4.6,3.4,1.4,0.3,Iris-setosa +5.0,3.4,1.5,0.2,Iris-setosa +4.4,2.9,1.4,0.2,Iris-setosa +4.9,3.1,1.5,0.1,Iris-setosa +5.4,3.7,1.5,0.2,Iris-setosa +4.8,3.4,1.6,0.2,Iris-setosa +4.8,3.0,1.4,0.1,Iris-setosa +4.3,3.0,1.1,0.1,Iris-setosa +5.8,4.0,1.2,0.2,Iris-setosa +5.7,4.4,1.5,0.4,Iris-setosa +5.4,3.9,1.3,0.4,Iris-setosa +5.1,3.5,1.4,0.3,Iris-setosa +5.7,3.8,1.7,0.3,Iris-setosa +5.1,3.8,1.5,0.3,Iris-setosa +5.4,3.4,1.7,0.2,Iris-setosa +5.1,3.7,1.5,0.4,Iris-setosa +4.6,3.6,1.0,0.2,Iris-setosa +5.1,3.3,1.7,0.5,Iris-setosa +4.8,3.4,1.9,0.2,Iris-setosa +5.0,3.0,1.6,0.2,Iris-setosa +5.0,3.4,1.6,0.4,Iris-setosa +5.2,3.5,1.5,0.2,Iris-setosa +5.2,3.4,1.4,0.2,Iris-setosa +4.7,3.2,1.6,0.2,Iris-setosa +4.8,3.1,1.6,0.2,Iris-setosa +5.4,3.4,1.5,0.4,Iris-setosa +5.2,4.1,1.5,0.1,Iris-setosa +5.5,4.2,1.4,0.2,Iris-setosa +4.9,3.1,1.5,0.1,Iris-setosa +5.0,3.2,1.2,0.2,Iris-setosa +5.5,3.5,1.3,0.2,Iris-setosa +4.9,3.1,1.5,0.1,Iris-setosa +4.4,3.0,1.3,0.2,Iris-setosa +5.1,3.4,1.5,0.2,Iris-setosa +5.0,3.5,1.3,0.3,Iris-setosa +4.5,2.3,1.3,0.3,Iris-setosa +4.4,3.2,1.3,0.2,Iris-setosa +5.0,3.5,1.6,0.6,Iris-setosa +5.1,3.8,1.9,0.4,Iris-setosa +4.8,3.0,1.4,0.3,Iris-setosa +5.1,3.8,1.6,0.2,Iris-setosa +4.6,3.2,1.4,0.2,Iris-setosa +5.3,3.7,1.5,0.2,Iris-setosa +5.0,3.3,1.4,0.2,Iris-setosa +7.0,3.2,4.7,1.4,Iris-versicolor +6.4,3.2,4.5,1.5,Iris-versicolor +6.9,3.1,4.9,1.5,Iris-versicolor +5.5,2.3,4.0,1.3,Iris-versicolor +6.5,2.8,4.6,1.5,Iris-versicolor +5.7,2.8,4.5,1.3,Iris-versicolor +6.3,3.3,4.7,1.6,Iris-versicolor +4.9,2.4,3.3,1.0,Iris-versicolor +6.6,2.9,4.6,1.3,Iris-versicolor +5.2,2.7,3.9,1.4,Iris-versicolor +5.0,2.0,3.5,1.0,Iris-versicolor +5.9,3.0,4.2,1.5,Iris-versicolor +6.0,2.2,4.0,1.0,Iris-versicolor +6.1,2.9,4.7,1.4,Iris-versicolor +5.6,2.9,3.6,1.3,Iris-versicolor +6.7,3.1,4.4,1.4,Iris-versicolor +5.6,3.0,4.5,1.5,Iris-versicolor +5.8,2.7,4.1,1.0,Iris-versicolor +6.2,2.2,4.5,1.5,Iris-versicolor +5.6,2.5,3.9,1.1,Iris-versicolor +5.9,3.2,4.8,1.8,Iris-versicolor +6.1,2.8,4.0,1.3,Iris-versicolor +6.3,2.5,4.9,1.5,Iris-versicolor +6.1,2.8,4.7,1.2,Iris-versicolor +6.4,2.9,4.3,1.3,Iris-versicolor +6.6,3.0,4.4,1.4,Iris-versicolor +6.8,2.8,4.8,1.4,Iris-versicolor +6.7,3.0,5.0,1.7,Iris-versicolor +6.0,2.9,4.5,1.5,Iris-versicolor +5.7,2.6,3.5,1.0,Iris-versicolor +5.5,2.4,3.8,1.1,Iris-versicolor +5.5,2.4,3.7,1.0,Iris-versicolor +5.8,2.7,3.9,1.2,Iris-versicolor +6.0,2.7,5.1,1.6,Iris-versicolor +5.4,3.0,4.5,1.5,Iris-versicolor +6.0,3.4,4.5,1.6,Iris-versicolor +6.7,3.1,4.7,1.5,Iris-versicolor +6.3,2.3,4.4,1.3,Iris-versicolor +5.6,3.0,4.1,1.3,Iris-versicolor +5.5,2.5,4.0,1.3,Iris-versicolor +5.5,2.6,4.4,1.2,Iris-versicolor +6.1,3.0,4.6,1.4,Iris-versicolor +5.8,2.6,4.0,1.2,Iris-versicolor +5.0,2.3,3.3,1.0,Iris-versicolor +5.6,2.7,4.2,1.3,Iris-versicolor +5.7,3.0,4.2,1.2,Iris-versicolor +5.7,2.9,4.2,1.3,Iris-versicolor +6.2,2.9,4.3,1.3,Iris-versicolor +5.1,2.5,3.0,1.1,Iris-versicolor +5.7,2.8,4.1,1.3,Iris-versicolor +6.3,3.3,6.0,2.5,Iris-virginica +5.8,2.7,5.1,1.9,Iris-virginica +7.1,3.0,5.9,2.1,Iris-virginica +6.3,2.9,5.6,1.8,Iris-virginica +6.5,3.0,5.8,2.2,Iris-virginica +7.6,3.0,6.6,2.1,Iris-virginica +4.9,2.5,4.5,1.7,Iris-virginica +7.3,2.9,6.3,1.8,Iris-virginica +6.7,2.5,5.8,1.8,Iris-virginica +7.2,3.6,6.1,2.5,Iris-virginica +6.5,3.2,5.1,2.0,Iris-virginica +6.4,2.7,5.3,1.9,Iris-virginica +6.8,3.0,5.5,2.1,Iris-virginica +5.7,2.5,5.0,2.0,Iris-virginica +5.8,2.8,5.1,2.4,Iris-virginica +6.4,3.2,5.3,2.3,Iris-virginica +6.5,3.0,5.5,1.8,Iris-virginica +7.7,3.8,6.7,2.2,Iris-virginica +7.7,2.6,6.9,2.3,Iris-virginica +6.0,2.2,5.0,1.5,Iris-virginica +6.9,3.2,5.7,2.3,Iris-virginica +5.6,2.8,4.9,2.0,Iris-virginica +7.7,2.8,6.7,2.0,Iris-virginica +6.3,2.7,4.9,1.8,Iris-virginica +6.7,3.3,5.7,2.1,Iris-virginica +7.2,3.2,6.0,1.8,Iris-virginica +6.2,2.8,4.8,1.8,Iris-virginica +6.1,3.0,4.9,1.8,Iris-virginica +6.4,2.8,5.6,2.1,Iris-virginica +7.2,3.0,5.8,1.6,Iris-virginica +7.4,2.8,6.1,1.9,Iris-virginica +7.9,3.8,6.4,2.0,Iris-virginica +6.4,2.8,5.6,2.2,Iris-virginica +6.3,2.8,5.1,1.5,Iris-virginica +6.1,2.6,5.6,1.4,Iris-virginica +7.7,3.0,6.1,2.3,Iris-virginica +6.3,3.4,5.6,2.4,Iris-virginica +6.4,3.1,5.5,1.8,Iris-virginica +6.0,3.0,4.8,1.8,Iris-virginica +6.9,3.1,5.4,2.1,Iris-virginica +6.7,3.1,5.6,2.4,Iris-virginica +6.9,3.1,5.1,2.3,Iris-virginica +5.8,2.7,5.1,1.9,Iris-virginica +6.8,3.2,5.9,2.3,Iris-virginica +6.7,3.3,5.7,2.5,Iris-virginica +6.7,3.0,5.2,2.3,Iris-virginica +6.3,2.5,5.0,1.9,Iris-virginica +6.5,3.0,5.2,2.0,Iris-virginica +6.2,3.4,5.4,2.3,Iris-virginica +5.9,3.0,5.1,1.8,Iris-virginica diff --git a/tensor/stats/metric/README.md b/tensor/stats/metric/README.md index 7b9a80d09f..7d1174182c 100644 --- a/tensor/stats/metric/README.md +++ b/tensor/stats/metric/README.md @@ -50,8 +50,8 @@ Metric functions cannot be computed in parallel, e.g., using VectorizeThreaded o * `CovarianceMatrix` computes the _covariance matrix_ for row-wise lists of patterns, where the result is a square matrix of cells x cells size ("cells" is number of elements in the patterns per row), and each value represents the extent to which value of a given cell covaries across the rows of the tensor with the value of another cell. For example, if the rows represent time, then the covariance matrix represents the extent to which the patterns tend to move in the same way over time. -* `PCA` and `SVD` operate on the `CovarianceMatrix` to extract the "principal components" of covariance, in terms of the _eigenvectors_ and corresponding _eigenvalues_ of this matrix. The eigenvector (component) with the largest eigenvalue is the "direction" in n-dimensional pattern space along which there is the greatest variance in the patterns across the rows. + See [matrix](../../matrix) for `EigSym` and `SVD` functions that compute the "principal components" (PCA) of covariance, in terms of the _eigenvectors_ and corresponding _eigenvalues_ of this matrix. The eigenvector (component) with the largest eigenvalue is the "direction" in n-dimensional pattern space along which there is the greatest variance in the patterns across the rows. -* `ProjectOnMatrixColumn` is a convenient function for projecting data along a vector extracted from a matrix, which allows you to project data along an eigenvector from the PCA or SVD functions. By doing this projection along the strongest 2 eigenvectors (those with the largest eigenvalues), you can visualize high-dimensional data in a 2D plot, which typically reveals important aspects of the structure of the underlying high-dimensional data, which is otherwise hard to see given the difficulty in visualizing high-dimensional spaces. + There is also a `matrix.ProjectOnMatrixColumn` convenience function for projecting data along a vector extracted from a matrix, which allows you to project data along an eigenvector from the PCA or SVD functions. By doing this projection along the strongest 2 eigenvectors (those with the largest eigenvalues), you can visualize high-dimensional data in a 2D plot, which typically reveals important aspects of the structure of the underlying high-dimensional data, which is otherwise hard to see given the difficulty in visualizing high-dimensional spaces. diff --git a/tensor/stats/metric/matrix.go b/tensor/stats/metric/matrix.go index 6a8e9c9683..3e7f96639b 100644 --- a/tensor/stats/metric/matrix.go +++ b/tensor/stats/metric/matrix.go @@ -5,13 +5,9 @@ package metric import ( - "cogentcore.org/core/base/errors" "cogentcore.org/core/math32/vecint" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/matrix" - "cogentcore.org/core/tensor/stats/stats" - "cogentcore.org/core/tensor/tmath" - "gonum.org/v1/gonum/mat" ) // MatrixOut computes the rows x rows square distance / similarity matrix @@ -195,110 +191,3 @@ func CovarianceMatrixOut(fun any, in tensor.Tensor, out tensor.Values) error { func CovarianceMatrix(fun any, in tensor.Tensor) tensor.Values { return tensor.CallOut1Gen1(CovarianceMatrixOut, fun, in) } - -// PCA performs the eigen decomposition of the given CovarianceMatrix, -// using principal components analysis (PCA), which is slower than [SVD]. -// The eigenvectors are same size as Covar. Each eigenvector is a column -// in this 2D square matrix, ordered *lowest* to *highest* across the columns, -// i.e., maximum eigenvector is the last column. -// The eigenvalues are the size of one row, ordered *lowest* to *highest*. -// Note that PCA produces results in the *opposite* order of [SVD]. -func PCA(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { - n := covar.DimSize(0) - cv, err := matrix.NewSymmetric(tensor.AsFloat64(covar)) - if err != nil { - return err - } - eigenvecs.SetShapeSizes(n, n) - vals.SetShapeSizes(n) - var eig mat.EigenSym - ok := eig.Factorize(cv, true) - if !ok { - return errors.New("gonum mat.EigenSym Factorize failed") - } - var ev mat.Dense - eig.VectorsTo(&ev) - matrix.CopyFromDense(eigenvecs, &ev) - fv := tensor.AsFloat64(vals) - eig.Values(fv.Values) - if fv != vals { - vals.(tensor.Values).CopyFrom(fv) - } - return nil -} - -// SVD performs the eigen decomposition of the given CovarianceMatrix, -// using singular value decomposition (SVD), which is faster than [PCA]. -// The eigenvectors are same size as Covar. Each eigenvector is a column -// in this 2D square matrix, ordered *highest* to *lowest* across the columns, -// i.e., maximum eigenvector is the last column. -// The eigenvalues are the size of one row, ordered *highest* to *lowest*. -// Note that SVD produces results in the *opposite* order of [PCA]. -func SVD(covar tensor.Tensor, eigenvecs, vals tensor.Values) error { - n := covar.DimSize(0) - cv, err := matrix.NewSymmetric(tensor.AsFloat64(covar)) - if err != nil { - return err - } - eigenvecs.SetShapeSizes(n, n) - vals.SetShapeSizes(n) - var eig mat.SVD - ok := eig.Factorize(cv, mat.SVDFull) // todo: benchmark different versions - if !ok { - return errors.New("gonum mat.SVD Factorize failed") - } - var ev mat.Dense - eig.UTo(&ev) - matrix.CopyFromDense(eigenvecs, &ev) - fv := tensor.AsFloat64(vals) - eig.Values(fv.Values) - if fv != vals { - vals.(tensor.Values).CopyFrom(fv) - } - return nil -} - -// ProjectOnMatrixColumnOut is a convenience function for projecting given vector -// of values along a specific column (2nd dimension) of the given 2D matrix, -// specified by the scalar colindex, putting results into out. -// If the vec is more than 1 dimensional, then it is treated as rows x cells, -// and each row of cells is projected through the matrix column, producing a -// 1D output with the number of rows. Otherwise a single number is produced. -// This is typically done with results from SVD or PCA. -func ProjectOnMatrixColumnOut(mtx, vec, colindex tensor.Tensor, out tensor.Values) error { - ci := int(colindex.Float1D(0)) - col := tensor.As1D(tensor.Reslice(mtx, tensor.Slice{}, ci)) - // fmt.Println(mtx.String(), col.String()) - rows, cells := vec.Shape().RowCellSize() - if rows > 0 && cells > 0 { - msum := tensor.NewFloat64Scalar(0) - out.SetShapeSizes(rows) - mout := tensor.NewFloat64(cells) - for i := range rows { - err := tmath.MulOut(tensor.Cells1D(vec, i), col, mout) - if err != nil { - return err - } - stats.SumOut(mout, msum) - out.SetFloat1D(msum.Float1D(0), i) - } - } else { - mout := tensor.NewFloat64(1) - tmath.MulOut(vec, col, mout) - stats.SumOut(mout, out) - } - return nil -} - -// ProjectOnMatrixColumn is a convenience function for projecting given vector -// of values along a specific column (2nd dimension) of the given 2D matrix, -// specified by the scalar colindex, putting results into out. -// If the vec is more than 1 dimensional, then it is treated as rows x cells, -// and each row of cells is projected through the matrix column, producing a -// 1D output with the number of rows. Otherwise a single number is produced. -// This is typically done with results from SVD or PCA. -func ProjectOnMatrixColumn(mtx, vec, colindex tensor.Tensor) tensor.Values { - out := tensor.NewOfType(vec.DataType()) - errors.Log(ProjectOnMatrixColumnOut(mtx, vec, colindex, out)) - return out -} diff --git a/tensor/stats/metric/matrix_test.go b/tensor/stats/metric/matrix_test.go new file mode 100644 index 0000000000..3450d289da --- /dev/null +++ b/tensor/stats/metric/matrix_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metric + +import ( + "testing" + + "cogentcore.org/core/base/tolassert" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" + "github.com/stretchr/testify/assert" +) + +func TestCovarIris(t *testing.T) { + // note: these results are verified against this example: + // https://plot.ly/ipython-notebooks/principal-component-analysis/ + + dt := table.New() + dt.AddFloat64Column("data", 4) + dt.AddStringColumn("class") + err := dt.OpenCSV("testdata/iris.data", tensor.Comma) + if err != nil { + t.Error(err) + } + data := dt.Column("data") + covar := tensor.NewFloat64() + err = CovarianceMatrixOut(Correlation, data, covar) + assert.NoError(t, err) + // fmt.Printf("covar: %s\n", covar.String()) + // tensor.SaveCSV(covar, "testdata/iris-covar.tsv", tensor.Tab) + + cv := []float64{1, -0.10936924995064935, 0.8717541573048719, 0.8179536333691635, + -0.10936924995064935, 1, -0.4205160964011548, -0.3565440896138057, + 0.8717541573048719, -0.4205160964011548, 1, 0.9627570970509667, + 0.8179536333691635, -0.3565440896138057, 0.9627570970509667, 1} + + tolassert.EqualTolSlice(t, cv, covar.Values, 1.0e-8) +} diff --git a/tensor/stats/metric/metric_test.go b/tensor/stats/metric/metric_test.go index caa43d5f80..a8f200b819 100644 --- a/tensor/stats/metric/metric_test.go +++ b/tensor/stats/metric/metric_test.go @@ -86,85 +86,3 @@ func TestMatrix(t *testing.T) { assert.InDelta(t, v, out.Float1D(i), 1.0e-8) } } - -func TestPCAIris(t *testing.T) { - // note: these results are verified against this example: - // https://plot.ly/ipython-notebooks/principal-component-analysis/ - - dt := table.New() - dt.AddFloat64Column("data", 4) - dt.AddStringColumn("class") - err := dt.OpenCSV("testdata/iris.data", tensor.Comma) - if err != nil { - t.Error(err) - } - data := dt.Column("data") - covar := tensor.NewFloat64() - err = CovarianceMatrixOut(Correlation, data, covar) - assert.NoError(t, err) - // fmt.Printf("correl: %s\n", covar.String()) - - vecs := tensor.NewFloat64() - vals := tensor.NewFloat64() - PCA(covar, vecs, vals) - - // fmt.Printf("correl vec: %v\n", vecs) - // fmt.Printf("correl val: %v\n", vals) - errtol := 1.0e-9 - corvals := []float64{0.020607707235624825, 0.14735327830509573, 0.9212209307072254, 2.910818083752054} - for i, v := range vals.Values { - assert.InDelta(t, corvals[i], v, errtol) - } - - colidx := tensor.NewFloat64Scalar(3) // strongest at end - prjns := tensor.NewFloat64() - err = ProjectOnMatrixColumnOut(vecs, data, colidx, prjns) - assert.NoError(t, err) - // tensor.SaveCSV(prjns, "testdata/pca_projection.csv", tensor.Comma) - trgprjns := []float64{ - 2.6692308782935146, - 2.696434011868953, - 2.4811633041648684, - 2.5715124347750256, - 2.5906582247213543, - 3.0080988099460613, - 2.490941664609344, - 2.7014546083439073, - 2.4615836931965167, - 2.6716628159090594, - } - for i, v := range prjns.Values[:10] { - assert.InDelta(t, trgprjns[i], v, errtol) - } - - //////////////////////////////////////////////////////////// - // SVD - - err = SVD(covar, vecs, vals) - assert.NoError(t, err) - // fmt.Printf("correl vec: %v\n", vecs) - // fmt.Printf("correl val: %v\n", vals) - for i, v := range vals.Values { - assert.InDelta(t, corvals[3-i], v, errtol) // opposite order - } - - colidx.SetFloat1D(0, 0) // strongest at start - err = ProjectOnMatrixColumnOut(vecs, data, colidx, prjns) - assert.NoError(t, err) - // tensor.SaveCSV(prjns, "testdata/svd_projection.csv", tensor.Comma) - trgprjns = []float64{ - -2.6692308782935172, - -2.696434011868955, - -2.48116330416487, - -2.5715124347750273, - -2.590658224721357, - -3.008098809946064, - -2.4909416646093456, - -2.70145460834391, - -2.4615836931965185, - -2.671662815909061, - } - for i, v := range prjns.Values[:10] { - assert.InDelta(t, trgprjns[i], v, errtol) - } -} diff --git a/tensor/table/io.go b/tensor/table/io.go index a36866f531..762d04ddac 100644 --- a/tensor/table/io.go +++ b/tensor/table/io.go @@ -20,7 +20,6 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" - "cogentcore.org/core/core" "cogentcore.org/core/tensor" ) @@ -495,14 +494,14 @@ func CleanCatTSV(filename string, sorts ...string) error { } f.Close() dt := New() - err = dt.OpenCSV(core.Filename(filename), tensor.Detect) + err = dt.OpenCSV(fsx.Filename(filename), tensor.Detect) if err != nil { slog.Error(err.Error()) return err } dt.SortColumns(tensor.Ascending, tensor.StableSort, sorts...) st := dt.New() - err = st.SaveCSV(core.Filename(filename), tensor.Tab, true) + err = st.SaveCSV(fsx.Filename(filename), tensor.Tab, true) if err != nil { slog.Error(err.Error()) } diff --git a/yaegicore/nogui/cogentcore_org-core-base-fileinfo.go b/yaegicore/nogui/cogentcore_org-core-base-fileinfo.go index e55fb93a22..f1ea76f300 100644 --- a/yaegicore/nogui/cogentcore_org-core-base-fileinfo.go +++ b/yaegicore/nogui/cogentcore_org-core-base-fileinfo.go @@ -89,6 +89,7 @@ func init() { "Icons": reflect.ValueOf(&fileinfo.Icons).Elem(), "Image": reflect.ValueOf(fileinfo.Image), "Ini": reflect.ValueOf(fileinfo.Ini), + "IsGeneratedFile": reflect.ValueOf(fileinfo.IsGeneratedFile), "IsMatch": reflect.ValueOf(fileinfo.IsMatch), "IsMatchList": reflect.ValueOf(fileinfo.IsMatchList), "Java": reflect.ValueOf(fileinfo.Java), diff --git a/yaegicore/nogui/cogentcore_org-core-tensor-matrix.go b/yaegicore/nogui/cogentcore_org-core-tensor-matrix.go index 55bc38bdc0..96b2f7338d 100644 --- a/yaegicore/nogui/cogentcore_org-core-tensor-matrix.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-matrix.go @@ -10,34 +10,42 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/matrix/matrix"] = map[string]reflect.Value{ // function, constant and variable definitions - "CallOut1": reflect.ValueOf(matrix.CallOut1), - "CallOut2": reflect.ValueOf(matrix.CallOut2), - "CopyFromDense": reflect.ValueOf(matrix.CopyFromDense), - "Det": reflect.ValueOf(matrix.Det), - "Diagonal": reflect.ValueOf(matrix.Diagonal), - "DiagonalIndices": reflect.ValueOf(matrix.DiagonalIndices), - "DiagonalN": reflect.ValueOf(matrix.DiagonalN), - "Identity": reflect.ValueOf(matrix.Identity), - "Inverse": reflect.ValueOf(matrix.Inverse), - "InverseOut": reflect.ValueOf(matrix.InverseOut), - "LogDet": reflect.ValueOf(matrix.LogDet), - "Mul": reflect.ValueOf(matrix.Mul), - "MulOut": reflect.ValueOf(matrix.MulOut), - "NewDense": reflect.ValueOf(matrix.NewDense), - "NewMatrix": reflect.ValueOf(matrix.NewMatrix), - "NewSymmetric": reflect.ValueOf(matrix.NewSymmetric), - "StringCheck": reflect.ValueOf(matrix.StringCheck), - "Trace": reflect.ValueOf(matrix.Trace), - "Tri": reflect.ValueOf(matrix.Tri), - "TriL": reflect.ValueOf(matrix.TriL), - "TriLIndicies": reflect.ValueOf(matrix.TriLIndicies), - "TriLNum": reflect.ValueOf(matrix.TriLNum), - "TriLView": reflect.ValueOf(matrix.TriLView), - "TriU": reflect.ValueOf(matrix.TriU), - "TriUIndicies": reflect.ValueOf(matrix.TriUIndicies), - "TriUNum": reflect.ValueOf(matrix.TriUNum), - "TriUView": reflect.ValueOf(matrix.TriUView), - "TriUpper": reflect.ValueOf(matrix.TriUpper), + "CallOut1": reflect.ValueOf(matrix.CallOut1), + "CallOut2": reflect.ValueOf(matrix.CallOut2), + "CopyFromDense": reflect.ValueOf(matrix.CopyFromDense), + "Det": reflect.ValueOf(matrix.Det), + "Diagonal": reflect.ValueOf(matrix.Diagonal), + "DiagonalIndices": reflect.ValueOf(matrix.DiagonalIndices), + "DiagonalN": reflect.ValueOf(matrix.DiagonalN), + "Eig": reflect.ValueOf(matrix.Eig), + "EigOut": reflect.ValueOf(matrix.EigOut), + "EigSym": reflect.ValueOf(matrix.EigSym), + "EigSymOut": reflect.ValueOf(matrix.EigSymOut), + "Identity": reflect.ValueOf(matrix.Identity), + "Inverse": reflect.ValueOf(matrix.Inverse), + "InverseOut": reflect.ValueOf(matrix.InverseOut), + "LogDet": reflect.ValueOf(matrix.LogDet), + "Mul": reflect.ValueOf(matrix.Mul), + "MulOut": reflect.ValueOf(matrix.MulOut), + "NewDense": reflect.ValueOf(matrix.NewDense), + "NewMatrix": reflect.ValueOf(matrix.NewMatrix), + "NewSymmetric": reflect.ValueOf(matrix.NewSymmetric), + "ProjectOnMatrixColumn": reflect.ValueOf(matrix.ProjectOnMatrixColumn), + "ProjectOnMatrixColumnOut": reflect.ValueOf(matrix.ProjectOnMatrixColumnOut), + "SVD": reflect.ValueOf(matrix.SVD), + "SVDOut": reflect.ValueOf(matrix.SVDOut), + "StringCheck": reflect.ValueOf(matrix.StringCheck), + "Trace": reflect.ValueOf(matrix.Trace), + "Tri": reflect.ValueOf(matrix.Tri), + "TriL": reflect.ValueOf(matrix.TriL), + "TriLIndicies": reflect.ValueOf(matrix.TriLIndicies), + "TriLNum": reflect.ValueOf(matrix.TriLNum), + "TriLView": reflect.ValueOf(matrix.TriLView), + "TriU": reflect.ValueOf(matrix.TriU), + "TriUIndicies": reflect.ValueOf(matrix.TriUIndicies), + "TriUNum": reflect.ValueOf(matrix.TriUNum), + "TriUView": reflect.ValueOf(matrix.TriUView), + "TriUpper": reflect.ValueOf(matrix.TriUpper), // type definitions "Matrix": reflect.ValueOf((*matrix.Matrix)(nil)), diff --git a/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go index 5d7074a9f8..b3ae42b314 100644 --- a/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go @@ -59,10 +59,6 @@ func init() { "MetricsN": reflect.ValueOf(metric.MetricsN), "MetricsValues": reflect.ValueOf(metric.MetricsValues), "NFunc": reflect.ValueOf(metric.NFunc), - "PCA": reflect.ValueOf(metric.PCA), - "ProjectOnMatrixColumn": reflect.ValueOf(metric.ProjectOnMatrixColumn), - "ProjectOnMatrixColumnOut": reflect.ValueOf(metric.ProjectOnMatrixColumnOut), - "SVD": reflect.ValueOf(metric.SVD), "SumSquares": reflect.ValueOf(metric.SumSquares), "SumSquaresBinTol": reflect.ValueOf(metric.SumSquaresBinTol), "SumSquaresBinTolOut": reflect.ValueOf(metric.SumSquaresBinTolOut), diff --git a/yaegicore/nogui/cogentcore_org-core-tensor-table.go b/yaegicore/nogui/cogentcore_org-core-tensor-table.go index 9b605ac86b..7ee1d5d517 100644 --- a/yaegicore/nogui/cogentcore_org-core-tensor-table.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-table.go @@ -10,6 +10,7 @@ import ( func init() { Symbols["cogentcore.org/core/tensor/table/table"] = map[string]reflect.Value{ // function, constant and variable definitions + "CleanCatTSV": reflect.ValueOf(table.CleanCatTSV), "ConfigFromDataValues": reflect.ValueOf(table.ConfigFromDataValues), "ConfigFromHeaders": reflect.ValueOf(table.ConfigFromHeaders), "ConfigFromTableHeaders": reflect.ValueOf(table.ConfigFromTableHeaders), From 972bb9f4768506cbf7716f943643f16c9d7ce6e0 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 11:24:40 -0700 Subject: [PATCH 220/311] use parallel threading for N>2 dim matrix ops which are very expensive and ideal targets. --- goal/interpreter/config.go | 15 +++--- tensor/matrix/eigen.go | 98 +++++++++++++++++++++----------------- tensor/matrix/ops.go | 75 ++++++++++++++++------------- 3 files changed, 105 insertions(+), 83 deletions(-) diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go index c770a29468..006fa8f0eb 100644 --- a/goal/interpreter/config.go +++ b/goal/interpreter/config.go @@ -16,7 +16,6 @@ import ( "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/logx" "cogentcore.org/core/goal" - "cogentcore.org/core/goal/gosl/gotosl" "github.com/cogentcore/yaegi/interp" ) @@ -125,17 +124,17 @@ func Build(c *Config) error { } } - cfg := &gotosl.Config{} - cfg.Debug = verbose - err := gotosl.Run(cfg) - if err != nil { - errs = append(errs, err) - } + // cfg := &gotosl.Config{} + // cfg.Debug = verbose + // err := gotosl.Run(cfg) + // if err != nil { + // errs = append(errs, err) + // } args := []string{"build"} if verbose { args = append(args, "-v") } - err = exec.Major().Run("go", args...) + err := exec.Major().Run("go", args...) if err != nil { errs = append(errs, err) } diff --git a/tensor/matrix/eigen.go b/tensor/matrix/eigen.go index 3b272e24e1..b38e581fc9 100644 --- a/tensor/matrix/eigen.go +++ b/tensor/matrix/eigen.go @@ -20,7 +20,8 @@ import ( // in this 2D square matrix, ordered *lowest* to *highest* across the columns, // i.e., maximum vector is the last column. // The values are the size of one row, ordered *lowest* to *highest*. -// If the input tensor is > 2D, it is treated as a list of 2D matricies. +// If the input tensor is > 2D, it is treated as a list of 2D matricies, +// and parallel threading is used where beneficial. func Eig(a tensor.Tensor) (vecs, vals *tensor.Float64) { vecs = tensor.NewFloat64() vals = tensor.NewFloat64() @@ -36,7 +37,8 @@ func Eig(a tensor.Tensor) (vecs, vals *tensor.Float64) { // in this 2D square matrix, ordered *lowest* to *highest* across the columns, // i.e., maximum vector is the last column. // The values are the size of one row, ordered *lowest* to *highest*. -// If the input tensor is > 2D, it is treated as a list of 2D matricies. +// If the input tensor is > 2D, it is treated as a list of 2D matricies, +// and parallel threading is used where beneficial. func EigOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { if err := StringCheck(a); err != nil { return err @@ -81,19 +83,21 @@ func EigOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { vecs.SetShapeSizes(nr, sz, sz) vals.SetShapeSizes(nr, sz) var errs []error - for r := range nr { - sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) - ma, _ := NewMatrix(sa) - do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) - var eig mat.Eigen - ok := eig.Factorize(ma, mat.EigenRight) - if !ok { - errs = append(errs, errors.New("gonum mat.Eigen Factorize failed")) - } - _ = do - // eig.VectorsTo(do) // todo: requires complex! - // eig.Values(vals.Values[r*sz : (r+1)*sz]) - } + tensor.VectorizeThreaded(ea.DimSize(1)*ea.DimSize(2), // todo: better compute estimate + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) + var eig mat.Eigen + ok := eig.Factorize(ma, mat.EigenRight) + if !ok { + errs = append(errs, errors.New("gonum mat.Eigen Factorize failed")) + } + _ = do + // eig.VectorsTo(do) // todo: requires complex! + // eig.Values(vals.Values[r*sz : (r+1)*sz]) + }) return errors.Join(errs...) } @@ -105,7 +109,8 @@ func EigOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { // i.e., maximum vector is the last column. // The values are the size of one row, ordered *lowest* to *highest*. // Note that Eig produces results in the *opposite* order of [SVD] (which is much faster). -// If the input tensor is > 2D, it is treated as a list of 2D matricies. +// If the input tensor is > 2D, it is treated as a list of 2D matricies, +// and parallel threading is used where beneficial. func EigSym(a tensor.Tensor) (vecs, vals *tensor.Float64) { vecs = tensor.NewFloat64() vals = tensor.NewFloat64() @@ -121,7 +126,8 @@ func EigSym(a tensor.Tensor) (vecs, vals *tensor.Float64) { // i.e., maximum vector is the last column. // The values are the size of one row, ordered *lowest* to *highest*. // Note that Eig produces results in the *opposite* order of [SVD] (which is much faster). -// If the input tensor is > 2D, it is treated as a list of 2D matricies. +// If the input tensor is > 2D, it is treated as a list of 2D matricies, +// and parallel threading is used where beneficial. func EigSymOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { if err := StringCheck(a); err != nil { return err @@ -165,18 +171,20 @@ func EigSymOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { vecs.SetShapeSizes(nr, sz, sz) vals.SetShapeSizes(nr, sz) var errs []error - for r := range nr { - sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) - ma, _ := NewSymmetric(sa) - do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) - var eig mat.EigenSym - ok := eig.Factorize(ma, true) - if !ok { - errs = append(errs, errors.New("gonum mat.Eigen Factorize failed")) - } - eig.VectorsTo(do) - eig.Values(vals.Values[r*sz : (r+1)*sz]) - } + tensor.VectorizeThreaded(ea.DimSize(1)*ea.DimSize(2), // todo: better compute estimate + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewSymmetric(sa) + do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) + var eig mat.EigenSym + ok := eig.Factorize(ma, true) + if !ok { + errs = append(errs, errors.New("gonum mat.Eigen Factorize failed")) + } + eig.VectorsTo(do) + eig.Values(vals.Values[r*sz : (r+1)*sz]) + }) return errors.Join(errs...) } @@ -188,7 +196,8 @@ func EigSymOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { // i.e., maximum vector is the first column. // The values are the size of one row ordered in alignment with the vectors. // Note that SVD produces results in the *opposite* order of [EigSym]. -// If the input tensor is > 2D, it is treated as a list of 2D matricies. +// If the input tensor is > 2D, it is treated as a list of 2D matricies, +// and parallel threading is used where beneficial. func SVD(a tensor.Tensor) (vecs, vals *tensor.Float64) { vecs = tensor.NewFloat64() vals = tensor.NewFloat64() @@ -204,7 +213,8 @@ func SVD(a tensor.Tensor) (vecs, vals *tensor.Float64) { // i.e., maximum vector is the first column. // The values are the size of one row ordered in alignment with the vectors. // Note that SVD produces results in the *opposite* order of [EigSym]. -// If the input tensor is > 2D, it is treated as a list of 2D matricies. +// If the input tensor is > 2D, it is treated as a list of 2D matricies, +// and parallel threading is used where beneficial. func SVDOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { if err := StringCheck(a); err != nil { return err @@ -248,18 +258,20 @@ func SVDOut(a tensor.Tensor, vecs, vals *tensor.Float64) error { vecs.SetShapeSizes(nr, sz, sz) vals.SetShapeSizes(nr, sz) var errs []error - for r := range nr { - sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) - ma, _ := NewSymmetric(sa) - do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) - var eig mat.SVD - ok := eig.Factorize(ma, mat.SVDFull) - if !ok { - errs = append(errs, errors.New("gonum mat.SVD Factorize failed")) - } - eig.UTo(do) - eig.Values(vals.Values[r*sz : (r+1)*sz]) - } + tensor.VectorizeThreaded(ea.DimSize(1)*ea.DimSize(2), // todo: better compute estimate + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewSymmetric(sa) + do, _ := NewDense(vecs.RowTensor(r).(*tensor.Float64)) + var eig mat.SVD + ok := eig.Factorize(ma, mat.SVDFull) + if !ok { + errs = append(errs, errors.New("gonum mat.SVD Factorize failed")) + } + eig.UTo(do) + eig.Values(vals.Values[r*sz : (r+1)*sz]) + }) return errors.Join(errs...) } diff --git a/tensor/matrix/ops.go b/tensor/matrix/ops.go index c7df78bc1f..b496b593e5 100644 --- a/tensor/matrix/ops.go +++ b/tensor/matrix/ops.go @@ -31,6 +31,7 @@ func CallOut2(fun func(a, b tensor.Tensor, out *tensor.Float64) error, a, b tens // Mul performs matrix multiplication, using the following rules based // on the shapes of the relevant tensors. If the tensor shapes are not // suitable, an error is logged (see [MulOut] for a version returning the error). +// N > 2 dimensional cases use parallel threading where beneficial. // - If both arguments are 2-D they are multiplied like conventional matrices. // - If either argument is N-D, N > 2, it is treated as a stack of matrices // residing in the last two indexes and broadcast accordingly. @@ -43,9 +44,9 @@ func Mul(a, b tensor.Tensor) *tensor.Float64 { } // MulOut performs matrix multiplication, into the given output tensor, -// using the following rules based -// on the shapes of the relevant tensors. If the tensor shapes are not -// suitable, a [gonum] [mat.ErrShape] error is returned. +// using the following rules based on the shapes of the relevant tensors. +// If the tensor shapes are not suitable, a [gonum] [mat.ErrShape] error is returned. +// N > 2 dimensional cases use parallel threading where beneficial. // - If both arguments are 2-D they are multiplied like conventional matrices. // The result has shape a.Rows, b.Columns. // - If either argument is N-D, N > 2, it is treated as a stack of matrices @@ -115,12 +116,14 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { mb, _ := NewMatrix(eb) nr := ea.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(1), eb.DimSize(1)) - for r := range nr { - sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) - ma, _ := NewMatrix(sa) - do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) - do.Mul(ma, mb) - } + tensor.VectorizeThreaded(ea.DimSize(1)*ea.DimSize(2)*eb.Len(), + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + do.Mul(ma, mb) + }) case nb > 2 && na == 2: if ea.DimSize(1) != eb.DimSize(1) { return mat.ErrShape @@ -128,12 +131,14 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { ma, _ := NewMatrix(ea) nr := eb.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(0), eb.DimSize(2)) - for r := range nr { - sb := tensor.Reslice(eb, r, tensor.FullAxis, tensor.FullAxis) - mb, _ := NewMatrix(sb) - do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) - do.Mul(ma, mb) - } + tensor.VectorizeThreaded(ea.Len()*eb.DimSize(1)*eb.DimSize(2), + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sb := tensor.Reslice(eb, r, tensor.FullAxis, tensor.FullAxis) + mb, _ := NewMatrix(sb) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + do.Mul(ma, mb) + }) case na > 2 && nb > 2: if ea.DimSize(0) != eb.DimSize(0) { return errors.New("matrix.Mul: a and b input matricies are > 2 dimensional; must have same outer dimension sizes") @@ -143,14 +148,16 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { } nr := ea.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(1), eb.DimSize(2)) - for r := range nr { - sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) - ma, _ := NewMatrix(sa) - sb := tensor.Reslice(eb, r, tensor.FullAxis, tensor.FullAxis) - mb, _ := NewMatrix(sb) - do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) - do.Mul(ma, mb) - } + tensor.VectorizeThreaded(ea.DimSize(1)*ea.DimSize(2)*eb.DimSize(1)*eb.DimSize(2), + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + sb := tensor.Reslice(eb, r, tensor.FullAxis, tensor.FullAxis) + mb, _ := NewMatrix(sb) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + do.Mul(ma, mb) + }) default: return mat.ErrShape } @@ -166,6 +173,8 @@ func MulOut(a, b tensor.Tensor, out *tensor.Float64) error { return nil } +// todo: following should handle N>2 dim case. + // Det returns the determinant of the given tensor. // For a 2D matrix [[a, b], [c, d]] it this is ad - bc. // See also [LogDet] for a version that is more numerically @@ -236,14 +245,16 @@ func InverseOut(a tensor.Tensor, out *tensor.Float64) error { nr := ea.DimSize(0) out.SetShapeSizes(nr, ea.DimSize(1), ea.DimSize(2)) var errs []error - for r := range nr { - sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) - ma, _ := NewMatrix(sa) - do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) - err := do.Inverse(ma) - if err != nil { - errs = append(errs, err) - } - } + tensor.VectorizeThreaded(ea.DimSize(1)*ea.DimSize(2), // todo: better compute estimate + func(tsr ...tensor.Tensor) int { return nr }, + func(r int, tsr ...tensor.Tensor) { + sa := tensor.Reslice(ea, r, tensor.FullAxis, tensor.FullAxis) + ma, _ := NewMatrix(sa) + do, _ := NewDense(out.RowTensor(r).(*tensor.Float64)) + err := do.Inverse(ma) + if err != nil { + errs = append(errs, err) + } + }) return errors.Join(errs...) } From 2dd807ba686ccbd2871b0dba7f60b4e1dd4316dc Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 11:46:31 -0700 Subject: [PATCH 221/311] wrap ndim index exprs in int() so they always work --- goal/transpile/transpile.go | 28 +++++++++++++++++++++------- goal/transpile/transpile_test.go | 12 ++++++------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index cabfddf3cc..7d2297e766 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -321,12 +321,11 @@ func (st *State) TranspileExec(ewords []string, output bool) Tokens { // Returns a positive index to resume processing at, if it is actually an // n-dimensional expr, and -1 if not, in which case the normal process resumes. func (st *State) TranspileGoNDimIndex(toks Tokens, gtoks *Tokens, idIdx, rbIdx int) int { - nc := 0 + var commas []int for i := idIdx + 2; i < rbIdx; i++ { tk := toks[i] if tk.Tok == token.COMMA { - nc++ - break + commas = append(commas, i) } if tk.Tok == token.LPAREN || tk.Tok == token.LBRACE || tk.Tok == token.LBRACK { rp := toks[i:rbIdx].RightMatching() @@ -335,13 +334,16 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, gtoks *Tokens, idIdx, rbIdx i } } } - if nc == 0 { // not multidim + if len(commas) == 0 { // not multidim return -1 } // now we need to determine if it is a Set based on what happens after rb isSet := false stok := token.ILLEGAL n := len(toks) + if toks[n-1].Tok == token.COMMENT { + n-- + } if n-rbIdx > 1 { ntk := toks[rbIdx+1].Tok if ntk == token.ASSIGN || (ntk >= token.ADD_ASSIGN && ntk <= token.QUO_ASSIGN) { @@ -367,13 +369,25 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, gtoks *Tokens, idIdx, rbIdx i gtoks.Add(token.IDENT, fun) gtoks.Add(token.LPAREN) if isSet { - gtoks.AddTokens(toks[rbIdx+2:]...) + gtoks.AddTokens(toks[rbIdx+2 : n]...) + gtoks.Add(token.COMMA) + } + sti := idIdx + 2 + for _, cp := range commas { + gtoks.Add(token.IDENT, "int") + gtoks.Add(token.LPAREN) + gtoks.AddTokens(toks[sti:cp]...) + gtoks.Add(token.RPAREN) gtoks.Add(token.COMMA) + sti = cp + 1 } - gtoks.AddTokens(toks[idIdx+2 : rbIdx]...) + gtoks.Add(token.IDENT, "int") + gtoks.Add(token.LPAREN) + gtoks.AddTokens(toks[sti:rbIdx]...) + gtoks.Add(token.RPAREN) gtoks.Add(token.RPAREN) if isSet { - return n + return len(toks) } else { return rbIdx } diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 955c8429fa..44ccf8df00 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -172,13 +172,13 @@ func TestTranspile(t *testing.T) { {"meta += ln", "meta += ln"}, {"var data map[string]any", "var data map[string]any"}, // non-math-mode tensor indexing: - {"x = a[1,f(2,3)]", `x = a.Value(1, f(2, 3))`}, + {"x = a[1,f(2,3)]", `x = a.Value(int(1), int(f(2, 3)))`}, {"x = a[1]", `x = a[1]`}, {"x = a[f(2,3)]", `x = a[f(2, 3)]`}, - {"a[1,2] = 55", `a.Set(55, 1, 2)`}, - {"a[1,2] += f(2,55)", `a.SetAdd(f(2, 55), 1, 2)`}, - {"a[1,2] *= f(2,55)", `a.SetMul(f(2, 55), 1, 2)`}, - {"Data[idx, Integ] = integ", `Data.Set(integ, idx, Integ)`}, + {"a[1,2] = 55", `a.Set(55, int(1), int(2))`}, + {"a[1,2] += f(2,55)", `a.SetAdd(f(2, 55), int(1), int(2))`}, + {"a[1,2] *= f(2,55)", `a.SetMul(f(2, 55), int(1), int(2))`}, + {"Data[idx, Integ] = integ", `Data.Set(integ, int(idx), int(Integ))`}, } st := NewState() @@ -213,7 +213,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"Data[idx, Integ] = integ", `Data.Set(integ, idx, Integ)`}, + {"Data[idx, Integ] = integ", `Data.Set(integ, int(idx), int(Integ))`}, } st := NewState() st.MathRecord = false From 83fbb1bbed2bb48d44450c571b34989cd5817d06 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 12:52:18 -0700 Subject: [PATCH 222/311] gosl: remove int() wrapper for tensor indexing calls --- goal/gosl/examples/basic/compute.go | 10 ++++++---- goal/gosl/gotosl/nodes.go | 7 +++++++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index b2291ec0b1..8b5604296f 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -11,6 +11,8 @@ import ( "cogentcore.org/core/tensor" ) +//go:generate gosl + //gosl:start //gosl:import "cogentcore.org/core/math32" @@ -49,10 +51,10 @@ type ParamStruct struct { // IntegFromRaw computes integrated value from current raw value func (ps *ParamStruct) IntegFromRaw(idx int) { - integ := Data.Value(idx, Integ) - integ += ps.Dt * (Data.Value(idx, Raw) - integ) - Data.Set(integ, idx, Integ) - Data.Set(math32.FastExp(-integ), idx, Exp) + integ := Data.Value(int(idx), int(Integ)) + integ += ps.Dt * (Data.Value(int(idx), int(Raw)) - integ) + Data.Set(integ, int(idx), int(Integ)) + Data.Set(math32.FastExp(-integ), int(idx), int(Exp)) } // Compute does the main computation. diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 92c13e3732..13d8f2f08e 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1680,6 +1680,13 @@ func (p *printer) tensorMethod(x *ast.CallExpr, vr *Var, methName string) { for i := stArg; i < n; i++ { ag := args[i] p.print("u32", token.LPAREN) + if ce, ok := ag.(*ast.CallExpr); ok { // get rid of int() wrapper from goal n-dim index + if fn, ok := ce.Fun.(*ast.Ident); ok { + if fn.Name == "int" { + ag = ce.Args[0] + } + } + } p.expr(ag) p.print(token.RPAREN) if i < n-1 { From 732b8314ace79c97435cb27ed31625ab2bdc9e97 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 12:55:58 -0700 Subject: [PATCH 223/311] use %.10g for default tensor sprintf format -- removes tiny residual differences from integer values. --- tensor/io.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensor/io.go b/tensor/io.go index dad3d5268b..1c533473b8 100644 --- a/tensor/io.go +++ b/tensor/io.go @@ -184,9 +184,9 @@ func Sprintf(format string, tsr Tensor, maxLen int) string { case tsr.IsString(): format = "%s\t" case reflectx.KindIsInt(tsr.DataType()): - format = "%g\t" + format = "%.10g\t" default: - format = "%g\t" + format = "%.10g\t" } } if tsr.NumDims() == 1 && tsr.DimSize(0) == 1 { // scalar special case From 980319fa40890b87550733e4dcf9e7172d3953d5 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 15:03:46 -0700 Subject: [PATCH 224/311] add bool Value, Set functions --- tensor/bool.go | 14 ++++++++++++++ tensor/stats/metric/misc.go | 14 +++++++++++++- .../cogentcore_org-core-tensor-stats-metric.go | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tensor/bool.go b/tensor/bool.go index 3f4d0efcd2..e1256322c9 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -127,6 +127,20 @@ func (tsr *Bool) RowTensor(row int) Values { return nil } // SetRowTensor not possible with Bool. func (tsr *Bool) SetRowTensor(val Values, row int) {} +///////////////////// Bool + +func (tsr *Bool) Value(i ...int) bool { + return tsr.Values.Index(tsr.shape.IndexTo1D(i...)) +} + +func (tsr *Bool) Set(val bool, i ...int) { + tsr.Values.Set(val, tsr.shape.IndexTo1D(i...)) +} + +func (tsr *Bool) Value1D(i int) bool { return tsr.Values.Index(i) } + +func (tsr *Bool) Set1D(val bool, i int) { tsr.Values.Set(val, i) } + ///////////////////// Strings func (tsr *Bool) String1D(off int) string { diff --git a/tensor/stats/metric/misc.go b/tensor/stats/metric/misc.go index 212373eeb3..cbe831104d 100644 --- a/tensor/stats/metric/misc.go +++ b/tensor/stats/metric/misc.go @@ -18,7 +18,19 @@ import ( // Note: this does _not_ use any existing Indexes for the probe, // but does for the vocab, and the returned index is the logical index // into any existing Indexes. -func ClosestRow(fun any, probe, vocab tensor.Tensor, out tensor.Values) error { +func ClosestRow(fun any, probe, vocab tensor.Tensor) tensor.Values { + return tensor.CallOut2Gen1(ClosestRowOut, fun, probe, vocab) +} + +// ClosestRowOut returns the closest fit between probe pattern and patterns in +// a "vocabulary" tensor with outermost row dimension, using given metric +// function, which must fit the MetricFunc signature. +// The metric *must have the Increasing property*, i.e., larger = further. +// Output is a 1D tensor with 2 elements: the row index and metric value for that row. +// Note: this does _not_ use any existing Indexes for the probe, +// but does for the vocab, and the returned index is the logical index +// into any existing Indexes. +func ClosestRowOut(fun any, probe, vocab tensor.Tensor, out tensor.Values) error { out.SetShapeSizes(2) mfun, err := AsMetricFunc(fun) if err != nil { diff --git a/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go b/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go index b3ae42b314..30220a537c 100644 --- a/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor-stats-metric.go @@ -13,6 +13,7 @@ func init() { "AsMetricFunc": reflect.ValueOf(metric.AsMetricFunc), "AsMetricOutFunc": reflect.ValueOf(metric.AsMetricOutFunc), "ClosestRow": reflect.ValueOf(metric.ClosestRow), + "ClosestRowOut": reflect.ValueOf(metric.ClosestRowOut), "Correlation": reflect.ValueOf(metric.Correlation), "CorrelationOut": reflect.ValueOf(metric.CorrelationOut), "CorrelationOut64": reflect.ValueOf(metric.CorrelationOut64), From 052427ed4d86f95a8344406d3c61b4e6d7005107 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 16:24:48 -0700 Subject: [PATCH 225/311] goal: key fix to decl parsing -- only if brack and paren depth 0 --- goal/math_test.go | 10 ++++++++++ goal/testdata/test.goal | 10 ++++++++++ goal/transpile/transpile.go | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/goal/math_test.go b/goal/math_test.go index 51d4ee7b50..250f196d0d 100644 --- a/goal/math_test.go +++ b/goal/math_test.go @@ -22,6 +22,16 @@ fmt.Println(x) fmt.Println(nd) fmt.Println(sz) fmt.Println(sh) + +type MyStru struct { + Name string + Doc string +} + +var VarCategories = []MyStru{ + {"Act", "basic activation variables, including conductances, current, Vm, spiking"}, + {"Learn", "calcium-based learning variables and other related learning factors"}, +} ` func TestMath(t *testing.T) { diff --git a/goal/testdata/test.goal b/goal/testdata/test.goal index fdd2a23b3e..a93be7ec5b 100644 --- a/goal/testdata/test.goal +++ b/goal/testdata/test.goal @@ -9,3 +9,13 @@ fmt.Println(x) fmt.Println(nd) fmt.Println(sz) fmt.Println(sh) + +type MyStru struct { + Name string + Doc string +} + +var VarCategories = []MyStru{ + {"Act", "basic activation variables, including conductances, current, Vm, spiking"}, + {"Learn", "calcium-based learning variables and other related learning factors"}, +} diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 7d2297e766..f5bfde76b5 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -30,7 +30,7 @@ func (st *State) TranspileLine(code string) string { if st.TypeDepth > 0 && st.BraceDepth == 0 { st.TypeDepth = 0 } - if st.DeclDepth > 0 && st.ParenDepth == 0 { + if st.DeclDepth > 0 && (st.ParenDepth == 0 && st.BraceDepth == 0) { st.DeclDepth = 0 } // logx.PrintlnDebug("depths: ", sh.ParenDepth, sh.BraceDepth, sh.BrackDepth) From 48b5e43be62a998d51ab3c739db62f9ee046b830 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 20:36:19 -0700 Subject: [PATCH 226/311] N-dimensional indexing uses recursive go parsing. --- goal/transpile/transpile.go | 26 ++++++++++++++++++++------ goal/transpile/transpile_test.go | 2 +- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index f5bfde76b5..83b7856d99 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -94,7 +94,7 @@ func (st *State) TranspileLineTokens(code string) Tokens { return st.TranspileExec(ewords, false) case t0.Tok == token.LBRACE: logx.PrintlnDebug("go: { } line") - return st.TranspileGo(toks[1:n-1], code[toks[1].Pos-1:toks[n-1].Pos-1]) + return st.TranspileGoRange(toks, code, 1, n-1) case t0.Tok == token.LBRACK: logx.PrintlnDebug("exec: [ ] line") return st.TranspileExec(ewords, false) // it processes the [ ] @@ -148,6 +148,20 @@ func (st *State) TranspileLineTokens(code string) Tokens { return toks } +// TranspileGoRange returns transpiled tokens assuming Go code, +// for given start, end (exclusive) range of given tokens and code. +// In general the positions in the tokens applies to the _original_ code +// so you should just keep the original code string. However, this is +// needed for a specific case. +func (st *State) TranspileGoRange(toks Tokens, code string, start, end int) Tokens { + codeSt := toks[start].Pos - 1 + codeEd := token.Pos(len(code)) + if end <= len(toks)-1 { + codeEd = toks[end].Pos - 1 + } + return st.TranspileGo(toks[start:end], code[codeSt:codeEd]) +} + // TranspileGo returns transpiled tokens assuming Go code. // Unpacks any encapsulated shell or math expressions. func (st *State) TranspileGo(toks Tokens, code string) Tokens { @@ -187,7 +201,7 @@ func (st *State) TranspileGo(toks Tokens, code string) Tokens { gtoks = append(gtoks, tok) continue } - idx := st.TranspileGoNDimIndex(toks, >oks, i-1, rm+i) + idx := st.TranspileGoNDimIndex(toks, code, >oks, i-1, rm+i) if idx > 0 { i = idx } else { @@ -320,7 +334,7 @@ func (st *State) TranspileExec(ewords []string, output bool) Tokens { // and the ] is at rbIdx. It puts the results in gtoks generated tokens. // Returns a positive index to resume processing at, if it is actually an // n-dimensional expr, and -1 if not, in which case the normal process resumes. -func (st *State) TranspileGoNDimIndex(toks Tokens, gtoks *Tokens, idIdx, rbIdx int) int { +func (st *State) TranspileGoNDimIndex(toks Tokens, code string, gtoks *Tokens, idIdx, rbIdx int) int { var commas []int for i := idIdx + 2; i < rbIdx; i++ { tk := toks[i] @@ -369,21 +383,21 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, gtoks *Tokens, idIdx, rbIdx i gtoks.Add(token.IDENT, fun) gtoks.Add(token.LPAREN) if isSet { - gtoks.AddTokens(toks[rbIdx+2 : n]...) + gtoks.AddTokens(st.TranspileGo(toks[rbIdx+2:n], code)...) gtoks.Add(token.COMMA) } sti := idIdx + 2 for _, cp := range commas { gtoks.Add(token.IDENT, "int") gtoks.Add(token.LPAREN) - gtoks.AddTokens(toks[sti:cp]...) + gtoks.AddTokens(st.TranspileGo(toks[sti:cp], code)...) gtoks.Add(token.RPAREN) gtoks.Add(token.COMMA) sti = cp + 1 } gtoks.Add(token.IDENT, "int") gtoks.Add(token.LPAREN) - gtoks.AddTokens(toks[sti:rbIdx]...) + gtoks.AddTokens(st.TranspileGo(toks[sti:rbIdx], code)...) gtoks.Add(token.RPAREN) gtoks.Add(token.RPAREN) if isSet { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 44ccf8df00..2e997d6889 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -213,7 +213,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"Data[idx, Integ] = integ", `Data.Set(integ, int(idx), int(Integ))`}, + {"Data[Idxs[idx, 25], Integ] = integ", `Data.Set(integ, int(Idxs.Value(int(idx), int(25))), int(Integ))`}, } st := NewState() st.MathRecord = false From 49a99418a944ba8a428e0a8b93656f7c59148471 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 21:12:02 -0700 Subject: [PATCH 227/311] fix comment handling --- goal/transpile/token.go | 5 +++++ goal/transpile/transpile.go | 5 +++++ goal/transpile/transpile_test.go | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/goal/transpile/token.go b/goal/transpile/token.go index 0ba82b3c23..335c63279d 100644 --- a/goal/transpile/token.go +++ b/goal/transpile/token.go @@ -205,6 +205,11 @@ func (tk Tokens) Code() string { } str += tok.String() prvIdent = true + case tok.Tok == token.COMMENT: + if str != "" { + str += " " + } + str += tok.String() case tok.IsGo(): if prvIdent { str += " " diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 83b7856d99..e4ea3854ac 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -355,7 +355,9 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, code string, gtoks *Tokens, i isSet := false stok := token.ILLEGAL n := len(toks) + hasComment := false if toks[n-1].Tok == token.COMMENT { + hasComment = true n-- } if n-rbIdx > 1 { @@ -400,6 +402,9 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, code string, gtoks *Tokens, i gtoks.AddTokens(st.TranspileGo(toks[sti:rbIdx], code)...) gtoks.Add(token.RPAREN) gtoks.Add(token.RPAREN) + if hasComment { + gtoks.AddTokens(toks[len(toks)-1]) + } if isSet { return len(toks) } else { diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 2e997d6889..363f34b642 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -176,9 +176,11 @@ func TestTranspile(t *testing.T) { {"x = a[1]", `x = a[1]`}, {"x = a[f(2,3)]", `x = a[f(2, 3)]`}, {"a[1,2] = 55", `a.Set(55, int(1), int(2))`}, + {"a[1,2] = 55 // and that is good", `a.Set(55, int(1), int(2)) // and that is good`}, {"a[1,2] += f(2,55)", `a.SetAdd(f(2, 55), int(1), int(2))`}, {"a[1,2] *= f(2,55)", `a.SetMul(f(2, 55), int(1), int(2))`}, {"Data[idx, Integ] = integ", `Data.Set(integ, int(idx), int(Integ))`}, + {"Data[Idxs[idx, 25], Integ] = integ", `Data.Set(integ, int(Idxs.Value(int(idx), int(25))), int(Integ))`}, } st := NewState() @@ -213,7 +215,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"Data[Idxs[idx, 25], Integ] = integ", `Data.Set(integ, int(Idxs.Value(int(idx), int(25))), int(Integ))`}, + {"a[1,2] = 55 // and that is good", `a.Set(55, int(1), int(2)) // and that is good`}, } st := NewState() st.MathRecord = false From e2231970c6a05b55a3ae82961470b6f0ff0e6e5e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 8 Oct 2024 21:35:03 -0700 Subject: [PATCH 228/311] fix comment handling for real, good tests --- goal/interpreter/config.go | 6 ++++-- goal/transpile/transpile.go | 6 +++--- goal/transpile/transpile_test.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go index 006fa8f0eb..9312d34714 100644 --- a/goal/interpreter/config.go +++ b/goal/interpreter/config.go @@ -112,10 +112,12 @@ func Build(c *Config) error { } else { fns = fsx.Filenames(".", ".goal") } + curpkg, _ := exec.Minor().Output("go", "list", "./") var errs []error for _, fn := range fns { + fpath := filepath.Join(curpkg, fn) if verbose { - fmt.Println(fn) + fmt.Println(fpath) } ofn := strings.TrimSuffix(fn, filepath.Ext(fn)) + ".go" err := goal.NewGoal().TranspileFile(fn, ofn) @@ -134,7 +136,7 @@ func Build(c *Config) error { if verbose { args = append(args, "-v") } - err := exec.Major().Run("go", args...) + err := exec.Verbose().Run("go", args...) if err != nil { errs = append(errs, err) } diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index e4ea3854ac..9b094e7510 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -402,10 +402,10 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, code string, gtoks *Tokens, i gtoks.AddTokens(st.TranspileGo(toks[sti:rbIdx], code)...) gtoks.Add(token.RPAREN) gtoks.Add(token.RPAREN) - if hasComment { - gtoks.AddTokens(toks[len(toks)-1]) - } if isSet { + if hasComment { + gtoks.AddTokens(toks[len(toks)-1]) + } return len(toks) } else { return rbIdx diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 363f34b642..4f267f701c 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -215,7 +215,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"a[1,2] = 55 // and that is good", `a.Set(55, int(1), int(2)) // and that is good`}, + {" exVm = 0.5 * (nvm + Neurons[Vm, ni, di]) // midpoint for this", `exVm = 0.5 *(nvm + Neurons.Value(int(Vm), int(ni), int(di))) // midpoint for this`}, } st := NewState() st.MathRecord = false From 9d3db3e88ab3e0f94a56d09324f467c8f134c757 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 9 Oct 2024 02:19:03 -0700 Subject: [PATCH 229/311] gosl: support tensor pointers for global vars --- goal/gosl/examples/basic/compute.go | 4 +--- goal/gosl/examples/basic/compute.goal | 2 +- goal/gosl/examples/basic/main.go | 7 ++++--- goal/gosl/gotosl/nodes.go | 12 ++++++++++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 8b5604296f..c9cd27eb66 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -11,8 +11,6 @@ import ( "cogentcore.org/core/tensor" ) -//go:generate gosl - //gosl:start //gosl:import "cogentcore.org/core/math32" @@ -27,7 +25,7 @@ var ( // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. // //gosl:dims 2 - Data tensor.Float32 + Data *tensor.Float32 ) const ( diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal index 89a42c956f..ed3d139af2 100644 --- a/goal/gosl/examples/basic/compute.goal +++ b/goal/gosl/examples/basic/compute.goal @@ -21,7 +21,7 @@ var ( // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. //gosl:dims 2 - Data tensor.Float32 + Data *tensor.Float32 ) const ( diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index bab4d70454..b02a39e2bb 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -33,15 +33,16 @@ func main() { Params = make([]ParamStruct, 1) Params[0].Defaults() - sltensor.SetShapeSizes(&Data, n, 3) // critically, makes GPU compatible Header with strides + Data = tensor.NewFloat32() + sltensor.SetShapeSizes(Data, n, 3) // critically, makes GPU compatible Header with strides nt := Data.Len() for i := range nt { Data.Set1D(rand.Float32(), i) } - var sd tensor.Float32 - sltensor.SetShapeSizes(&sd, n, 3) + sd := tensor.NewFloat32() + sltensor.SetShapeSizes(sd, n, 3) for i := range nt { sd.Set1D(Data.Value1D(i), i) } diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 13d8f2f08e..0760c2012f 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -2498,8 +2498,16 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { } else { sel, ok := vs.Type.(*ast.SelectorExpr) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname)) - continue + st, ok := vs.Type.(*ast.StarExpr) + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname)) + continue + } + sel, ok = st.X.(*ast.SelectorExpr) + if !ok { + errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname)) + continue + } } sid, ok := sel.X.(*ast.Ident) if !ok { From 2cdc6b874661cd8cf831de3d3c373d4e7e9e5739 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 9 Oct 2024 13:11:37 -0700 Subject: [PATCH 230/311] base/generate and enumgen were filtering out generated files, which excludes .go files generated from goal, preventing enums from being processed. removed those exclusions. there should not be any comment-directive code in generated files unless it is intentional. --- base/generate/generate.go | 6 +++--- enums/enumgen/enumgen.go | 12 +++++++++++- enums/enumgen/generator.go | 6 +++--- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/base/generate/generate.go b/base/generate/generate.go index 4a41bd7e3a..3b8023851c 100644 --- a/base/generate/generate.go +++ b/base/generate/generate.go @@ -63,9 +63,9 @@ func PrintHeader(w io.Writer, pkg string, imports ...string) { // stopped and the error value is returned. func Inspect(pkg *packages.Package, f func(n ast.Node) (bool, error)) error { for _, file := range pkg.Syntax { - if ast.IsGenerated(file) { - continue - } + // if ast.IsGenerated(file) { + // continue + // } var terr error var terrNode ast.Node ast.Inspect(file, func(n ast.Node) bool { diff --git a/enums/enumgen/enumgen.go b/enums/enumgen/enumgen.go index 69d71fcc0e..5b394afc15 100644 --- a/enums/enumgen/enumgen.go +++ b/enums/enumgen/enumgen.go @@ -10,8 +10,11 @@ package enumgen import ( "fmt" + "log/slog" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/generate" + "cogentcore.org/core/base/logx" "golang.org/x/tools/go/packages" ) @@ -43,9 +46,16 @@ func ParsePackages(cfg *Config) ([]*packages.Package, error) { func Generate(cfg *Config) error { //types:add pkgs, err := ParsePackages(cfg) if err != nil { + if logx.UserLevel <= slog.LevelInfo { + errors.Log(err) + } return err } - return GeneratePkgs(cfg, pkgs) + err = GeneratePkgs(cfg, pkgs) + if logx.UserLevel <= slog.LevelInfo { + errors.Log(err) + } + return err } // GeneratePkgs generates enum methods using diff --git a/enums/enumgen/generator.go b/enums/enumgen/generator.go index 08babc821c..6a8b9c8b24 100644 --- a/enums/enumgen/generator.go +++ b/enums/enumgen/generator.go @@ -159,9 +159,9 @@ func (g *Generator) Generate() (bool, error) { for _, typ := range g.Types { values := make([]Value, 0, 100) for _, file := range g.Pkg.Syntax { - if ast.IsGenerated(file) { - continue - } + // if ast.IsGenerated(file) { + // continue + // } var terr error ast.Inspect(file, func(n ast.Node) bool { if terr != nil { From 35497ece218fea46a2463987af1f52a2a96a4597 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 9 Oct 2024 14:20:27 -0700 Subject: [PATCH 231/311] goal: dec is same as inc --- goal/transpile/transpile.go | 2 +- goal/transpile/transpile_test.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 9b094e7510..1773ed5170 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -121,7 +121,7 @@ func (st *State) TranspileLineTokens(code string) Tokens { } logx.PrintlnDebug("go keyword") return st.TranspileGo(toks, code) - case toks[n-1].Tok == token.INC: + case toks[n-1].Tok == token.INC || toks[n-1].Tok == token.DEC: return st.TranspileGo(toks, code) case t0pn > 0: // path expr logx.PrintlnDebug("exec: path...") diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 4f267f701c..d06730b8b3 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -168,7 +168,9 @@ func TestTranspile(t *testing.T) { {"type Result struct { JobID string `width:\"60\"`", "type Result struct { JobID string `width:\"60\"`"}, {"func RunInExamples(fun func()) {", "RunInExamples := func(fun func()) {"}, {"ctr++", "ctr++"}, + {"ctr--", "ctr--"}, {"stru.ctr++", "stru.ctr++"}, + {"stru.ctr--", "stru.ctr--"}, {"meta += ln", "meta += ln"}, {"var data map[string]any", "var data map[string]any"}, // non-math-mode tensor indexing: From 9a7d9415f63d852cdad94dddae24b490f8950128 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 10 Oct 2024 12:52:08 -0700 Subject: [PATCH 232/311] gosl & goal have better error reporting and gosl inserts //line comment at top; gosl crash guard; parsing fixes --- goal/gosl/examples/basic/compute.go | 68 ++++++++-------- goal/gosl/examples/basic/gosl.go | 3 +- goal/gosl/examples/basic/shaders/Compute.wgsl | 4 +- goal/gosl/examples/rand/gosl.go | 3 +- goal/gosl/gotosl/gengpu.go | 26 +++--- goal/gosl/gotosl/nodes.go | 80 ++++++++++++------- goal/gosl/gotosl/printer.go | 7 ++ goal/gosl/gotosl/testdata/gosl.golden | 3 +- goal/gosl/gotosl/typegen.go | 4 - goal/transpile/execwords.go | 2 +- goal/transpile/state.go | 4 +- goal/transpile/transpile.go | 6 +- goal/transpile/transpile_test.go | 3 +- 13 files changed, 120 insertions(+), 93 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index c9cd27eb66..95de2c7af0 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -1,5 +1,5 @@ // Code generated by "goal build"; DO NOT EDIT. - +//line compute.goal:1 // Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,68 +7,68 @@ package main import ( - "cogentcore.org/core/math32" - "cogentcore.org/core/tensor" +"cogentcore.org/core/math32" +"cogentcore.org/core/tensor" ) +goal.Run("garbage") + //gosl:start //gosl:import "cogentcore.org/core/math32" //gosl:vars var ( - // Params are the parameters for the computation. - // - //gosl:read-only - Params []ParamStruct - - // Data is the data on which the computation operates. - // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. - // - //gosl:dims 2 - Data *tensor.Float32 +// Params are the parameters for the computation. +//gosl:read-only +Params[]ParamStruct + +// Data is the data on which the computation operates. +// 2D: outer index is data, inner index is: Raw, Integ, Exp vars. +//gosl:dims 2 +Data *tensor.Float32 ) const ( - Raw int = iota - Integ - Exp +Raw int = iota +Integ +Exp ) // ParamStruct has the test params type ParamStruct struct { - // rate constant in msec - Tau float32 +// rate constant in msec +Tau float32 - // 1/Tau - Dt float32 +// 1/Tau +Dt float32 - pad float32 - pad1 float32 +pad float32 +pad1 float32 } // IntegFromRaw computes integrated value from current raw value -func (ps *ParamStruct) IntegFromRaw(idx int) { - integ := Data.Value(int(idx), int(Integ)) - integ += ps.Dt * (Data.Value(int(idx), int(Raw)) - integ) - Data.Set(integ, int(idx), int(Integ)) - Data.Set(math32.FastExp(-integ), int(idx), int(Exp)) +func(ps *ParamStruct)IntegFromRaw(idx int) { +integ := Data.Value(int(idx), int(Integ)) +integ += ps.Dt *(Data.Value(int(idx), int(Raw)) - integ) +Data.Set(integ, int(idx), int(Integ)) +Data.Set(math32.FastExp( - integ), int(idx), int(Exp)) } // Compute does the main computation. -func Compute(i uint32) { //gosl:kernel - Params[0].IntegFromRaw(int(i)) +func Compute(i uint32) { //gosl:kernel +Params[0].IntegFromRaw(int(i)) } //gosl:end // note: only core compute code needs to be in shader -- all init is done CPU-side -func (ps *ParamStruct) Defaults() { - ps.Tau = 5 - ps.Update() +func(ps *ParamStruct)Defaults() { +ps.Tau = 5 +ps.Update() } -func (ps *ParamStruct) Update() { - ps.Dt = 1.0 / ps.Tau +func(ps *ParamStruct)Update() { +ps.Dt = 1.0 / ps.Tau } diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index d1a5889eb2..5170fc2bac 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -42,6 +42,7 @@ func GPUInit() { { sgp := vars.AddGroup(gpu.Storage) var vr *gpu.Var + _ = vr vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) vr.ReadOnly = true vr = sgp.Add("Data", gpu.Float32, 1, gpu.ComputeShader) @@ -103,7 +104,6 @@ func RunOneCompute(n int, syncVars ...GPUVars) { RunComputeCPU(n) } } - // RunDone must be called after Run* calls to start compute kernels. // This actually submits the kernel jobs to the GPU, and adds commands // to synchronize the given variables back from the GPU to the CPU. @@ -120,7 +120,6 @@ func RunDone(syncVars ...GPUVars) { SyncFromGPU(syncVars...) } - // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { sy := GPUSystem diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index ddee8d11e0..89beed0bf9 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -1,7 +1,7 @@ // Code generated by "gosl"; DO NOT EDIT // kernel: Compute -// // Params are the parameters for the computation. // +// // Params are the parameters for the computation. @group(0) @binding(0) var Params: array; @group(0) @binding(1) @@ -24,12 +24,10 @@ fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { //gosl:vars // Params are the parameters for the computation. -// //gosl:read-only // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. -// //gosl:dims 2 const Raw: i32 = 0; diff --git a/goal/gosl/examples/rand/gosl.go b/goal/gosl/examples/rand/gosl.go index ae8340b836..1839cc2218 100644 --- a/goal/gosl/examples/rand/gosl.go +++ b/goal/gosl/examples/rand/gosl.go @@ -42,6 +42,7 @@ func GPUInit() { { sgp := vars.AddGroup(gpu.Storage) var vr *gpu.Var + _ = vr vr = sgp.AddStruct("Seed", int(unsafe.Sizeof(Seeds{})), 1, gpu.ComputeShader) vr.ReadOnly = true vr = sgp.AddStruct("Data", int(unsafe.Sizeof(Rnds{})), 1, gpu.ComputeShader) @@ -103,7 +104,6 @@ func RunOneCompute(n int, syncVars ...GPUVars) { RunComputeCPU(n) } } - // RunDone must be called after Run* calls to start compute kernels. // This actually submits the kernel jobs to the GPU, and adds commands // to synchronize the given variables back from the GPU to the CPU. @@ -120,7 +120,6 @@ func RunDone(syncVars ...GPUVars) { SyncFromGPU(syncVars...) } - // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { sy := GPUSystem diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index c590a164fe..e5ab2f2ab8 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -134,7 +134,7 @@ func (st *State) GenGPUSystemInit(sy *System) string { gtyp = "gpu.Uniform" } b.WriteString(fmt.Sprintf("\t\t\tsgp := vars.AddGroup(%s)\n", gtyp)) - b.WriteString("\t\t\tvar vr *gpu.Var\n") + b.WriteString("\t\t\tvar vr *gpu.Var\n\t\t\t_ = vr\n") for _, vr := range gp.Vars { if vr.Tensor { typ := strings.TrimPrefix(vr.Type, "tensor.") @@ -207,30 +207,24 @@ func RunOne%[1]s(n int, syncVars ...GPUVars) { Run%[1]sCPU(n) } } - -// RunDone%[3]s must be called after Run* calls to start compute kernels. +` + // 1 = sysname (blank for 1 default), 2 = system var + runDone := `// RunDone%[1]s must be called after Run* calls to start compute kernels. // This actually submits the kernel jobs to the GPU, and adds commands // to synchronize the given variables back from the GPU to the CPU. // After this function completes, the GPU results will be available in // the specified variables. -func RunDone%[3]s(syncVars ...GPUVars) { +func RunDone%[1]s(syncVars ...GPUVars) { if !UseGPU { return } sy := %[2]s sy.ComputeEncoder.End() - %[3]sReadFromGPU(syncVars...) + %[1]sReadFromGPU(syncVars...) sy.EndComputePass() - %[3]sSyncFromGPU(syncVars...) + %[1]sSyncFromGPU(syncVars...) } -` - - for _, kn := range sy.Kernels { - b.WriteString(fmt.Sprintf(run, kn.Name, syvar, synm)) - } - - toGPU := ` // %[1]sToGPU copies given variables to the GPU for the system. func %[1]sToGPU(vars ...GPUVars) { sy := %[2]s @@ -238,7 +232,11 @@ func %[1]sToGPU(vars ...GPUVars) { for _, vr := range vars { switch vr { ` - b.WriteString(fmt.Sprintf(toGPU, synm, syvar)) + + for _, kn := range sy.Kernels { + b.WriteString(fmt.Sprintf(run, kn.Name, syvar, synm)) + } + b.WriteString(fmt.Sprintf(runDone, synm, syvar)) for gi, gp := range sy.Groups { for _, vr := range gp.Vars { diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 0760c2012f..5783a5813e 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -27,8 +27,6 @@ import ( "strings" "unicode" "unicode/utf8" - - "cogentcore.org/core/base/errors" ) // Formatting issues: @@ -490,7 +488,7 @@ func (p *printer) assignRwArgs(rwargs []rwArg) { p.expr(rw.idx) p.print(token.ASSIGN) tv := rw.tmpVar - if tv[0] == '&' { + if len(tv) > 0 && tv[0] == '&' { tv = tv[1:] } p.print(tv) @@ -620,7 +618,13 @@ func (p *printer) pathType(x *ast.SelectorExpr) (types.Type, error) { return nil, fmt.Errorf("gosl pathType: path not a pure selector path") } np := len(paths) - bt, err := getStructType(p.getIdType(paths[np-1])) + idt := p.getIdType(paths[np-1]) + if idt == nil { + err := fmt.Errorf("gosl pathType ERROR: cannot find type for name: %q", paths[np-1].Name) + p.userError(err) + return nil, err + } + bt, err := p.getStructType(idt) if err != nil { return nil, err } @@ -633,7 +637,7 @@ func (p *printer) pathType(x *ast.SelectorExpr) (types.Type, error) { if pi == 0 { return f.Type(), nil } else { - bt, err = getStructType(f.Type()) + bt, err = p.getStructType(f.Type()) if err != nil { return nil, err } @@ -1515,7 +1519,7 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa break } err = fmt.Errorf("gosl methodPath ERROR: path for method call must be simple list of fields, not %#v:", cur.X) - errors.Log(err) + p.userError(err) return } if p.isPtrArg(baseRecv) { @@ -1523,26 +1527,32 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa } else { recvPath = "&" + baseRecv.Name } - bt, err := getStructType(p.getIdType(baseRecv)) + idt := p.getIdType(baseRecv) + if idt == nil { + err = fmt.Errorf("gosl methodPath ERROR: cannot find type for name: %q", baseRecv.Name) + p.userError(err) + return + } + bt, err := p.getStructType(idt) if err != nil { return } curt := bt np := len(paths) for pi := np - 1; pi >= 0; pi-- { - p := paths[pi] - recvPath += "." + p - f := fieldByName(curt, p) + pth := paths[pi] + recvPath += "." + pth + f := fieldByName(curt, pth) if f == nil { - err = fmt.Errorf("gosl ERROR: field not found %q in type: %q:", p, curt.String()) - errors.Log(err) + err = fmt.Errorf("gosl ERROR: field not found %q in type: %q:", pth, curt.String()) + p.userError(err) return } if pi == 0 { pathType = f.Type() recvType = getLocalTypeName(f.Type()) } else { - curt, err = getStructType(f.Type()) + curt, err = p.getStructType(f.Type()) if err != nil { return } @@ -1574,7 +1584,7 @@ func getLocalTypeName(typ types.Type) string { return nm } -func getStructType(typ types.Type) (*types.Struct, error) { +func (p *printer) getStructType(typ types.Type) (*types.Struct, error) { typ = typ.Underlying() if st, ok := typ.(*types.Struct); ok { return st, nil @@ -1592,10 +1602,11 @@ func getStructType(typ types.Type) (*types.Struct, error) { } } err := fmt.Errorf("gosl ERROR: type is not a struct and it should be: %q %+t", typ.String(), typ) - return nil, errors.Log(err) + p.userError(err) + return nil, err } -func getNamedType(typ types.Type) (*types.Named, error) { +func (p *printer) getNamedType(typ types.Type) (*types.Named, error) { if nmd, ok := typ.(*types.Named); ok { return nmd, nil } @@ -1613,7 +1624,8 @@ func getNamedType(typ types.Type) (*types.Named, error) { } } err := fmt.Errorf("gosl ERROR: type is not a named type: %q %+t", typ.String(), typ) - return nil, errors.Log(err) + p.userError(err) + return nil, err } // gosl: globalVar looks up whether the id in an IndexExpr is a global gosl variable. @@ -1631,7 +1643,12 @@ func (p *printer) globalVar(idx *ast.IndexExpr) (isGlobal bool, tmpVar, typName isReadOnly = gvr.ReadOnly tmpVar = strings.ToLower(id.Name) vtyp = p.getIdType(id) - nmd, err := getNamedType(vtyp) + if vtyp == nil { + err := fmt.Errorf("gosl globalVar ERROR: cannot find type for name: %q", id.Name) + p.userError(err) + return + } + nmd, err := p.getNamedType(vtyp) if err == nil { vtyp = nmd } @@ -1648,7 +1665,7 @@ func (p *printer) methodIndex(idx *ast.IndexExpr) (recvPath, recvType string, pa id, ok := idx.X.(*ast.Ident) if !ok { err = fmt.Errorf("gosl methodIndex ERROR: must have a recv variable identifier, not %#v:", idx.X) - errors.Log(err) + p.userError(err) return } isGlobal, tmpVar, typName, vtyp, isReadOnly := p.globalVar(idx) @@ -1746,7 +1763,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { } } else { err := fmt.Errorf("gosl methodExpr ERROR: path expression for method call must be simple list of fields, not %#v:", path.X) - errors.Log(err) + p.userError(err) return } args := x.Args @@ -2454,6 +2471,7 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, tok token.Token) { func (p *printer) systemVars(d *ast.GenDecl, sysname string) { sy := p.GoToSL.System(sysname) var gp *Group + var err error for _, s := range d.Specs { vs := s.(*ast.ValueSpec) dirs, docs := p.findDirective(vs.Doc) @@ -2484,14 +2502,16 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { sy.Groups = append(sy.Groups, gp) } if len(vs.Names) != 1 { - errors.Log(fmt.Errorf("gosl: system %q: vars must have only 1 variable per line", sysname)) + err = fmt.Errorf("gosl: system %q: vars must have only 1 variable per line", sysname) + p.userError(err) } nm := vs.Names[0].Name typ := "" if sl, ok := vs.Type.(*ast.ArrayType); ok { id, ok := sl.Elt.(*ast.Ident) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: Var type not recognized: %#v", sysname, sl.Elt)) + err = fmt.Errorf("gosl: system %q: Var type not recognized: %#v", sysname, sl.Elt) + p.userError(err) continue } typ = "[]" + id.Name @@ -2500,18 +2520,22 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { if !ok { st, ok := vs.Type.(*ast.StarExpr) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname)) + err = fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname) + p.userError(err) + continue } sel, ok = st.X.(*ast.SelectorExpr) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname)) + err = fmt.Errorf("gosl: system %q: Var types must be []slices or tensor.Float32, tensor.Uint32", sysname) + p.userError(err) continue } } sid, ok := sel.X.(*ast.Ident) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: Var type selector is not recognized: %#v", sysname, sel.X)) + err = fmt.Errorf("gosl: system %q: Var type selector is not recognized: %#v", sysname, sel.X) + p.userError(err) continue } typ = sid.Name + "." + sel.Sel.Name @@ -2521,12 +2545,14 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { vr.Tensor = true dstr, ok := directiveAfter(dirs, "dims") if !ok { - errors.Log(fmt.Errorf("gosl: system %q: variable %q tensor vars require //gosl:dims to specify number of dimensions", sysname, nm)) + err = fmt.Errorf("gosl: system %q: variable %q tensor vars require //gosl:dims to specify number of dimensions", sysname, nm) + p.userError(err) continue } dims, err := strconv.Atoi(dstr) if !ok { - errors.Log(fmt.Errorf("gosl: system %q: variable %q tensor dims parse error: %s", sysname, nm, err.Error())) + err = fmt.Errorf("gosl: system %q: variable %q tensor dims parse error: %s", sysname, nm, err.Error()) + p.userError(err) } vr.SetTensorKind() vr.TensorDims = dims diff --git a/goal/gosl/gotosl/printer.go b/goal/gosl/gotosl/printer.go index 1c1a78aca4..cafe7e895e 100644 --- a/goal/gosl/gotosl/printer.go +++ b/goal/gosl/gotosl/printer.go @@ -23,6 +23,7 @@ import ( "text/tabwriter" "unicode" + "cogentcore.org/core/base/fsx" "golang.org/x/tools/go/packages" ) @@ -120,6 +121,12 @@ func (p *printer) internalError(msg ...any) { } } +func (p *printer) userError(err error) { + fname := fsx.DirAndFile(p.pos.String()) + fmt.Print(fname + ": ") + fmt.Println(err.Error()) +} + // commentsHaveNewline reports whether a list of comments belonging to // an *ast.CommentGroup contains newlines. Because the position information // may only be partially correct, we also have to read the comment text. diff --git a/goal/gosl/gotosl/testdata/gosl.golden b/goal/gosl/gotosl/testdata/gosl.golden index 70455d54fa..93266aa626 100644 --- a/goal/gosl/gotosl/testdata/gosl.golden +++ b/goal/gosl/gotosl/testdata/gosl.golden @@ -42,6 +42,7 @@ func GPUInit() { { sgp := vars.AddGroup(gpu.Storage) var vr *gpu.Var + _ = vr vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) vr.ReadOnly = true vr = sgp.AddStruct("Data", int(unsafe.Sizeof(DataStruct{})), 1, gpu.ComputeShader) @@ -103,7 +104,6 @@ func RunOneCompute(n int, syncVars ...GPUVars) { RunComputeCPU(n) } } - // RunDone must be called after Run* calls to start compute kernels. // This actually submits the kernel jobs to the GPU, and adds commands // to synchronize the given variables back from the GPU to the CPU. @@ -120,7 +120,6 @@ func RunDone(syncVars ...GPUVars) { SyncFromGPU(syncVars...) } - // ToGPU copies given variables to the GPU for the system. func ToGPU(vars ...GPUVars) { sy := GPUSystem diff --git a/goal/gosl/gotosl/typegen.go b/goal/gosl/gotosl/typegen.go index 947b65642f..386f37e9df 100644 --- a/goal/gosl/gotosl/typegen.go +++ b/goal/gosl/gotosl/typegen.go @@ -102,10 +102,6 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.fi var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getLocalTypeName", Args: []string{"typ"}, Returns: []string{"string"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getStructType", Args: []string{"typ"}, Returns: []string{"Struct", "error"}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getNamedType", Args: []string{"typ"}, Returns: []string{"Named", "error"}}) - var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isTypeName", Args: []string{"x"}, Returns: []string{"bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.stripParens", Args: []string{"x"}, Returns: []string{"Expr"}}) diff --git a/goal/transpile/execwords.go b/goal/transpile/execwords.go index 74bbca6241..6a66cb1ace 100644 --- a/goal/transpile/execwords.go +++ b/goal/transpile/execwords.go @@ -162,7 +162,7 @@ func ExecWords(ln string) ([]string, error) { // ExecWordIsCommand returns true if given exec word is a command-like string // (excluding any paths) func ExecWordIsCommand(f string) bool { - if strings.Contains(f, "(") || strings.Contains(f, "=") { + if strings.Contains(f, "(") || strings.Contains(f, "[") || strings.Contains(f, "=") { return false } return true diff --git a/goal/transpile/state.go b/goal/transpile/state.go index 9317158e5c..e953eea3dd 100644 --- a/goal/transpile/state.go +++ b/goal/transpile/state.go @@ -129,7 +129,7 @@ func main() { src := st.Code() res := []byte(src) bsrc := res - gen := "// Code generated by \"goal build\"; DO NOT EDIT.\n\n" + gen := fmt.Sprintf("// Code generated by \"goal build\"; DO NOT EDIT.\n//line %s:1\n", in) if hasPackage { bsrc = []byte(gen + src) res, err = format.Source(bsrc) @@ -139,7 +139,7 @@ func main() { } if err != nil { res = bsrc - slog.Error(err.Error()) + fmt.Println(err.Error()) } else { err = st.DepthError() } diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 1773ed5170..13bcb2c4d0 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -61,7 +61,7 @@ func (st *State) TranspileLineTokens(code string) Tokens { st.AddError(err) return nil } - logx.PrintlnDebug("\n########## line:\n", code, "\nTokens:\n", toks.String(), "\nWords:\n", ewords) + logx.PrintlnDebug("\n########## line:\n", code, "\nTokens:", len(toks), "\n", toks.String(), "\nWords:", len(ewords), "\n", ewords) if toks[0].Tok == token.TYPE { st.TypeDepth++ @@ -122,6 +122,7 @@ func (st *State) TranspileLineTokens(code string) Tokens { logx.PrintlnDebug("go keyword") return st.TranspileGo(toks, code) case toks[n-1].Tok == token.INC || toks[n-1].Tok == token.DEC: + logx.PrintlnDebug("go ++ / --") return st.TranspileGo(toks, code) case t0pn > 0: // path expr logx.PrintlnDebug("exec: path...") @@ -138,6 +139,9 @@ func (st *State) TranspileLineTokens(code string) Tokens { case f0exec && en > 1 && ewords[0] != "set" && toks.IsAssignExpr(): logx.PrintlnDebug("go: assignment or defn") return st.TranspileGo(toks, code) + case f0exec && en > 1 && ewords[0] != "set" && toks.IsAssignExpr(): + logx.PrintlnDebug("go: assignment or defn") + return st.TranspileGo(toks, code) case f0exec: // now any ident logx.PrintlnDebug("exec: ident..") return st.TranspileExec(ewords, false) diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index d06730b8b3..902c7ffebb 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -183,6 +183,7 @@ func TestTranspile(t *testing.T) { {"a[1,2] *= f(2,55)", `a.SetMul(f(2, 55), int(1), int(2))`}, {"Data[idx, Integ] = integ", `Data.Set(integ, int(idx), int(Integ))`}, {"Data[Idxs[idx, 25], Integ] = integ", `Data.Set(integ, int(Idxs.Value(int(idx), int(25))), int(Integ))`}, + {"Layers[NeuronIxs[NrnLayIndex, ni]].GatherSpikes(&Ctx[0], ni, di)", `Layers[NeuronIxs.Value(int(NrnLayIndex), int(ni))].GatherSpikes( & Ctx[0], ni, di)`}, } st := NewState() @@ -217,7 +218,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {" exVm = 0.5 * (nvm + Neurons[Vm, ni, di]) // midpoint for this", `exVm = 0.5 *(nvm + Neurons.Value(int(Vm), int(ni), int(di))) // midpoint for this`}, + {"Layers[NeuronIxs[NrnLayIndex, ni]].GatherSpikes(&Ctx[0], ni, di)", `Layers[NeuronIxs.Value(int(NrnLayIndex), int(ni))].GatherSpikes( & Ctx[0], ni, di)`}, } st := NewState() st.MathRecord = false From 684904ff6bf4ee2ba54e5d4c6addf95cffd3bbb1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 11 Oct 2024 03:56:02 -0700 Subject: [PATCH 233/311] gosl: auto Get* functions for global vars, that GPU re-interprets, to use within a block. --- goal/gosl/examples/basic/compute.go | 67 ++++++++++--------- goal/gosl/examples/basic/compute.goal | 5 +- goal/gosl/examples/basic/gosl.go | 9 ++- goal/gosl/examples/basic/shaders/Compute.wgsl | 11 ++- goal/gosl/examples/rand/gosl.go | 16 +++++ goal/gosl/gotosl/gengpu.go | 18 +++++ goal/gosl/gotosl/gotosl.go | 43 ++++++++++++ goal/gosl/gotosl/nodes.go | 64 ++++++++++++++++-- goal/gosl/gotosl/testdata/Compute.golden | 7 +- goal/gosl/gotosl/testdata/basic.go | 19 ++++-- goal/gosl/gotosl/testdata/gosl.golden | 16 +++++ goal/gosl/gotosl/typegen.go | 4 +- math32/fastexp.go | 4 +- math32/minmax/avgmax.go | 4 +- math32/minmax/minmax32.go | 4 +- 15 files changed, 232 insertions(+), 59 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 95de2c7af0..f071751cad 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -7,68 +7,69 @@ package main import ( -"cogentcore.org/core/math32" -"cogentcore.org/core/tensor" + "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" ) -goal.Run("garbage") - //gosl:start //gosl:import "cogentcore.org/core/math32" //gosl:vars var ( -// Params are the parameters for the computation. -//gosl:read-only -Params[]ParamStruct - -// Data is the data on which the computation operates. -// 2D: outer index is data, inner index is: Raw, Integ, Exp vars. -//gosl:dims 2 -Data *tensor.Float32 + // Params are the parameters for the computation. + // + // gosl:read-only + Params []ParamStruct + + // Data is the data on which the computation operates. + // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + // + //gosl:dims 2 + Data *tensor.Float32 ) const ( -Raw int = iota -Integ -Exp + Raw int = iota + Integ + Exp ) // ParamStruct has the test params type ParamStruct struct { -// rate constant in msec -Tau float32 + // rate constant in msec + Tau float32 -// 1/Tau -Dt float32 + // 1/Tau + Dt float32 -pad float32 -pad1 float32 + pad float32 + pad1 float32 } // IntegFromRaw computes integrated value from current raw value -func(ps *ParamStruct)IntegFromRaw(idx int) { -integ := Data.Value(int(idx), int(Integ)) -integ += ps.Dt *(Data.Value(int(idx), int(Raw)) - integ) -Data.Set(integ, int(idx), int(Integ)) -Data.Set(math32.FastExp( - integ), int(idx), int(Exp)) +func (ps *ParamStruct) IntegFromRaw(idx int) { + integ := Data.Value(int(idx), int(Integ)) + integ += ps.Dt * (Data.Value(int(idx), int(Raw)) - integ) + Data.Set(integ, int(idx), int(Integ)) + Data.Set(math32.FastExp(-integ), int(idx), int(Exp)) } // Compute does the main computation. -func Compute(i uint32) { //gosl:kernel -Params[0].IntegFromRaw(int(i)) +func Compute(i uint32) { //gosl:kernel + params := GetParams(0) + params.IntegFromRaw(int(i)) } //gosl:end // note: only core compute code needs to be in shader -- all init is done CPU-side -func(ps *ParamStruct)Defaults() { -ps.Tau = 5 -ps.Update() +func (ps *ParamStruct) Defaults() { + ps.Tau = 5 + ps.Update() } -func(ps *ParamStruct)Update() { -ps.Dt = 1.0 / ps.Tau +func (ps *ParamStruct) Update() { + ps.Dt = 1.0 / ps.Tau } diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal index ed3d139af2..9cc4ec19b4 100644 --- a/goal/gosl/examples/basic/compute.goal +++ b/goal/gosl/examples/basic/compute.goal @@ -15,7 +15,7 @@ import ( //gosl:vars var ( // Params are the parameters for the computation. - //gosl:read-only + // gosl:read-only Params []ParamStruct // Data is the data on which the computation operates. @@ -53,7 +53,8 @@ func (ps *ParamStruct) IntegFromRaw(idx int) { // Compute does the main computation. func Compute(i uint32) { //gosl:kernel - Params[0].IntegFromRaw(int(i)) + params := GetParams(0) + params.IntegFromRaw(int(i)) } //gosl:end diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index 5170fc2bac..2752aa041b 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -44,7 +44,6 @@ func GPUInit() { var vr *gpu.Var _ = vr vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) - vr.ReadOnly = true vr = sgp.Add("Data", gpu.Float32, 1, gpu.ComputeShader) sgp.SetNValues(1) } @@ -169,3 +168,11 @@ func SyncFromGPU(vars ...GPUVars) { } } } + +// GetParams returns a pointer to the given global variable: +// [Params] []ParamStruct at given index. +// To ensure that values are updated on the GPU, you must call [SetParams]. +// after all changes have been made. +func GetParams(idx uint32) *ParamStruct { + return &Params[idx] +} diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 89beed0bf9..597f630b2d 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -1,7 +1,7 @@ // Code generated by "gosl"; DO NOT EDIT // kernel: Compute -// // Params are the parameters for the computation. +// // Params are the parameters for the computation. // // gosl:read-only @group(0) @binding(0) var Params: array; @group(0) @binding(1) @@ -24,10 +24,12 @@ fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { //gosl:vars // Params are the parameters for the computation. -//gosl:read-only +// +// gosl:read-only // Data is the data on which the computation operates. // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. +// //gosl:dims 2 const Raw: i32 = 0; @@ -58,7 +60,10 @@ fn ParamStruct_IntegFromRaw(ps: ptr, idx: i32) { // Compute does the main computation. fn Compute(i: u32) { //gosl:kernel - var params=Params[0]; ParamStruct_IntegFromRaw(¶ms, i32(i)); + var params = Params[0]; + ParamStruct_IntegFromRaw(¶ms, i32(i)); +Params[0] = params; + } diff --git a/goal/gosl/examples/rand/gosl.go b/goal/gosl/examples/rand/gosl.go index 1839cc2218..7c297442cf 100644 --- a/goal/gosl/examples/rand/gosl.go +++ b/goal/gosl/examples/rand/gosl.go @@ -169,3 +169,19 @@ func SyncFromGPU(vars ...GPUVars) { } } } + +// GetSeed returns a pointer to the given global variable: +// [Seed] []Seeds at given index. +// To ensure that values are updated on the GPU, you must call [SetSeed]. +// after all changes have been made. +func GetSeed(idx int) *Seeds { + return &Seed[idx] +} + +// GetData returns a pointer to the given global variable: +// [Data] []Rnds at given index. +// To ensure that values are updated on the GPU, you must call [SetData]. +// after all changes have been made. +func GetData(idx int) *Rnds { + return &Data[idx] +} diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index e5ab2f2ab8..97f39f3fea 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -296,5 +296,23 @@ func %[1]sSyncFromGPU(vars ...GPUVars) { } b.WriteString("\t\t}\n\t}\n}\n") + getFun := ` +// Get%[1]s returns a pointer to the given global variable: +// [%[1]s] []%[2]s at given index. +// To ensure that values are updated on the GPU, you must call [Set%[1]s]. +// after all changes have been made. +func Get%[1]s(idx uint32) *%[2]s { + return &%[1]s[idx] +} +` + for _, gp := range sy.Groups { + for _, vr := range gp.Vars { + if vr.Tensor { + continue + } + b.WriteString(fmt.Sprintf(getFun, vr.Name, vr.SLType())) + } + } + return b.String() } diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index ccbd55d327..a52b4f44a5 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -6,12 +6,14 @@ package gotosl import ( "fmt" + "go/ast" "os" "path/filepath" "reflect" "strings" "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/stack" ) // System represents a ComputeSystem, and its kernels and variables. @@ -131,6 +133,16 @@ type File struct { Lines [][]byte } +// GetGlobalVar holds GetVar expression, to Set variable back when done. +type GetGlobalVar struct { + // global variable + Var *Var + // name of temporary variable + TmpVar string + // index passed to the Get function + IdxExpr ast.Expr +} + // State holds the current Go -> WGSL processing state. type State struct { // Config options. @@ -156,6 +168,9 @@ type State struct { // There is an initial "Default" system when system is not specified. Systems map[string]*System + // GetFuncs is a map of GetVar, SetVar function names for global vars. + GetFuncs map[string]*Var + // SLImportFiles are all the extracted and translated WGSL files in shaders/imports, // which are copied into the generated shader kernel files. SLImportFiles []*File @@ -165,6 +180,10 @@ type State struct { // ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation. ExcludeMap map[string]bool + + // GetVarStack is a stack per function definition of GetVar variables + // that need to be set at the end. + GetVarStack stack.Stack[map[string]*GetGlobalVar] } func (st *State) Init(cfg *Config) { @@ -248,3 +267,27 @@ func (st *State) GlobalVar(vrnm string) *Var { } return nil } + +// GetTempVar returns temp var for global variable of given name, if found. +func (st *State) GetTempVar(vrnm string) *GetGlobalVar { + if st == nil || st.GetVarStack == nil { + return nil + } + gvars := st.GetVarStack.Peek() + return gvars[vrnm] +} + +// VarsAdded is called when a set of vars has been added; update relevant maps etc. +func (st *State) VarsAdded() { + st.GetFuncs = make(map[string]*Var) + for _, sy := range st.Systems { + for _, gp := range sy.Groups { + for _, vr := range gp.Vars { + if vr.Tensor { + continue + } + st.GetFuncs["Get"+vr.Name] = vr + } + } + } +} diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 5783a5813e..f431593dd3 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -513,6 +513,11 @@ func (p *printer) goslFixArgs(args []ast.Expr, params *types.Tuple) ([]ast.Expr, nn := normalizedNumber(x) nn.Value = tnm + "(" + nn.Value + ")" ags[i] = nn + case *ast.Ident: + if gvar := p.GoToSL.GetTempVar(x.Name); gvar != nil { + x.Name = "&" + x.Name + ags[i] = x + } case *ast.IndexExpr: isGlobal, tmpVar, _, _, isReadOnly := p.globalVar(x) if isGlobal { @@ -1301,7 +1306,7 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { if len(x.Args) > 1 { depth++ } - + fid, isid := x.Fun.(*ast.Ident) // Conversions to literal function types or <-chan // types require parentheses around the type. paren := false @@ -1320,7 +1325,7 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { } args := x.Args var rwargs []rwArg - if fid, ok := x.Fun.(*ast.Ident); ok { + if isid { if obj, ok := p.pkg.TypesInfo.Uses[fid]; ok { if ft, ok := obj.(*types.Func); ok { sig := ft.Type().(*types.Signature) @@ -1660,6 +1665,34 @@ func (p *printer) globalVar(idx *ast.IndexExpr) (isGlobal bool, tmpVar, typName return } +// gosl: replace GetVar function call with assignment of local var +func (p *printer) getGlobalVar(ae *ast.AssignStmt, gvr *Var) { + tmpVar := ae.Lhs[0].(*ast.Ident).Name + cf := ae.Rhs[0].(*ast.CallExpr) + p.print("var", blank, tmpVar, blank, token.ASSIGN, blank, gvr.Name, token.LBRACK) + p.expr(cf.Args[0]) + p.print(token.RBRACK, token.SEMICOLON) + gvars := p.GoToSL.GetVarStack.Peek() + gvars[tmpVar] = &GetGlobalVar{Var: gvr, TmpVar: tmpVar, IdxExpr: cf.Args[0]} + p.GoToSL.GetVarStack[len(p.GoToSL.GetVarStack)-1] = gvars +} + +// gosl: set non-read-only global vars back from temp var +func (p *printer) setGlobalVars(gvrs map[string]*GetGlobalVar) { + for _, gvr := range gvrs { + if gvr.Var.ReadOnly { + continue + } + p.print(formfeed) + p.print(gvr.Var.Name, token.LBRACK) + p.expr(gvr.IdxExpr) + p.print(token.RBRACK, blank, token.ASSIGN, blank) + tmpVar := strings.ToLower(gvr.Var.Name) + p.print(tmpVar) + p.print(token.SEMICOLON) + } +} + // gosl: methodIndex processes an index expression as receiver type of method call func (p *printer) methodIndex(idx *ast.IndexExpr) (recvPath, recvType string, pathType types.Type, isReadOnly bool, err error) { id, ok := idx.X.(*ast.Ident) @@ -1742,8 +1775,13 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { if typ != nil { recvType = getLocalTypeName(typ) if strings.HasPrefix(recvType, "invalid") { - pathIsPackage = true - recvType = id.Name // is a package path + if gvar := p.GoToSL.GetTempVar(id.Name); gvar != nil { + recvType = gvar.Var.SLType() + recvPath = "&" + recvPath + } else { + pathIsPackage = true + recvType = id.Name // is a package path + } } else { pathType = typ recvPath = recvPath @@ -1809,7 +1847,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { p.setPos(x.Rparen) p.print(token.RPAREN) - p.assignRwArgs(rwargs) + p.assignRwArgs(rwargs) // gosl: assign temp var back to global var } func (p *printer) expr0(x ast.Expr, depth int) { @@ -1866,9 +1904,14 @@ func (p *printer) stmtList(list []ast.Stmt, nindent int, nextIsRBrace bool) { // block prints an *ast.BlockStmt; it always spans at least two lines. func (p *printer) block(b *ast.BlockStmt, nindent int) { + p.GoToSL.GetVarStack.Push(make(map[string]*GetGlobalVar)) p.setPos(b.Lbrace) p.print(token.LBRACE) p.stmtList(b.List, nindent, true) + getVars := p.GoToSL.GetVarStack.Pop() + if len(getVars) > 0 { // gosl: set the get vars + p.setGlobalVars(getVars) + } p.linebreak(p.lineFor(b.Rbrace), 1, ignore, true) p.setPos(b.Rbrace) p.print(token.RBRACE) @@ -2052,6 +2095,16 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, nosemi bool) { depth++ } if s.Tok == token.DEFINE { + if ce, ok := s.Rhs[0].(*ast.CallExpr); ok { + if fid, ok := ce.Fun.(*ast.Ident); ok { + if strings.HasPrefix(fid.Name, "Get") { + if gvr, ok := p.GoToSL.GetFuncs[fid.Name]; ok { + p.getGlobalVar(s, gvr) // replace GetVar function call with assignment of local var + return + } + } + } + } p.print("var", blank) // we don't know if it is var or let.. } p.exprList(s.Pos(), s.Lhs, depth, 0, s.TokPos, false) @@ -2562,6 +2615,7 @@ func (p *printer) systemVars(d *ast.GenDecl, sysname string) { fmt.Println("\tAdded var:", nm, typ, "to group:", gp.Name) } } + p.GoToSL.VarsAdded() } func (p *printer) genDecl(d *ast.GenDecl) { diff --git a/goal/gosl/gotosl/testdata/Compute.golden b/goal/gosl/gotosl/testdata/Compute.golden index 252e19440c..3e687cd8fd 100644 --- a/goal/gosl/gotosl/testdata/Compute.golden +++ b/goal/gosl/gotosl/testdata/Compute.golden @@ -166,7 +166,8 @@ fn ParamStruct_AnotherMeth(ps: ptr, ds: ptr WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.GetGlobalVar", IDName: "get-global-var", Doc: "GetGlobalVar holds GetVar expression, to Set variable back when done.", Fields: []types.Field{{Name: "Var", Doc: "global variable"}, {Name: "TmpVar", Doc: "name of temporary variable"}, {Name: "IdxExpr", Doc: "index passed to the Get function"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current Go -> WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "GetFuncs", Doc: "GetFuncs is a map of GetVar, SetVar function names for global vars."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}, {Name: "GetVarStack", Doc: "GetVarStack is a stack per function definition of GetVar variables\nthat need to be set at the end."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.exprListMode", IDName: "expr-list-mode"}) diff --git a/math32/fastexp.go b/math32/fastexp.go index 8bd1f3046b..422e36081b 100644 --- a/math32/fastexp.go +++ b/math32/fastexp.go @@ -65,7 +65,7 @@ func FastExp3(x float32) float32 { } */ -//gosl:start fastexp +//gosl:start // FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph // It does not have any of the sanity checking of a standard method -- returns @@ -81,4 +81,4 @@ func FastExp(x float32) float32 { return math.Float32frombits(uint32(i)) } -//gosl:end fastexp +//gosl:end diff --git a/math32/minmax/avgmax.go b/math32/minmax/avgmax.go index 7877ccd4d2..a89e914908 100644 --- a/math32/minmax/avgmax.go +++ b/math32/minmax/avgmax.go @@ -6,7 +6,7 @@ package minmax import "fmt" -//gosl:start minmax +//gosl:start const ( MaxFloat32 float32 = 3.402823466e+38 @@ -69,7 +69,7 @@ func (am *AvgMax32) CalcAvg() { } } -//gosl:end minmax +//gosl:end func (am *AvgMax32) String() string { return fmt.Sprintf("{Avg: %g, Max: %g, Sum: %g, MaxIndex: %d, N: %d}", am.Avg, am.Max, am.Sum, am.MaxIndex, am.N) diff --git a/math32/minmax/minmax32.go b/math32/minmax/minmax32.go index 2f5d9fff36..eaa7e68f99 100644 --- a/math32/minmax/minmax32.go +++ b/math32/minmax/minmax32.go @@ -6,7 +6,7 @@ package minmax import "fmt" -//gosl:start minmax +//gosl:start // F32 represents a min / max range for float32 values. // Supports clipping, renormalizing, etc @@ -119,7 +119,7 @@ func (mr *F32) ClipNormValue(val float32) float32 { return mr.NormValue(val) } -//gosl:end minmax +//gosl:end func (mr *F32) String() string { return fmt.Sprintf("{%g %g}", mr.Min, mr.Max) From 69f246b68a20d6ad27bae54c7b85c6c49f218ef1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 11 Oct 2024 04:27:35 -0700 Subject: [PATCH 234/311] use gosl imports -- works great! --- goal/gosl/gotosl/extract.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index 318bdce629..d258a33331 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -125,8 +125,14 @@ func (st *State) AppendGoHeader(lines [][]byte) [][]byte { "cogentcore.org/core/goal/gosl/slbool" "cogentcore.org/core/goal/gosl/slrand" "cogentcore.org/core/goal/gosl/sltype" -) `)) + for impath := range st.GoImports { + if strings.Contains(impath, "core/goal/gosl") { + continue + } + olns = append(olns, []byte("\t\""+impath+"\"")) + } + olns = append(olns, []byte(")")) olns = append(olns, lines...) SlBoolReplace(olns) return olns From 3fa7bc0f1a22b117ea49acd35211bfb5f9ff1074 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 11 Oct 2024 04:47:09 -0700 Subject: [PATCH 235/311] key bugfix: was not removing imported package prefixes --- goal/gosl/gotosl/extract.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index d258a33331..5916735c4c 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -16,6 +16,11 @@ import ( // ExtractFiles processes all the package files and saves the corresponding // .go files with simple go header. func (st *State) ExtractFiles() { + st.ImportPackages = make(map[string]bool) + for impath := range st.GoImports { + _, pkg := filepath.Split(impath) + st.ImportPackages[pkg] = true + } for fn, fl := range st.GoFiles { fl.Lines = st.ExtractGosl(fl.Lines) WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) @@ -28,11 +33,6 @@ func (st *State) ExtractImports() { if len(st.GoImports) == 0 { return } - st.ImportPackages = make(map[string]bool) - for impath := range st.GoImports { - _, pkg := filepath.Split(impath) - st.ImportPackages[pkg] = true - } for _, im := range st.GoImports { for fn, fl := range im { fl.Lines = st.ExtractGosl(fl.Lines) From 5f8497497fb6cf9c56c1b5e3fbd04087a17c362c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 11 Oct 2024 16:38:51 -0700 Subject: [PATCH 236/311] must do vars file first to get vars; still not quite working for pl.Inhib.Fun() but closer. --- goal/gosl/gotosl/extract.go | 37 +++++++++++++++++++++++------------ goal/gosl/gotosl/files.go | 1 + goal/gosl/gotosl/gotosl.go | 14 +++++++++++-- goal/gosl/gotosl/nodes.go | 11 ++++++++++- goal/gosl/gotosl/translate.go | 17 ++++++++++++++-- 5 files changed, 63 insertions(+), 17 deletions(-) diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index 5916735c4c..78e2b42d64 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -19,10 +19,18 @@ func (st *State) ExtractFiles() { st.ImportPackages = make(map[string]bool) for impath := range st.GoImports { _, pkg := filepath.Split(impath) - st.ImportPackages[pkg] = true + if pkg != "math32" { + st.ImportPackages[pkg] = true + } } + for fn, fl := range st.GoFiles { - fl.Lines = st.ExtractGosl(fl.Lines) + hasVars := false + fl.Lines, hasVars = st.ExtractGosl(fl.Lines) + if hasVars { + st.GoVarsFiles[fn] = fl + delete(st.GoFiles, fn) + } WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) } } @@ -33,21 +41,23 @@ func (st *State) ExtractImports() { if len(st.GoImports) == 0 { return } - for _, im := range st.GoImports { + for impath, im := range st.GoImports { + _, pkg := filepath.Split(impath) for fn, fl := range im { - fl.Lines = st.ExtractGosl(fl.Lines) - WriteFileLines(filepath.Join(st.ImportsDir, fn), st.AppendGoHeader(fl.Lines)) + fl.Lines, _ = st.ExtractGosl(fl.Lines) + WriteFileLines(filepath.Join(st.ImportsDir, pkg+"-"+fn), st.AppendGoHeader(fl.Lines)) } } } // ExtractGosl gosl comment-directive tagged regions from given file. -func (st *State) ExtractGosl(lines [][]byte) [][]byte { +func (st *State) ExtractGosl(lines [][]byte) (outLines [][]byte, hasVars bool) { key := []byte("//gosl:") start := []byte("start") wgsl := []byte("wgsl") nowgsl := []byte("nowgsl") end := []byte("end") + vars := []byte("vars") imp := []byte("import") kernel := []byte("//gosl:kernel") fnc := []byte("func") @@ -55,7 +65,6 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { inReg := false inHlsl := false inNoHlsl := false - var outLns [][]byte for li, ln := range lines { tln := bytes.TrimSpace(ln) isKey := bytes.HasPrefix(tln, key) @@ -67,11 +76,14 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { switch { case inReg && isKey && bytes.HasPrefix(keyStr, end): if inHlsl || inNoHlsl { - outLns = append(outLns, ln) + outLines = append(outLines, ln) } inReg = false inHlsl = false inNoHlsl = false + case inReg && isKey && bytes.HasPrefix(keyStr, vars): + hasVars = true + outLines = append(outLines, ln) case inReg: for pkg := range st.ImportPackages { // remove package prefixes if !bytes.Contains(ln, imp) { @@ -100,20 +112,20 @@ func (st *State) ExtractGosl(lines [][]byte) [][]byte { fmt.Println("\tAdded kernel:", fnm, "args:", args, "system:", sy.Name) } } - outLns = append(outLns, ln) + outLines = append(outLines, ln) case isKey && bytes.HasPrefix(keyStr, start): inReg = true case isKey && bytes.HasPrefix(keyStr, nowgsl): inReg = true inNoHlsl = true - outLns = append(outLns, ln) // key to include self here + outLines = append(outLines, ln) // key to include self here case isKey && bytes.HasPrefix(keyStr, wgsl): inReg = true inHlsl = true - outLns = append(outLns, ln) + outLines = append(outLines, ln) } } - return outLns + return } // AppendGoHeader appends Go header @@ -125,6 +137,7 @@ func (st *State) AppendGoHeader(lines [][]byte) [][]byte { "cogentcore.org/core/goal/gosl/slbool" "cogentcore.org/core/goal/gosl/slrand" "cogentcore.org/core/goal/gosl/sltype" + "cogentcore.org/core/tensor" `)) for impath := range st.GoImports { if strings.Contains(impath, "core/goal/gosl") { diff --git a/goal/gosl/gotosl/files.go b/goal/gosl/gotosl/files.go index 0c542c1dca..a6a94a9b6a 100644 --- a/goal/gosl/gotosl/files.go +++ b/goal/gosl/gotosl/files.go @@ -77,6 +77,7 @@ func IsWGSLFile(f fs.DirEntry) bool { func (st *State) ProjectFiles() { fls := fsx.Filenames(".", ".go") st.GoFiles = make(map[string]*File) + st.GoVarsFiles = make(map[string]*File) for _, fn := range fls { fl := &File{Name: fn} var err error diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index a52b4f44a5..233de3efc2 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -157,6 +157,10 @@ type State struct { // GoFiles are all the files with gosl content in current directory. GoFiles map[string]*File + // GoVarsFiles are all the files with gosl:vars content in current directory. + // These must be processed first! they are moved from GoFiles to here. + GoVarsFiles map[string]*File + // GoImports has all the imported files. GoImports map[string]map[string]*File @@ -273,8 +277,14 @@ func (st *State) GetTempVar(vrnm string) *GetGlobalVar { if st == nil || st.GetVarStack == nil { return nil } - gvars := st.GetVarStack.Peek() - return gvars[vrnm] + nv := len(st.GetVarStack) + for i := nv - 1; i >= 0; i-- { + gvars := st.GetVarStack[i] + if gv, ok := gvars[vrnm]; ok { + return gv + } + } + return nil } // VarsAdded is called when a set of vars has been added; update relevant maps etc. diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index f431593dd3..d84648db2c 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1532,7 +1532,15 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa } else { recvPath = "&" + baseRecv.Name } - idt := p.getIdType(baseRecv) + var idt types.Type + if gvar := p.GoToSL.GetTempVar(baseRecv.Name); gvar != nil { + id := &ast.Ident{Name: gvar.Var.SLType()} + if obj, ok := p.pkg.TypesInfo.Defs[id]; ok { + idt = obj.Type() + } + } else { + idt = p.getIdType(baseRecv) + } if idt == nil { err = fmt.Errorf("gosl methodPath ERROR: cannot find type for name: %q", baseRecv.Name) p.userError(err) @@ -1540,6 +1548,7 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa } bt, err := p.getStructType(idt) if err != nil { + fmt.Println(baseRecv) return } curt := bt diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 82edfb09f3..542ad0fb32 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -49,8 +49,14 @@ func (st *State) TranslateDir(pf string) error { slrandCopied := false sltypeCopied := false - for _, gofp := range files { + done := make(map[string]bool) + + doFile := func(gofp string) { _, gofn := filepath.Split(gofp) + if _, ok := done[gofn]; ok { + return + } + done[gofn] = true wgfn := wgslFile(gofn) if st.Config.Debug { fmt.Printf("###################################\nTranslating Go file: %s\n", gofn) @@ -69,7 +75,7 @@ func (st *State) TranslateDir(pf string) error { } if afile == nil { fmt.Printf("Warning: File named: %s not found in Loaded package\n", gofn) - continue + return } var buf bytes.Buffer @@ -103,6 +109,13 @@ func (st *State) TranslateDir(pf string) error { st.SLImportFiles = append(st.SLImportFiles, &File{Name: wgfn, Lines: exsl}) } + + for fn := range st.GoVarsFiles { // do varsFiles first!! + doFile(fn) + } + for _, gofp := range files { + doFile(gofp) + } return nil } From f7e1ea1d0b5b8e7c3531ea2b2d2eb19da1c29258 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 12 Oct 2024 00:58:15 -0700 Subject: [PATCH 237/311] gosl tensor handles SetAdd etc properly. --- goal/gosl/examples/basic/shaders/Compute.wgsl | 4 +-- goal/gosl/gotosl/nodes.go | 25 ++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 597f630b2d..b80d5d1411 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -67,12 +67,12 @@ Params[0] = params; } -///////////// import: "fastexp.wgsl" +///////////// import: "math32-fastexp.wgsl" // FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph // It does not have any of the sanity checking of a standard method -- returns // nonsense when arg is out of range. Runs in 2.23ns vs. 6.3ns for 64bit which is faster -// than Exp actually. +// than exp actually. fn FastExp(x: f32) -> f32 { if (x <= -88.02969) { // this doesn't add anything and -exp is main use-case anyway return f32(0.0); diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index d84648db2c..bca8278da2 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1117,7 +1117,11 @@ func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int) { xline := p.pos.Line // before the operator (it may be on the next line!) yline := p.lineFor(x.Y.Pos()) p.setPos(x.OpPos) - p.print(x.Op) + if x.Op == token.AND_NOT { + p.print(token.AND, blank, token.TILDE) + } else { + p.print(x.Op) + } if xline != yline && xline > 0 && yline > 0 { // at least one line break, but respect an extra empty line // in the source @@ -1726,7 +1730,7 @@ func (p *printer) tensorMethod(x *ast.CallExpr, vr *Var, methName string) { args := x.Args stArg := 0 - if methName == "Set" { + if strings.HasPrefix(methName, "Set") { stArg = 1 } p.print(vr.Name, token.LBRACK) @@ -1753,8 +1757,21 @@ func (p *printer) tensorMethod(x *ast.CallExpr, vr *Var, methName string) { } } p.print(token.RPAREN, token.RBRACK) - if methName == "Set" { - p.print(blank, token.ASSIGN, blank) + if strings.HasPrefix(methName, "Set") { + opnm := strings.TrimPrefix(methName, "Set") + tok := token.ASSIGN + switch opnm { + case "Add": + tok = token.ADD_ASSIGN + case "Sub": + tok = token.SUB_ASSIGN + case "Mul": + tok = token.MUL_ASSIGN + case "Div": + tok = token.QUO_ASSIGN + } + + p.print(blank, tok, blank) p.expr(args[0]) } } From 2e82b9f708208bb55055936a113636e1efa69e02 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 12 Oct 2024 01:34:54 -0700 Subject: [PATCH 238/311] get type of Get* temp vars, looking up in type package scope. --- goal/gosl/gotosl/nodes.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index bca8278da2..395b77a329 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1538,8 +1538,12 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa } var idt types.Type if gvar := p.GoToSL.GetTempVar(baseRecv.Name); gvar != nil { - id := &ast.Ident{Name: gvar.Var.SLType()} - if obj, ok := p.pkg.TypesInfo.Defs[id]; ok { + var id ast.Ident + id = *baseRecv + id.Name = gvar.Var.SLType() + // fmt.Println("type name:", id.Name) + obj := p.pkg.Types.Scope().Lookup(id.Name) + if obj != nil { idt = obj.Type() } } else { @@ -1696,7 +1700,7 @@ func (p *printer) setGlobalVars(gvrs map[string]*GetGlobalVar) { if gvr.Var.ReadOnly { continue } - p.print(formfeed) + p.print(formfeed, "\t") p.print(gvr.Var.Name, token.LBRACK) p.expr(gvr.IdxExpr) p.print(token.RBRACK, blank, token.ASSIGN, blank) From 9b2bbb5b439d5340e6b73fef9ecbdedcbb82aa10 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 12 Oct 2024 03:01:23 -0700 Subject: [PATCH 239/311] start on atomic -- needs atomic type -- will require some infra for that. --- goal/gosl/gotosl/genkernel.go | 1 + goal/gosl/gotosl/nodes.go | 9 ++++++++- goal/gosl/gotosl/testdata/basic.go | 4 +++- goal/gosl/gotosl/typegen.go | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index 79681a0206..3faf4f0a68 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -83,6 +83,7 @@ func (st *State) GenTensorFuncs(sy *System) string { if _, ok := done[fn]; ok { continue } + done[fn] = true tconv := "" switch vr.TensorKind { case reflect.Float32: diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 395b77a329..750e871b92 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1851,7 +1851,14 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { } // fmt.Println(pathIsPackage, recvType, methName, recvPath) if pathIsPackage { - p.print(recvType + "." + methName) + if recvType == "atomic" { + switch methName { + case "AddInt32": + p.print("atomicAdd") + } + } else { + p.print(recvType + "." + methName) + } p.setPos(x.Lparen) p.print(token.LPAREN) } else { diff --git a/goal/gosl/gotosl/testdata/basic.go b/goal/gosl/gotosl/testdata/basic.go index a125fea92d..42dca43c3a 100644 --- a/goal/gosl/gotosl/testdata/basic.go +++ b/goal/gosl/gotosl/testdata/basic.go @@ -2,6 +2,7 @@ package test import ( "math" + "sync/atomic" "cogentcore.org/core/goal/gosl/slbool" "cogentcore.org/core/math32" @@ -81,7 +82,7 @@ type DataStruct struct { // exp of integ Exp float32 - pad float32 + Int int32 } // SubParamStruct has the test sub-params @@ -119,6 +120,7 @@ func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) float32 { if newVal < -10 || ps.Option.IsTrue() { newVal = -10 } + atomic.AddInt32(&(ds.Int), int32(newVal)) ds.Integ += newVal ds.Exp = math32.Exp(-ds.Integ) var a float32 diff --git a/goal/gosl/gotosl/typegen.go b/goal/gosl/gotosl/typegen.go index 3d223a5789..4f44675e55 100644 --- a/goal/gosl/gotosl/typegen.go +++ b/goal/gosl/gotosl/typegen.go @@ -20,7 +20,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Fi var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.GetGlobalVar", IDName: "get-global-var", Doc: "GetGlobalVar holds GetVar expression, to Set variable back when done.", Fields: []types.Field{{Name: "Var", Doc: "global variable"}, {Name: "TmpVar", Doc: "name of temporary variable"}, {Name: "IdxExpr", Doc: "index passed to the Get function"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current Go -> WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "GetFuncs", Doc: "GetFuncs is a map of GetVar, SetVar function names for global vars."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}, {Name: "GetVarStack", Doc: "GetVarStack is a stack per function definition of GetVar variables\nthat need to be set at the end."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current Go -> WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoVarsFiles", Doc: "GoVarsFiles are all the files with gosl:vars content in current directory.\nThese must be processed first! they are moved from GoFiles to here."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "GetFuncs", Doc: "GetFuncs is a map of GetVar, SetVar function names for global vars."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}, {Name: "GetVarStack", Doc: "GetVarStack is a stack per function definition of GetVar variables\nthat need to be set at the end."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.exprListMode", IDName: "expr-list-mode"}) From 2ebb515748b23b53cb4c302d1ba7ca4087562e87 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 13 Oct 2024 03:08:22 -0700 Subject: [PATCH 240/311] major reorganization to generate only those functions that are used in a given kernel, using our own call graph (works without requiring code to actually build). --- goal/gosl/gotosl/callgraph.go | 75 +++++++++++++++++++++++ goal/gosl/gotosl/extract.go | 2 +- goal/gosl/gotosl/files.go | 1 + goal/gosl/gotosl/genkernel.go | 21 ------- goal/gosl/gotosl/gotosl.go | 15 +++-- goal/gosl/gotosl/nodes.go | 56 +++++++++++------ goal/gosl/gotosl/printer.go | 1 + goal/gosl/gotosl/sledits.go | 16 +++++ goal/gosl/gotosl/translate.go | 112 +++++++++++++++++++++------------- 9 files changed, 212 insertions(+), 87 deletions(-) create mode 100644 goal/gosl/gotosl/callgraph.go diff --git a/goal/gosl/gotosl/callgraph.go b/goal/gosl/gotosl/callgraph.go new file mode 100644 index 0000000000..d2eeee0ef5 --- /dev/null +++ b/goal/gosl/gotosl/callgraph.go @@ -0,0 +1,75 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gotosl + +import ( + "fmt" + "sort" + + "golang.org/x/exp/maps" +) + +// Function represents the call graph of functions +type Function struct { + Name string + Funcs map[string]*Function +} + +func NewFunction(name string) *Function { + return &Function{Name: name, Funcs: make(map[string]*Function)} +} + +// get or add a function of given name +func (st *State) RecycleFunc(name string) *Function { + fn, ok := st.FuncGraph[name] + if !ok { + fn = NewFunction(name) + st.FuncGraph[name] = fn + } + return fn +} + +func getAllFuncs(f *Function, all map[string]*Function) { + for fnm, fn := range f.Funcs { + _, ok := all[fnm] + if ok { + continue + } + all[fnm] = fn + getAllFuncs(fn, all) + } +} + +// AllFuncs returns aggregated list of all functions called be given function +func (st *State) AllFuncs(name string) map[string]*Function { + fn, ok := st.FuncGraph[name] + if !ok { + fmt.Printf("gosl: ERROR kernel function named: %q not found\n", name) + return nil + } + all := make(map[string]*Function) + all[name] = fn + getAllFuncs(fn, all) + // cfs := maps.Keys(all) + // sort.Strings(cfs) + // for _, cfnm := range cfs { + // fmt.Println("\t" + cfnm) + // } + return all +} + +func (st *State) PrintFuncGraph() { + funs := maps.Keys(st.FuncGraph) + sort.Strings(funs) + for _, fname := range funs { + fmt.Println(fname) + fn := st.FuncGraph[fname] + cfs := maps.Keys(fn.Funcs) + sort.Strings(cfs) + for _, cfnm := range cfs { + fmt.Println("\t" + cfnm) + } + } +} diff --git a/goal/gosl/gotosl/extract.go b/goal/gosl/gotosl/extract.go index 78e2b42d64..3df7dd75e4 100644 --- a/goal/gosl/gotosl/extract.go +++ b/goal/gosl/gotosl/extract.go @@ -131,7 +131,7 @@ func (st *State) ExtractGosl(lines [][]byte) (outLines [][]byte, hasVars bool) { // AppendGoHeader appends Go header func (st *State) AppendGoHeader(lines [][]byte) [][]byte { olns := make([][]byte, 0, len(lines)+10) - olns = append(olns, []byte("package main")) + olns = append(olns, []byte("package imports")) olns = append(olns, []byte(`import ( "math" "cogentcore.org/core/goal/gosl/slbool" diff --git a/goal/gosl/gotosl/files.go b/goal/gosl/gotosl/files.go index a6a94a9b6a..df701b41a0 100644 --- a/goal/gosl/gotosl/files.go +++ b/goal/gosl/gotosl/files.go @@ -190,6 +190,7 @@ func (st *State) CopyPackageFile(fnm, packagePath string) error { fmfn := filepath.Join(dir, fnm) lines, err := CopyFile(fmfn, tofn) if err == nil { + lines = SlRemoveComments(lines) st.SLImportFiles = append(st.SLImportFiles, &File{Name: fnm, Lines: lines}) } return nil diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index 3faf4f0a68..da33fdf680 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -5,32 +5,11 @@ package gotosl import ( - "bytes" "fmt" - "path/filepath" "reflect" "strings" ) -// GenKernel generates and writes the WGSL kernel code for given kernel -func (st *State) GenKernel(sy *System, kn *Kernel) { - hdr := st.GenKernelHeader(sy, kn) - - lines := bytes.Split([]byte(hdr), []byte("\n")) - for _, im := range st.SLImportFiles { - lines = append(lines, []byte("")) - lines = append(lines, []byte(fmt.Sprintf("///////////// import: %q", im.Name))) - lines = append(lines, im.Lines...) - } - kn.Lines = lines - kfn := kn.Name + ".wgsl" - fn := filepath.Join(st.Config.Output, kfn) - kn.Filename = fn - WriteFileLines(fn, lines) - - st.CompileFile(kfn) -} - // GenKernelHeader returns the novel generated WGSL kernel code // for given kernel, which goes at the top of the resulting file. func (st *State) GenKernelHeader(sy *System, kn *Kernel) string { diff --git a/goal/gosl/gotosl/gotosl.go b/goal/gosl/gotosl/gotosl.go index 233de3efc2..edd81747c8 100644 --- a/goal/gosl/gotosl/gotosl.go +++ b/goal/gosl/gotosl/gotosl.go @@ -188,6 +188,15 @@ type State struct { // GetVarStack is a stack per function definition of GetVar variables // that need to be set at the end. GetVarStack stack.Stack[map[string]*GetGlobalVar] + + // GetFuncGraph is true if getting the function graph (first pass) + GetFuncGraph bool + + // KernelFuncs are the list of functions to include for current kernel. + KernelFuncs map[string]*Function + + // FuncGraph is the call graph of functions, for dead code elimination + FuncGraph map[string]*Function } func (st *State) Init(cfg *Config) { @@ -226,12 +235,6 @@ func (st *State) Run() error { st.ExtractImports() // get .go from imports st.TranslateDir("./" + st.ImportsDir) - for _, sy := range st.Systems { - for _, kn := range sy.Kernels { - st.GenKernel(sy, kn) - } - } - st.GenGPU() return nil diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 750e871b92..6c3d57e8ce 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -684,16 +684,14 @@ func (p *printer) ptrType(x ast.Expr) (ast.Expr, bool) { } // gosl: printMethRecv prints the method recv prefix for function. returns true if recv is ptr -func (p *printer) printMethRecv() bool { - isPtr := false +func (p *printer) printMethRecv() (isPtr bool, typnm string) { if u, ok := p.curMethRecv.Type.(*ast.StarExpr); ok { - p.expr(u.X) + typnm = u.X.(*ast.Ident).Name isPtr = true } else { - p.expr(p.curMethRecv.Type) + typnm = p.curMethRecv.Type.(*ast.Ident).Name } - p.print("_") - return isPtr + return } // combinesWithName reports whether a name followed by the expression x @@ -1330,6 +1328,9 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { args := x.Args var rwargs []rwArg if isid { + if p.curFunc != nil { + p.curFunc.Funcs[fid.Name] = p.GoToSL.RecycleFunc(fid.Name) + } if obj, ok := p.pkg.TypesInfo.Uses[fid]; ok { if ft, ok := obj.(*types.Func); ok { sig := ft.Type().(*types.Signature) @@ -1863,7 +1864,11 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { p.print(token.LPAREN) } else { recvType = strings.TrimPrefix(recvType, "imports.") // no! - p.print(recvType + "_" + methName) + fname := recvType + "_" + methName + if p.curFunc != nil { + p.curFunc.Funcs[fname] = p.GoToSL.RecycleFunc(fname) + } + p.print(fname) p.setPos(x.Lparen) p.print(token.LPAREN) p.print(recvPath) @@ -2559,6 +2564,9 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, tok token.Token) { // gosl: process system global vars func (p *printer) systemVars(d *ast.GenDecl, sysname string) { + if !p.GoToSL.GetFuncGraph { + return + } sy := p.GoToSL.System(sysname) var gp *Group var err error @@ -2874,36 +2882,50 @@ func (p *printer) methRecvType(typ ast.Expr) string { } func (p *printer) funcDecl(d *ast.FuncDecl) { - p.setComment(d.Doc) - p.setPos(d.Pos()) - // We have to save startCol only after emitting FUNC; otherwise it can be on a - // different line (all whitespace preceding the FUNC is emitted only when the - // FUNC is emitted). - startCol := p.out.Column - len("func ") + fname := "" if d.Recv != nil { for ex := range p.ExcludeFunctions { if d.Name.Name == ex { return } } - p.print("fn", blank) if d.Recv.List[0].Names != nil { p.curMethRecv = d.Recv.List[0] - if p.printMethRecv() { + isptr, typnm := p.printMethRecv() + if isptr { p.curPtrArgs = []*ast.Ident{p.curMethRecv.Names[0]} } + fname = typnm + "_" + d.Name.Name // fmt.Printf("cur func recv: %v\n", p.curMethRecv) } // p.parameters(d.Recv, funcParam) // method: print receiver // p.print(blank) } else { - p.print("fn", blank) + fname = d.Name.Name } - p.expr(d.Name) + if p.GoToSL.GetFuncGraph { + p.curFunc = p.GoToSL.RecycleFunc(fname) + } else { + _, ok := p.GoToSL.KernelFuncs[fname] + if !ok { + return + } + } + p.setComment(d.Doc) + p.setPos(d.Pos()) + // We have to save startCol only after emitting FUNC; otherwise it can be on a + // different line (all whitespace preceding the FUNC is emitted only when the + // FUNC is emitted). + startCol := p.out.Column - len("func ") + p.print("fn", blank, fname) p.signature(d.Type, d.Recv) p.funcBody(p.distanceFrom(d.Pos(), startCol), vtab, d.Body) p.curPtrArgs = nil p.curMethRecv = nil + if p.GoToSL.GetFuncGraph { + p.GoToSL.FuncGraph[fname] = p.curFunc + p.curFunc = nil + } } func (p *printer) decl(decl ast.Decl) { diff --git a/goal/gosl/gotosl/printer.go b/goal/gosl/gotosl/printer.go index cafe7e895e..dc2a591f19 100644 --- a/goal/gosl/gotosl/printer.go +++ b/goal/gosl/gotosl/printer.go @@ -109,6 +109,7 @@ type printer struct { // current arguments to function that are pointers and thus need dereferencing // when accessing fields curPtrArgs []*ast.Ident + curFunc *Function curMethRecv *ast.Field // current method receiver, also included in curPtrArgs if ptr curReturnType *ast.Ident } diff --git a/goal/gosl/gotosl/sledits.go b/goal/gosl/gotosl/sledits.go index 3361b2c869..701805fcf1 100644 --- a/goal/gosl/gotosl/sledits.go +++ b/goal/gosl/gotosl/sledits.go @@ -95,6 +95,22 @@ func MathReplaceAll(mat, ln []byte) []byte { } } +func SlRemoveComments(lines [][]byte) [][]byte { + comm := []byte("//") + olns := make([][]byte, 0, len(lines)) + for _, ln := range lines { + ts := bytes.TrimSpace(ln) + if len(ts) == 0 { + continue + } + if bytes.HasPrefix(ts, comm) { + continue + } + olns = append(olns, ln) + } + return olns +} + // SlEditsReplace replaces Go with equivalent WGSL code // returns true if has slrand. or sltype. // to auto include that header file if so. diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 542ad0fb32..0bc8e8af66 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -9,7 +9,6 @@ import ( "fmt" "go/ast" "go/token" - "io/ioutil" "log" "os" "os/exec" @@ -22,8 +21,8 @@ import ( // TranslateDir translate all .Go files in given directory to WGSL. func (st *State) TranslateDir(pf string) error { - nl := []byte("\n") pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName | packages.NeedFiles | packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesSizes | packages.NeedTypesInfo}, pf) + // pkgs, err := packages.Load(&packages.Config{Mode: packages.LoadAllSyntax}, pf) if err != nil { return errors.Log(err) } @@ -36,6 +35,7 @@ func (st *State) TranslateDir(pf string) error { err := fmt.Errorf("No Go files found in package: %+v", pkg) return errors.Log(err) } + // fmt.Printf("go files: %+v", pkg.GoFiles) // return nil, err files := pkg.GoFiles @@ -46,22 +46,14 @@ func (st *State) TranslateDir(pf string) error { fmt.Println(serr) } - slrandCopied := false - sltypeCopied := false - - done := make(map[string]bool) + st.FuncGraph = make(map[string]*Function) + st.GetFuncGraph = true - doFile := func(gofp string) { + doFile := func(gofp string, buf *bytes.Buffer) { _, gofn := filepath.Split(gofp) - if _, ok := done[gofn]; ok { - return - } - done[gofn] = true - wgfn := wgslFile(gofn) if st.Config.Debug { fmt.Printf("###################################\nTranslating Go file: %s\n", gofn) } - var afile *ast.File var fpos token.Position for _, sy := range pkg.Syntax { @@ -78,44 +70,80 @@ func (st *State) TranslateDir(pf string) error { return } - var buf bytes.Buffer pcfg := PrintConfig{GoToSL: st, Mode: printerMode, Tabwidth: tabWidth, ExcludeFunctions: st.ExcludeMap} - pcfg.Fprint(&buf, pkg, afile) - // ioutil.WriteFile(filepath.Join(*outDir, fn+".tmp"), buf.Bytes(), 0644) - slfix, hasSltype, hasSlrand := SlEdits(buf.Bytes()) - if hasSlrand && !slrandCopied { - hasSltype = true - if st.Config.Debug { - fmt.Printf("\tcopying slrand.wgsl to shaders\n") - } - st.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/goal/gosl/slrand") - slrandCopied = true - } - if hasSltype && !sltypeCopied { - if st.Config.Debug { - fmt.Printf("\tcopying sltype.wgsl to shaders\n") - } - st.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/goal/gosl/sltype") - sltypeCopied = true - } - exsl := st.ExtractWGSL(slfix) - - if !st.Config.Keep { + pcfg.Fprint(buf, pkg, afile) + if !st.GetFuncGraph && !st.Config.Keep { os.Remove(fpos.Filename) } - - slfn := filepath.Join(st.ImportsDir, wgfn) - ioutil.WriteFile(slfn, bytes.Join(exsl, nl), 0644) - - st.SLImportFiles = append(st.SLImportFiles, &File{Name: wgfn, Lines: exsl}) } + // first pass is just to get the call graph: for fn := range st.GoVarsFiles { // do varsFiles first!! - doFile(fn) + var buf bytes.Buffer + doFile(fn, &buf) } for _, gofp := range files { - doFile(gofp) + _, gofn := filepath.Split(gofp) + if _, ok := st.GoVarsFiles[gofn]; ok { + continue + } + var buf bytes.Buffer + doFile(gofp, &buf) + } + + // st.PrintFuncGraph() + + st.CopyPackageFile("slrand.wgsl", "cogentcore.org/core/goal/gosl/slrand") + st.CopyPackageFile("sltype.wgsl", "cogentcore.org/core/goal/gosl/sltype") + + doKernelFile := func(fname string, lines [][]byte) [][]byte { + _, gofn := filepath.Split(fname) + var buf bytes.Buffer + doFile(fname, &buf) + slfix, hasSltype, hasSlrand := SlEdits(buf.Bytes()) + _ = hasSlrand + _ = hasSltype + slfix = SlRemoveComments(slfix) + exsl := st.ExtractWGSL(slfix) + lines = append(lines, []byte("")) + lines = append(lines, []byte(fmt.Sprintf("///////////// import: %q", gofn))) + lines = append(lines, exsl...) + return lines } + + // next pass is per kernel + st.GetFuncGraph = false + for _, sy := range st.Systems { + for _, kn := range sy.Kernels { + // if st.Config.Debug { + fmt.Printf("###################################\nTranslating Kernel file: %s\n", kn.Name) + // } + hdr := st.GenKernelHeader(sy, kn) + lines := bytes.Split([]byte(hdr), []byte("\n")) + st.KernelFuncs = st.AllFuncs(kn.Name) + if st.KernelFuncs == nil { + continue + } + for fn := range st.GoVarsFiles { // do varsFiles first!! + lines = doKernelFile(fn, lines) + } + for _, gofp := range files { + lines = doKernelFile(gofp, lines) + } + for _, im := range st.SLImportFiles { + lines = append(lines, []byte("")) + lines = append(lines, []byte(fmt.Sprintf("///////////// import: %q", im.Name))) + lines = append(lines, im.Lines...) + } + kn.Lines = lines + kfn := kn.Name + ".wgsl" + fn := filepath.Join(st.Config.Output, kfn) + kn.Filename = fn + WriteFileLines(fn, lines) + st.CompileFile(kfn) + } + } + return nil } From b96d53098f11d49298b02511db8be9f3937c4fa3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 13 Oct 2024 03:27:21 -0700 Subject: [PATCH 241/311] fix tmpvar name for Get -> Set --- goal/gosl/gotosl/nodes.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 6c3d57e8ce..fcda3bc1dc 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1705,8 +1705,7 @@ func (p *printer) setGlobalVars(gvrs map[string]*GetGlobalVar) { p.print(gvr.Var.Name, token.LBRACK) p.expr(gvr.IdxExpr) p.print(token.RBRACK, blank, token.ASSIGN, blank) - tmpVar := strings.ToLower(gvr.Var.Name) - p.print(tmpVar) + p.print(gvr.TmpVar) p.print(token.SEMICOLON) } } From 6a9a4d69bd6fecbb143cc320fd5fcbee86fb3274 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 14 Oct 2024 00:43:51 -0700 Subject: [PATCH 242/311] add package.function calls for callgraph -- fixes FastExp --- goal/gosl/gotosl/nodes.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index fcda3bc1dc..32c05341c1 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1858,6 +1858,9 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { } } else { p.print(recvType + "." + methName) + if p.curFunc != nil { + p.curFunc.Funcs[methName] = p.GoToSL.RecycleFunc(methName) + } } p.setPos(x.Lparen) p.print(token.LPAREN) From 00be93f1367754d09d032466a5c210acc39b65f1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 14 Oct 2024 00:58:27 -0700 Subject: [PATCH 243/311] get path type for Get temp vars --- goal/gosl/gotosl/nodes.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 32c05341c1..bdde7cf5ee 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1539,14 +1539,7 @@ func (p *printer) methodPath(x *ast.SelectorExpr) (recvPath, recvType string, pa } var idt types.Type if gvar := p.GoToSL.GetTempVar(baseRecv.Name); gvar != nil { - var id ast.Ident - id = *baseRecv - id.Name = gvar.Var.SLType() - // fmt.Println("type name:", id.Name) - obj := p.pkg.Types.Scope().Lookup(id.Name) - if obj != nil { - idt = obj.Type() - } + idt = p.getTypeNameType(gvar.Var.SLType()) } else { idt = p.getIdType(baseRecv) } @@ -1602,6 +1595,14 @@ func (p *printer) getIdType(id *ast.Ident) types.Type { return nil } +func (p *printer) getTypeNameType(typeName string) types.Type { + obj := p.pkg.Types.Scope().Lookup(typeName) + if obj != nil { + return obj.Type() + } + return nil +} + func getLocalTypeName(typ types.Type) string { _, nm := path.Split(typ.String()) return nm @@ -1808,6 +1809,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { if gvar := p.GoToSL.GetTempVar(id.Name); gvar != nil { recvType = gvar.Var.SLType() recvPath = "&" + recvPath + pathType = p.getTypeNameType(gvar.Var.SLType()) } else { pathIsPackage = true recvType = id.Name // is a package path From c9c5807f389a73fc26f2b008659e85a39231e6b8 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 14 Oct 2024 01:13:35 -0700 Subject: [PATCH 244/311] set global var from temp _before_ final return statement --- goal/gosl/gotosl/nodes.go | 16 +++++++++++++++- goal/gosl/gotosl/translate.go | 12 ++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index bdde7cf5ee..45291eced7 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1953,11 +1953,25 @@ func (p *printer) block(b *ast.BlockStmt, nindent int) { p.GoToSL.GetVarStack.Push(make(map[string]*GetGlobalVar)) p.setPos(b.Lbrace) p.print(token.LBRACE) - p.stmtList(b.List, nindent, true) + nstmt := len(b.List) + retLast := false + if nstmt > 1 { + if _, ok := b.List[nstmt-1].(*ast.ReturnStmt); ok { + retLast = true + } + } + if retLast { + p.stmtList(b.List[:nstmt-1], nindent, true) + } else { + p.stmtList(b.List, nindent, true) + } getVars := p.GoToSL.GetVarStack.Pop() if len(getVars) > 0 { // gosl: set the get vars p.setGlobalVars(getVars) } + if retLast { + p.stmt(b.List[nstmt-1], true, false) + } p.linebreak(p.lineFor(b.Rbrace), 1, ignore, true) p.setPos(b.Rbrace) p.print(token.RBRACE) diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 0bc8e8af66..b4a5d91a7c 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -13,9 +13,11 @@ import ( "os" "os/exec" "path/filepath" + "sort" "cogentcore.org/core/base/errors" "cogentcore.org/core/goal/gosl/alignsl" + "golang.org/x/exp/maps" "golang.org/x/tools/go/packages" ) @@ -113,8 +115,14 @@ func (st *State) TranslateDir(pf string) error { // next pass is per kernel st.GetFuncGraph = false - for _, sy := range st.Systems { - for _, kn := range sy.Kernels { + sys := maps.Keys(st.Systems) + sort.Strings(sys) + for _, snm := range sys { + sy := st.Systems[snm] + kns := maps.Keys(sy.Kernels) + sort.Strings(kns) + for _, knm := range kns { + kn := sy.Kernels[knm] // if st.Config.Debug { fmt.Printf("###################################\nTranslating Kernel file: %s\n", kn.Name) // } From 95082fd1dd579f1937f3563f9044921af0b062e9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 14 Oct 2024 01:56:55 -0700 Subject: [PATCH 245/311] prevent redundant include, fix tests and exampels. --- goal/gosl/examples/basic/shaders/Compute.wgsl | 209 ++++++++++++--- goal/gosl/examples/rand/gosl.go | 4 +- goal/gosl/examples/rand/shaders/Compute.wgsl | 242 +++-------------- goal/gosl/gotosl/testdata/Compute.golden | 251 ++++++++++++------ goal/gosl/gotosl/testdata/basic.go | 3 +- goal/gosl/gotosl/translate.go | 4 + goal/gosl/gotosl/typegen.go | 12 +- 7 files changed, 391 insertions(+), 334 deletions(-) diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index b80d5d1411..469b475ce0 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -17,68 +17,201 @@ fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { } -///////////// import: "compute.wgsl" - -//gosl:import "cogentcore.org/core/math32" - -//gosl:vars - -// Params are the parameters for the computation. -// -// gosl:read-only - -// Data is the data on which the computation operates. -// 2D: outer index is data, inner index is: Raw, Integ, Exp vars. -// -//gosl:dims 2 - +///////////// import: "compute.go" const Raw: i32 = 0; const Integ: i32 = 1; - const Exp: i32 = 2; - -// ParamStruct has the test params struct ParamStruct { - - // rate constant in msec Tau: f32, - - // 1/Tau Dt: f32, - pad: f32, pad1: f32, } - -// IntegFromRaw computes integrated value from current raw value fn ParamStruct_IntegFromRaw(ps: ptr, idx: i32) { var integ = Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))]; integ += (*ps).Dt * (Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Raw))] - integ); Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Integ))] = integ; Data[IndexF322D(Data[0], Data[1], u32(idx),u32(Exp))] = FastExp(-integ); } - -// Compute does the main computation. fn Compute(i: u32) { //gosl:kernel var params = Params[0]; ParamStruct_IntegFromRaw(¶ms, i32(i)); -Params[0] = params; - + Params[0] = params; } - -///////////// import: "math32-fastexp.wgsl" - -// FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph -// It does not have any of the sanity checking of a standard method -- returns -// nonsense when arg is out of range. Runs in 2.23ns vs. 6.3ns for 64bit which is faster -// than exp actually. +///////////// import: "math32-fastexp.go" fn FastExp(x: f32) -> f32 { if (x <= -88.02969) { // this doesn't add anything and -exp is main use-case anyway return f32(0.0); } var i = i32(12102203*x) + i32(127)*(i32(1)<<23); var m = i >> 7 & 0xFFFF; // copy mantissa - i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11); - return bitcast(u32(i)); + i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11);return bitcast(u32(i)); +} + +///////////// import: "slrand.wgsl" +fn Philox2x32round(counter: su64, key: u32) -> su64 { + let mul = Uint32Mul64(u32(0xD256D193), counter.x); + var ctr: su64; + ctr.x = mul.y ^ key ^ counter.y; + ctr.y = mul.x; + return ctr; +} +fn Philox2x32bumpkey(key: u32) -> u32 { + return key + u32(0x9E3779B9); +} +fn Philox2x32(counter: su64, key: u32) -> vec2 { + var ctr = Philox2x32round(counter, key); // 1 + var ky = Philox2x32bumpkey(key); + ctr = Philox2x32round(ctr, ky); // 2 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 3 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 4 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 5 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 6 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 7 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 8 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 9 + ky = Philox2x32bumpkey(ky); + return Philox2x32round(ctr, ky); // 10 +} +fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Philox2x32(Uint64Add32(counter, funcIndex), key); +} +fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { + return Philox2x32(Uint64Add32(counter, funcIndex), key).x; +} +fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} +fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); +} +fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} +fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); +} +fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { + return (RandFloat32(counter, funcIndex, key) < p); +} +fn sincospi(x: f32) -> vec2 { + let PIf = 3.1415926535897932; + var r: vec2; + r.x = cos(PIf*x); + r.y = sin(PIf*x); + return r; +} +fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + let ur = RandUint32Vec2(counter, funcIndex, key); + var f = sincospi(Uint32ToFloat32Range11(ur.x)); + let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. + return f * r; +} +fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { + return RandFloat32Vec2(counter, funcIndex, key).x; +} +fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { + let v = RandFloat32(counter, funcIndex, key); + return u32(v * f32(n)); +} +struct RandCounter { + Counter: su64, + HiSeed: u32, + pad: u32, +} +fn RandCounter_Reset(ct: ptr) { + (*ct).Counter.x = u32(0); + (*ct).Counter.y = (*ct).HiSeed; +} +fn RandCounter_Seed(ct: ptr, seed: u32) { + (*ct).HiSeed = seed; + RandCounter_Reset(ct); +} +fn RandCounter_Add(ct: ptr, inc: u32) { + (*ct).Counter = Uint64Add32((*ct).Counter, inc); +} + +///////////// import: "sltype.wgsl" +alias su64 = vec2; +fn Uint32Mul64(a: u32, b: u32) -> su64 { + let LOMASK = (((u32(1))<<16)-1); + var r: su64; + r.x = a * b; /* full low multiply */ + let ahi = a >> 16; + let alo = a & LOMASK; + let bhi = b >> 16; + let blo = b & LOMASK; + let ahbl = ahi * blo; + let albh = alo * bhi; + let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); + var hit = ahi*bhi + (ahbl>>16) + (albh>>16); + hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ + /* carry from the sum with alo*blo */ + if ((r.x >> u32(16)) < (ahbl_albh&LOMASK)) { + hit += u32(1); + } + r.y = hit; + return r; +} +/* +fn Uint32Mul64(a: u32, b: u32) -> su64 { + return su64(a) * su64(b); +} +*/ +fn Uint64Add32(a: su64, b: u32) -> su64 { + if (b == 0) { + return a; + } + var s = a; + if (s.x > u32(0xffffffff) - b) { + s.y++; + s.x = (b - 1) - (u32(0xffffffff) - s.x); + } else { + s.x += b; + } + return s; +} +fn Uint64Incr(a: su64) -> su64 { + var s = a; + if(s.x == 0xffffffff) { + s.y++; + s.x = u32(0); + } else { + s.x++; + } + return s; +} +fn Uint32ToFloat32(val: u32) -> f32 { + let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + var f = f32(val) * factor + halffactor; + if (f == 1.0) { // exclude 1 + return bitcast(0x3F7FFFFF); + } + return f; +} +fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32(val.x); + r.y = Uint32ToFloat32(val.y); + return r; +} +fn Uint32ToFloat32Range11(val: u32) -> f32 { + let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + return (f32(val) * factor + halffactor); } +fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32Range11(val.x); + r.y = Uint32ToFloat32Range11(val.y); + return r; +} \ No newline at end of file diff --git a/goal/gosl/examples/rand/gosl.go b/goal/gosl/examples/rand/gosl.go index 7c297442cf..31e4e10960 100644 --- a/goal/gosl/examples/rand/gosl.go +++ b/goal/gosl/examples/rand/gosl.go @@ -174,7 +174,7 @@ func SyncFromGPU(vars ...GPUVars) { // [Seed] []Seeds at given index. // To ensure that values are updated on the GPU, you must call [SetSeed]. // after all changes have been made. -func GetSeed(idx int) *Seeds { +func GetSeed(idx uint32) *Seeds { return &Seed[idx] } @@ -182,6 +182,6 @@ func GetSeed(idx int) *Seeds { // [Data] []Rnds at given index. // To ensure that values are updated on the GPU, you must call [SetData]. // after all changes have been made. -func GetData(idx int) *Rnds { +func GetData(idx uint32) *Rnds { return &Data[idx] } diff --git a/goal/gosl/examples/rand/shaders/Compute.wgsl b/goal/gosl/examples/rand/shaders/Compute.wgsl index 113f10fa95..f6e3078847 100644 --- a/goal/gosl/examples/rand/shaders/Compute.wgsl +++ b/goal/gosl/examples/rand/shaders/Compute.wgsl @@ -12,21 +12,39 @@ fn main(@builtin(global_invocation_id) idx: vec3) { } -///////////// import: "slrand.wgsl" -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/slrand -// See README.md there for documentation. - -// These random number generation (RNG) functions are optimized for -// use on the GPU, with equivalent Go versions available in slrand.go. -// This is using the Philox2x32 counter-based RNG. - -// #include "sltype.wgsl" +///////////// import: "rand.go" +struct Seeds { + Seed: su64, + pad: i32, + pad1: i32, +} +struct Rnds { + Uints: vec2, + pad: i32, + pad1: i32, + Floats: vec2, + pad2: i32, + pad3: i32, + Floats11: vec2, + pad4: i32, + pad5: i32, + Gauss: vec2, + pad6: i32, + pad7: i32, +} +fn Rnds_RndGen(r: ptr, counter: su64, idx: u32) { + (*r).Uints = RandUint32Vec2(counter, u32(0), idx); + (*r).Floats = RandFloat32Vec2(counter, u32(1), idx); + (*r).Floats11 = RandFloat32Range11Vec2(counter, u32(2), idx); + (*r).Gauss = RandFloat32NormVec2(counter, u32(3), idx); +} +fn Compute(i: u32) { //gosl:kernel + var data=Data[i]; + Rnds_RndGen(&data, Seed[0].Seed, i); + Data[i]=data; +} -// Philox2x32round does one round of updating of the counter. +///////////// import: "slrand.wgsl" fn Philox2x32round(counter: su64, key: u32) -> su64 { let mul = Uint32Mul64(u32(0xD256D193), counter.x); var ctr: su64; @@ -34,19 +52,10 @@ fn Philox2x32round(counter: su64, key: u32) -> su64 { ctr.y = mul.x; return ctr; } - -// Philox2x32bumpkey does one round of updating of the key fn Philox2x32bumpkey(key: u32) -> u32 { return key + u32(0x9E3779B9); } - -// Philox2x32 implements the stateless counter-based RNG algorithm -// returning a random number as two uint32 values, given a -// counter and key input that determine the result. -// The input counter is not modified. fn Philox2x32(counter: su64, key: u32) -> vec2 { - // this is an unrolled loop of 10 updates based on initial counter and key, - // which produces the random deviation deterministically based on these inputs. var ctr = Philox2x32round(counter, key); // 1 var ky = Philox2x32bumpkey(key); ctr = Philox2x32round(ctr, ky); // 2 @@ -65,79 +74,29 @@ fn Philox2x32(counter: su64, key: u32) -> vec2 { ky = Philox2x32bumpkey(ky); ctr = Philox2x32round(ctr, ky); // 9 ky = Philox2x32bumpkey(ky); - return Philox2x32round(ctr, ky); // 10 } - -//////////////////////////////////////////////////////////// -// Methods below provide a standard interface with more -// readable names, mapping onto the Go rand methods. -// -// They assume a global shared counter, which is then -// incremented by a function index, defined for each function -// consuming random numbers that _could_ be called within a parallel -// processing loop. At the end of the loop, the global counter should -// be incremented by the total possible number of such functions. -// This results in fully resproducible results, invariant to -// specific processing order, and invariant to whether any one function -// actually calls the random number generator. - -// RandUint32Vec2 returns two uniformly distributed 32 unsigned integers, -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { return Philox2x32(Uint64Add32(counter, funcIndex), key); } - -// RandUint32 returns a uniformly distributed 32 unsigned integer, -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { return Philox2x32(Uint64Add32(counter, funcIndex), key).x; } - -// RandFloat32Vec2 returns two uniformly distributed float32 values in range (0,1), -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); } - -// RandFloat32 returns a uniformly distributed float32 value in range (0,1), -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); } - -// RandFloat32Range11Vec2 returns two uniformly distributed float32 values in range [-1,1], -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); } - -// RandFloat32Range11 returns a uniformly distributed float32 value in range [-1,1], -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); } - -// RandBoolP returns a bool true value with probability p -// based on given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { return (RandFloat32(counter, funcIndex, key) < p); } - fn sincospi(x: f32) -> vec2 { let PIf = 3.1415926535897932; var r: vec2; @@ -145,93 +104,38 @@ fn sincospi(x: f32) -> vec2 { r.y = sin(PIf*x); return r; } - -// RandFloat32NormVec2 returns two random float32 numbers -// distributed according to the normal, Gaussian distribution -// with zero mean and unit variance. -// This is done very efficiently using the Box-Muller algorithm -// that consumes two random 32 bit uint values. -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { let ur = RandUint32Vec2(counter, funcIndex, key); var f = sincospi(Uint32ToFloat32Range11(ur.x)); let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. return f * r; } - -// RandFloat32Norm returns a random float32 number -// distributed according to the normal, Gaussian distribution -// with zero mean and unit variance. -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { return RandFloat32Vec2(counter, funcIndex, key).x; } - -// RandUint32N returns a uint32 in the range [0,N). -// Uses given global shared counter, function index offset from that -// counter for this specific random number call, and key as unique -// index of the item being processed. fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { let v = RandFloat32(counter, funcIndex, key); return u32(v * f32(n)); } - -// Counter is used for storing the random counter using aligned 16 byte -// storage, with convenience functions for typical use cases. -// It retains a copy of the last Seed value, which is applied to -// the Hi uint32 value. struct RandCounter { Counter: su64, HiSeed: u32, pad: u32, } - -// Reset resets counter to last set Seed state. fn RandCounter_Reset(ct: ptr) { (*ct).Counter.x = u32(0); (*ct).Counter.y = (*ct).HiSeed; } - -// Seed sets the Hi uint32 value from given seed, saving it in Seed field. -// Each increment in seed generates a unique sequence of over 4 billion numbers, -// so it is reasonable to just use incremental values there, but more widely -// spaced numbers will result in longer unique sequences. -// Resets Lo to 0. -// This same seed will be restored during Reset fn RandCounter_Seed(ct: ptr, seed: u32) { (*ct).HiSeed = seed; RandCounter_Reset(ct); } - -// Add increments the counter by given amount. -// Call this after completing a pass of computation -// where the value passed here is the max of funcIndex+1 -// used for any possible random calls during that pass. fn RandCounter_Add(ct: ptr, inc: u32) { (*ct).Counter = Uint64Add32((*ct).Counter, inc); } - ///////////// import: "sltype.wgsl" -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Original file is in Go package: github.com/cogentcore/core/gpu/gosl/sltype -// See README.md there for documentation. - -// This file emulates uint64 (u64) using 2 uint32 integers. -// and defines conversions between uint and float. - -// define a u64 type as an alias. -// if / when u64 actually happens, will make it easier to update. alias su64 = vec2; - -// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using vec2). fn Uint32Mul64(a: u32, b: u32) -> su64 { let LOMASK = (((u32(1))<<16)-1); var r: su64; @@ -240,10 +144,8 @@ fn Uint32Mul64(a: u32, b: u32) -> su64 { let alo = a & LOMASK; let bhi = b >> 16; let blo = b & LOMASK; - let ahbl = ahi * blo; let albh = alo * bhi; - let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); var hit = ahi*bhi + (ahbl>>16) + (albh>>16); hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ @@ -254,16 +156,11 @@ fn Uint32Mul64(a: u32, b: u32) -> su64 { r.y = hit; return r; } - /* -// Uint32Mul64 multiplies two uint32 numbers into a uint64 (using su64). fn Uint32Mul64(a: u32, b: u32) -> su64 { return su64(a) * su64(b); } */ - - -// Uint64Add32 adds given uint32 number to given uint64 (using vec2). fn Uint64Add32(a: su64, b: u32) -> su64 { if (b == 0) { return a; @@ -277,8 +174,6 @@ fn Uint64Add32(a: su64, b: u32) -> su64 { } return s; } - -// Uint64Incr returns increment of the given uint64 (using vec2). fn Uint64Incr(a: su64) -> su64 { var s = a; if(s.x == 0xffffffff) { @@ -289,12 +184,6 @@ fn Uint64Incr(a: su64) -> su64 { } return s; } - -// Uint32ToFloat32 converts a uint32 integer into a float32 -// in the (0,1) interval (i.e., exclusive of 1). -// This differs from the Go standard by excluding 0, which is handy for passing -// directly to Log function, and from the reference Philox code by excluding 1 -// which is in the Go standard and most other standard RNGs. fn Uint32ToFloat32(val: u32) -> f32 { let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); let halffactor = f32(0.5) * factor; @@ -304,85 +193,20 @@ fn Uint32ToFloat32(val: u32) -> f32 { } return f; } - -// note: there is no overloading of user-defined functions -// https://github.com/gpuweb/gpuweb/issues/876 - -// Uint32ToFloat32Vec2 converts two uint 32 bit integers -// into two corresponding 32 bit f32 values -// in the (0,1) interval (i.e., exclusive of 1). fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { var r: vec2; r.x = Uint32ToFloat32(val.x); r.y = Uint32ToFloat32(val.y); return r; } - -// Uint32ToFloat32Range11 converts a uint32 integer into a float32 -// in the [-1..1] interval (inclusive of -1 and 1, never identically == 0). fn Uint32ToFloat32Range11(val: u32) -> f32 { let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); let halffactor = f32(0.5) * factor; return (f32(val) * factor + halffactor); } - -// Uint32ToFloat32Range11Vec2 converts two uint32 integers into two float32 -// in the [-1,1] interval (inclusive of -1 and 1, never identically == 0). fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { var r: vec2; r.x = Uint32ToFloat32Range11(val.x); r.y = Uint32ToFloat32Range11(val.y); return r; -} - - - - -///////////// import: "rand.wgsl" - -//gosl:vars - -//gosl:read-only - -// Data - -struct Seeds { - Seed: su64, - pad: i32, - pad1: i32, -} - -struct Rnds { - Uints: vec2, - pad: i32, - pad1: i32, - Floats: vec2, - pad2: i32, - pad3: i32, - Floats11: vec2, - pad4: i32, - pad5: i32, - Gauss: vec2, - pad6: i32, - pad7: i32, -} - -// RndGen calls random function calls to test generator. -// Note that the counter to the outer-most computation function -// is passed by *value*, so the same counter goes to each element -// as it is computed, but within this scope, counter is passed by -// reference (as a pointer) so subsequent calls get a new counter value. -// The counter should be incremented by the number of random calls -// outside of the overall update function. -fn Rnds_RndGen(r: ptr, counter: su64, idx: u32) { - (*r).Uints = RandUint32Vec2(counter, u32(0), idx); - (*r).Floats = RandFloat32Vec2(counter, u32(1), idx); - (*r).Floats11 = RandFloat32Range11Vec2(counter, u32(2), idx); - (*r).Gauss = RandFloat32NormVec2(counter, u32(3), idx); -} - -fn Compute(i: u32) { //gosl:kernel - var data=Data[i]; - Rnds_RndGen(&data, Seed[0].Seed, i); - Data[i]=data; -} +} \ No newline at end of file diff --git a/goal/gosl/gotosl/testdata/Compute.golden b/goal/gosl/gotosl/testdata/Compute.golden index 3e687cd8fd..4f7caade21 100644 --- a/goal/gosl/gotosl/testdata/Compute.golden +++ b/goal/gosl/gotosl/testdata/Compute.golden @@ -13,112 +13,43 @@ fn main(@builtin(global_invocation_id) idx: vec3) { } -///////////// import: "basic.wgsl" - -//gosl:vars - -// Params are the parameters for the computation. -//gosl:read-only - -// Data is the data on which the computation operates. - -// FastExp is a quartic spline approximation to the Exp function, by N.N. Schraudolph -// It does not have any of the sanity checking of a standard method -- returns -// nonsense when arg is out of range. Runs in 2.23ns vs. 6.3ns for 64bit which is faster -// than exp actually. -fn FastExp(x: f32) -> f32 { - if (x <= -88.76731) { // this doesn't add anything and -exp is main use-case anyway - return f32(0); - } - var i = i32(12102203*x) + i32(127)*(i32(1)<<23); - var m = i >> 7 & 0xFFFF; // copy mantissa - i += (((((((((((3537 * m) >> 16) + 13668) * m) >> 18) + 15817) * m) >> 14) - 80470) * m) >> 11); - return bitcast(u32(i)); -} - -// NeuronFlags are bit-flags encoding relevant binary state for neurons +///////////// import: "basic.go" alias NeuronFlags = i32; - -// The neuron flags - -// NeuronOff flag indicates that this neuron has been turned off (i.e., lesioned) const NeuronOff: NeuronFlags = 0x01; - -// NeuronHasExt means the neuron has external input in its Ext field const NeuronHasExt: NeuronFlags = 0x02; // note: 1<<2 does NOT work - -// NeuronHasTarg means the neuron has external target input in its Target field const NeuronHasTarg: NeuronFlags = 0x04; - -// NeuronHasCmpr means the neuron has external comparison input in its Target field -- used for computing -// comparison statistics but does not drive neural activity ever const NeuronHasCmpr: NeuronFlags = 0x08; - -// Modes are evaluation modes (Training, Testing, etc) alias Modes = i32; - -// The evaluation modes - const NoEvalMode: Modes = 0; - -// AllModes indicates that the log should occur over all modes present in other items. const AllModes: Modes = 1; - -// Train is this a training mode for the env const Train: Modes = 2; - -// Test is this a test mode for the env const Test: Modes = 3; - -// DataStruct has the test data struct DataStruct { - - // raw value Raw: f32, - - // integrated value Integ: f32, - - // exp of integ Exp: f32, - - pad: f32, + Int: i32, } - -// SubParamStruct has the test sub-params struct SubParamStruct { A: f32, B: f32, C: f32, D: f32, } - fn SubParamStruct_Sum(sp: ptr) -> f32 { return (*sp).A + (*sp).B + (*sp).C + (*sp).D; } - fn SubParamStruct_SumPlus(sp: ptr, extra: f32) -> f32 { return SubParamStruct_Sum(sp) + extra; } - -// ParamStruct has the test params struct ParamStruct { - - // rate constant in msec Tau: f32, - - // 1/Tau Dt: f32, Option: i32, // note: standard bool doesn't work - pad: f32, // comment this out to trigger alignment warning - - // extra parameters Subs: SubParamStruct, } - fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr) -> f32 { - // note: the following are just to test basic control structures var newVal = (*ps).Dt * ((*ds).Raw - (*ds).Integ); if (newVal < -10 || (*ps).Option == 1) { newVal = f32(-10); @@ -126,18 +57,14 @@ fn ParamStruct_IntegFromRaw(ps: ptr, ds: ptr, ds: ptr, ptrarg: ptr) { for (var i = 0; i < 10; i++) { (*ds).Integ *= f32(0.99); } var flag: NeuronFlags; flag &= ~NeuronHasExt; // clear flag -- op doesn't exist in C - var mode = Test; switch (mode) { // note: no fallthrough! case Test: { @@ -153,21 +80,183 @@ fn ParamStruct_AnotherMeth(ps: ptr, ds: ptr su64 { + let mul = Uint32Mul64(u32(0xD256D193), counter.x); + var ctr: su64; + ctr.x = mul.y ^ key ^ counter.y; + ctr.y = mul.x; + return ctr; +} +fn Philox2x32bumpkey(key: u32) -> u32 { + return key + u32(0x9E3779B9); +} +fn Philox2x32(counter: su64, key: u32) -> vec2 { + var ctr = Philox2x32round(counter, key); // 1 + var ky = Philox2x32bumpkey(key); + ctr = Philox2x32round(ctr, ky); // 2 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 3 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 4 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 5 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 6 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 7 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 8 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 9 + ky = Philox2x32bumpkey(ky); + return Philox2x32round(ctr, ky); // 10 +} +fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Philox2x32(Uint64Add32(counter, funcIndex), key); +} +fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { + return Philox2x32(Uint64Add32(counter, funcIndex), key).x; +} +fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} +fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); +} +fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} +fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); +} +fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { + return (RandFloat32(counter, funcIndex, key) < p); +} +fn sincospi(x: f32) -> vec2 { + let PIf = 3.1415926535897932; + var r: vec2; + r.x = cos(PIf*x); + r.y = sin(PIf*x); + return r; +} +fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + let ur = RandUint32Vec2(counter, funcIndex, key); + var f = sincospi(Uint32ToFloat32Range11(ur.x)); + let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. + return f * r; +} +fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { + return RandFloat32Vec2(counter, funcIndex, key).x; +} +fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { + let v = RandFloat32(counter, funcIndex, key); + return u32(v * f32(n)); +} +struct RandCounter { + Counter: su64, + HiSeed: u32, + pad: u32, +} +fn RandCounter_Reset(ct: ptr) { + (*ct).Counter.x = u32(0); + (*ct).Counter.y = (*ct).HiSeed; +} +fn RandCounter_Seed(ct: ptr, seed: u32) { + (*ct).HiSeed = seed; + RandCounter_Reset(ct); +} +fn RandCounter_Add(ct: ptr, inc: u32) { + (*ct).Counter = Uint64Add32((*ct).Counter, inc); +} +///////////// import: "sltype.wgsl" +alias su64 = vec2; +fn Uint32Mul64(a: u32, b: u32) -> su64 { + let LOMASK = (((u32(1))<<16)-1); + var r: su64; + r.x = a * b; /* full low multiply */ + let ahi = a >> 16; + let alo = a & LOMASK; + let bhi = b >> 16; + let blo = b & LOMASK; + let ahbl = ahi * blo; + let albh = alo * bhi; + let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); + var hit = ahi*bhi + (ahbl>>16) + (albh>>16); + hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ + /* carry from the sum with alo*blo */ + if ((r.x >> u32(16)) < (ahbl_albh&LOMASK)) { + hit += u32(1); + } + r.y = hit; + return r; +} +/* +fn Uint32Mul64(a: u32, b: u32) -> su64 { + return su64(a) * su64(b); +} +*/ +fn Uint64Add32(a: su64, b: u32) -> su64 { + if (b == 0) { + return a; + } + var s = a; + if (s.x > u32(0xffffffff) - b) { + s.y++; + s.x = (b - 1) - (u32(0xffffffff) - s.x); + } else { + s.x += b; + } + return s; +} +fn Uint64Incr(a: su64) -> su64 { + var s = a; + if(s.x == 0xffffffff) { + s.y++; + s.x = u32(0); + } else { + s.x++; + } + return s; +} +fn Uint32ToFloat32(val: u32) -> f32 { + let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + var f = f32(val) * factor + halffactor; + if (f == 1.0) { // exclude 1 + return bitcast(0x3F7FFFFF); + } + return f; +} +fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32(val.x); + r.y = Uint32ToFloat32(val.y); + return r; +} +fn Uint32ToFloat32Range11(val: u32) -> f32 { + let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + return (f32(val) * factor + halffactor); } +fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32Range11(val.x); + r.y = Uint32ToFloat32Range11(val.y); + return r; +} \ No newline at end of file diff --git a/goal/gosl/gotosl/testdata/basic.go b/goal/gosl/gotosl/testdata/basic.go index 42dca43c3a..6eb20ac6be 100644 --- a/goal/gosl/gotosl/testdata/basic.go +++ b/goal/gosl/gotosl/testdata/basic.go @@ -2,7 +2,6 @@ package test import ( "math" - "sync/atomic" "cogentcore.org/core/goal/gosl/slbool" "cogentcore.org/core/math32" @@ -120,7 +119,7 @@ func (ps *ParamStruct) IntegFromRaw(ds *DataStruct) float32 { if newVal < -10 || ps.Option.IsTrue() { newVal = -10 } - atomic.AddInt32(&(ds.Int), int32(newVal)) + // atomic.AddInt32(&(ds.Int), int32(newVal)) ds.Integ += newVal ds.Exp = math32.Exp(-ds.Integ) var a float32 diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index b4a5d91a7c..04e741f41b 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -136,6 +136,10 @@ func (st *State) TranslateDir(pf string) error { lines = doKernelFile(fn, lines) } for _, gofp := range files { + _, gofn := filepath.Split(gofp) + if _, ok := st.GoVarsFiles[gofn]; ok { + continue + } lines = doKernelFile(gofp, lines) } for _, im := range st.SLImportFiles { diff --git a/goal/gosl/gotosl/typegen.go b/goal/gosl/gotosl/typegen.go index 4f44675e55..01c9225066 100644 --- a/goal/gosl/gotosl/typegen.go +++ b/goal/gosl/gotosl/typegen.go @@ -6,6 +6,8 @@ import ( "cogentcore.org/core/types" ) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Function", IDName: "function", Doc: "Function represents the call graph of functions", Fields: []types.Field{{Name: "Name"}, {Name: "Funcs"}}}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Config", IDName: "config", Doc: "Config has the configuration info for the gosl system.", Fields: []types.Field{{Name: "Output", Doc: "Output is the output directory for shader code,\nrelative to where gosl is invoked; must not be an empty string."}, {Name: "Exclude", Doc: "Exclude is a comma-separated list of names of functions to exclude from exporting to WGSL."}, {Name: "Keep", Doc: "Keep keeps temporary converted versions of the source files, for debugging."}, {Name: "Debug", Doc: "\tDebug enables debugging messages while running."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.System", IDName: "system", Doc: "System represents a ComputeSystem, and its kernels and variables.", Fields: []types.Field{{Name: "Name"}, {Name: "Kernels", Doc: "Kernels are the kernels using this compute system."}, {Name: "Groups", Doc: "Groups are the variables for this compute system."}}}) @@ -20,7 +22,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Fi var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.GetGlobalVar", IDName: "get-global-var", Doc: "GetGlobalVar holds GetVar expression, to Set variable back when done.", Fields: []types.Field{{Name: "Var", Doc: "global variable"}, {Name: "TmpVar", Doc: "name of temporary variable"}, {Name: "IdxExpr", Doc: "index passed to the Get function"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current Go -> WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoVarsFiles", Doc: "GoVarsFiles are all the files with gosl:vars content in current directory.\nThese must be processed first! they are moved from GoFiles to here."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "GetFuncs", Doc: "GetFuncs is a map of GetVar, SetVar function names for global vars."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}, {Name: "GetVarStack", Doc: "GetVarStack is a stack per function definition of GetVar variables\nthat need to be set at the end."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.State", IDName: "state", Doc: "State holds the current Go -> WGSL processing state.", Fields: []types.Field{{Name: "Config", Doc: "Config options."}, {Name: "ImportsDir", Doc: "path to shaders/imports directory."}, {Name: "Package", Doc: "name of the package"}, {Name: "GoFiles", Doc: "GoFiles are all the files with gosl content in current directory."}, {Name: "GoVarsFiles", Doc: "GoVarsFiles are all the files with gosl:vars content in current directory.\nThese must be processed first! they are moved from GoFiles to here."}, {Name: "GoImports", Doc: "GoImports has all the imported files."}, {Name: "ImportPackages", Doc: "ImportPackages has short package names, to remove from go code\nso everything lives in same main package."}, {Name: "Systems", Doc: "Systems has the kernels and variables for each system.\nThere is an initial \"Default\" system when system is not specified."}, {Name: "GetFuncs", Doc: "GetFuncs is a map of GetVar, SetVar function names for global vars."}, {Name: "SLImportFiles", Doc: "SLImportFiles are all the extracted and translated WGSL files in shaders/imports,\nwhich are copied into the generated shader kernel files."}, {Name: "GPUFile", Doc: "generated Go GPU gosl.go file contents"}, {Name: "ExcludeMap", Doc: "ExcludeMap is the compiled map of functions to exclude in Go -> WGSL translation."}, {Name: "GetVarStack", Doc: "GetVarStack is a stack per function definition of GetVar variables\nthat need to be set at the end."}, {Name: "GetFuncGraph", Doc: "GetFuncGraph is true if getting the function graph (first pass)"}, {Name: "KernelFuncs", Doc: "KernelFuncs are the list of functions to include for current kernel."}, {Name: "FuncGraph", Doc: "FuncGraph is the call graph of functions, for dead code elimination"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.exprListMode", IDName: "expr-list-mode"}) @@ -36,7 +38,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.pm var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.commentInfo", IDName: "comment-info", Fields: []types.Field{{Name: "cindex"}, {Name: "comment"}, {Name: "commentOffset"}, {Name: "commentNewline"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.printer", IDName: "printer", Embeds: []types.Field{{Name: "PrintConfig", Doc: "Configuration (does not change after initialization)"}, {Name: "commentInfo", Doc: "Information about p.comments[p.cindex]; set up by nextComment."}}, Fields: []types.Field{{Name: "fset"}, {Name: "pkg"}, {Name: "output", Doc: "Current state"}, {Name: "indent"}, {Name: "level"}, {Name: "mode"}, {Name: "endAlignment"}, {Name: "impliedSemi"}, {Name: "lastTok"}, {Name: "prevOpen"}, {Name: "wsbuf"}, {Name: "goBuild"}, {Name: "plusBuild"}, {Name: "pos", Doc: "Positions\nThe out position differs from the pos position when the result\nformatting differs from the source formatting (in the amount of\nwhite space). If there's a difference and SourcePos is set in\nConfigMode, //line directives are used in the output to restore\noriginal source positions for a reader."}, {Name: "out"}, {Name: "last"}, {Name: "linePtr"}, {Name: "sourcePosErr"}, {Name: "comments", Doc: "The list of all source comments, in order of appearance."}, {Name: "useNodeComments"}, {Name: "nodeSizes", Doc: "Cache of already computed node sizes."}, {Name: "cachedPos", Doc: "Cache of most recently computed line position."}, {Name: "cachedLine"}, {Name: "curPtrArgs", Doc: "current arguments to function that are pointers and thus need dereferencing\nwhen accessing fields"}, {Name: "curMethRecv"}, {Name: "curReturnType"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.printer", IDName: "printer", Embeds: []types.Field{{Name: "PrintConfig", Doc: "Configuration (does not change after initialization)"}, {Name: "commentInfo", Doc: "Information about p.comments[p.cindex]; set up by nextComment."}}, Fields: []types.Field{{Name: "fset"}, {Name: "pkg"}, {Name: "output", Doc: "Current state"}, {Name: "indent"}, {Name: "level"}, {Name: "mode"}, {Name: "endAlignment"}, {Name: "impliedSemi"}, {Name: "lastTok"}, {Name: "prevOpen"}, {Name: "wsbuf"}, {Name: "goBuild"}, {Name: "plusBuild"}, {Name: "pos", Doc: "Positions\nThe out position differs from the pos position when the result\nformatting differs from the source formatting (in the amount of\nwhite space). If there's a difference and SourcePos is set in\nConfigMode, //line directives are used in the output to restore\noriginal source positions for a reader."}, {Name: "out"}, {Name: "last"}, {Name: "linePtr"}, {Name: "sourcePosErr"}, {Name: "comments", Doc: "The list of all source comments, in order of appearance."}, {Name: "useNodeComments"}, {Name: "nodeSizes", Doc: "Cache of already computed node sizes."}, {Name: "cachedPos", Doc: "Cache of most recently computed line position."}, {Name: "cachedLine"}, {Name: "curPtrArgs", Doc: "current arguments to function that are pointers and thus need dereferencing\nwhen accessing fields"}, {Name: "curFunc"}, {Name: "curMethRecv"}, {Name: "curReturnType"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.trimmer", IDName: "trimmer", Doc: "A trimmer is an io.Writer filter for stripping tabwriter.Escape\ncharacters, trailing blanks and tabs, and for converting formfeed\nand vtab characters into newlines and htabs (in case no tabwriter\nis used). Text bracketed by tabwriter.Escape characters is passed\nthrough unchanged.", Fields: []types.Field{{Name: "output"}, {Name: "state"}, {Name: "space"}}}) @@ -48,6 +50,10 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Co var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/goal/gosl/gotosl.Replace", IDName: "replace", Fields: []types.Field{{Name: "From"}, {Name: "To"}}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.NewFunction", Args: []string{"name"}, Returns: []string{"Function"}}) + +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.getAllFuncs", Args: []string{"f", "all"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.formatDocComment", Doc: "formatDocComment reformats the doc comment list,\nreturning the canonical formatting.", Args: []string{"list"}, Returns: []string{"Comment"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.isDirective", Doc: "isDirective reports whether c is a comment directive.\nSee go.dev/issue/37974.\nThis code is also in go/ast.", Args: []string{"c"}, Returns: []string{"bool"}}) @@ -142,6 +148,8 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.Sl var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.MathReplaceAll", Args: []string{"mat", "ln"}, Returns: []string{"[]byte"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlRemoveComments", Args: []string{"lines"}, Returns: []string{"[][]byte"}}) + var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlEditsReplace", Doc: "SlEditsReplace replaces Go with equivalent WGSL code\nreturns true if has slrand. or sltype.\nto auto include that header file if so.", Args: []string{"lines"}, Returns: []string{"bool", "bool"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/gosl/gotosl.SlBoolReplace", Doc: "SlBoolReplace replaces all the slbool methods with literal int32 expressions.", Args: []string{"lines"}}) From 97110180650ec79240cf32fd7c53484c6cf58b0e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 17 Oct 2024 11:34:42 -0700 Subject: [PATCH 246/311] important fix for enumgen, typegen: exclude enumgen.go, typegen.go files (but not generated files generically, which is key for goal files) --- base/fileinfo/typegen.go | 2 +- base/generate/generate.go | 26 +++++++++++++++++++------- enums/enumgen/generator.go | 8 ++++---- enums/enumgen/testdata/enumgen.go | 2 +- enums/enumgen/typegen.go | 2 -- filetree/enumgen.go | 2 +- filetree/typegen.go | 29 ++++++++++++++++++++++++----- goal/interpreter/typegen.go | 2 +- plot/plotcore/typegen.go | 9 +++++++-- plot/typegen.go | 2 +- spell/dict/typegen.go | 2 +- styles/typegen.go | 4 ++-- tensor/table/typegen.go | 2 +- texteditor/highlighting/typegen.go | 2 +- texteditor/typegen.go | 2 +- types/typegen/generator.go | 2 +- types/typegen/testdata/typegen.go | 2 +- types/typegen/typegen_gen.go | 2 -- xyz/typegen.go | 4 +--- 19 files changed, 68 insertions(+), 38 deletions(-) diff --git a/base/fileinfo/typegen.go b/base/fileinfo/typegen.go index 0b89bbbf00..e8f4015e30 100644 --- a/base/fileinfo/typegen.go +++ b/base/fileinfo/typegen.go @@ -6,4 +6,4 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/base/fileinfo.FileInfo", IDName: "file-info", Doc: "FileInfo represents the information about a given file / directory,\nincluding icon, mimetype, etc", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Duplicate", Doc: "Duplicate creates a copy of given file -- only works for regular files, not\ndirectories.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Returns: []string{"string", "error"}}, {Name: "Delete", Doc: "Delete moves the file to the trash / recycling bin.\nOn mobile and web, it deletes it directly.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Returns: []string{"error"}}, {Name: "Rename", Doc: "Rename renames (moves) this file to given new path name.\nUpdates the FileInfo setting to the new name, although it might\nbe out of scope if it moved into a new path", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"path"}, Returns: []string{"newpath", "err"}}}, Fields: []types.Field{{Name: "Ic", Doc: "icon for file"}, {Name: "Name", Doc: "name of the file, without any path"}, {Name: "Size", Doc: "size of the file"}, {Name: "Kind", Doc: "type of file / directory; shorter, more user-friendly\nversion of mime type, based on category"}, {Name: "Mime", Doc: "full official mime type of the contents"}, {Name: "Cat", Doc: "functional category of the file, based on mime data etc"}, {Name: "Known", Doc: "known file type"}, {Name: "Mode", Doc: "file mode bits"}, {Name: "ModTime", Doc: "time that contents (only) were last modified"}, {Name: "VCS", Doc: "version control system status, when enabled"}, {Name: "Path", Doc: "full path to file, including name; for file functions"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/base/fileinfo.FileInfo", IDName: "file-info", Doc: "FileInfo represents the information about a given file / directory,\nincluding icon, mimetype, etc", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Duplicate", Doc: "Duplicate creates a copy of given file -- only works for regular files, not\ndirectories.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Returns: []string{"string", "error"}}, {Name: "Delete", Doc: "Delete moves the file to the trash / recycling bin.\nOn mobile and web, it deletes it directly.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Returns: []string{"error"}}, {Name: "Rename", Doc: "Rename renames (moves) this file to given new path name.\nUpdates the FileInfo setting to the new name, although it might\nbe out of scope if it moved into a new path", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"path"}, Returns: []string{"newpath", "err"}}}, Fields: []types.Field{{Name: "Ic", Doc: "icon for file"}, {Name: "Name", Doc: "name of the file, without any path"}, {Name: "Size", Doc: "size of the file"}, {Name: "Kind", Doc: "type of file / directory; shorter, more user-friendly\nversion of mime type, based on category"}, {Name: "Mime", Doc: "full official mime type of the contents"}, {Name: "Cat", Doc: "functional category of the file, based on mime data etc"}, {Name: "Known", Doc: "known file type"}, {Name: "Mode", Doc: "file mode bits"}, {Name: "ModTime", Doc: "time that contents (only) were last modified"}, {Name: "VCS", Doc: "version control system status, when enabled"}, {Name: "Generated", Doc: "Generated indicates that the file is generated and should not be edited.\nFor Go files, this regex: `^// Code generated .* DO NOT EDIT\\.$` is used."}, {Name: "Path", Doc: "full path to file, including name; for file functions"}}}) diff --git a/base/generate/generate.go b/base/generate/generate.go index 3b8023851c..c2446d2723 100644 --- a/base/generate/generate.go +++ b/base/generate/generate.go @@ -54,18 +54,30 @@ func PrintHeader(w io.Writer, pkg string, imports ...string) { } } -// Inspect goes through all of the files in the given package -// and calls the given function on each node in files that -// are not generated. The bool return value from the given function +// ExcludeFile returns true if the given file is on the exclude list. +func ExcludeFile(pkg *packages.Package, file *ast.File, exclude ...string) bool { + fpos := pkg.Fset.Position(file.FileStart) + _, fname := filepath.Split(fpos.Filename) + for _, ex := range exclude { + if fname == ex { + return true + } + } + return false +} + +// Inspect goes through all of the files in the given package, +// except those listed in the exclude list, and calls the given +// function on each node. The bool return value from the given function // indicates whether to continue traversing down the AST tree // of that node and look at its children. If a non-nil error value // is returned by the given function, the traversal of the tree is // stopped and the error value is returned. -func Inspect(pkg *packages.Package, f func(n ast.Node) (bool, error)) error { +func Inspect(pkg *packages.Package, f func(n ast.Node) (bool, error), exclude ...string) error { for _, file := range pkg.Syntax { - // if ast.IsGenerated(file) { - // continue - // } + if ExcludeFile(pkg, file, exclude...) { + continue + } var terr error var terrNode ast.Node ast.Inspect(file, func(n ast.Node) bool { diff --git a/enums/enumgen/generator.go b/enums/enumgen/generator.go index 6a8b9c8b24..26405ba79a 100644 --- a/enums/enumgen/generator.go +++ b/enums/enumgen/generator.go @@ -70,7 +70,7 @@ func (g *Generator) PrintHeader() { // or enums:bitflag. It stores the resulting types in [Generator.Types]. func (g *Generator) FindEnumTypes() error { g.Types = []*Type{} - return generate.Inspect(g.Pkg, g.InspectForType) + return generate.Inspect(g.Pkg, g.InspectForType, "enumgen.go", "typegen.go") } // AllowedEnumTypes are the types that can be used for enums @@ -159,9 +159,9 @@ func (g *Generator) Generate() (bool, error) { for _, typ := range g.Types { values := make([]Value, 0, 100) for _, file := range g.Pkg.Syntax { - // if ast.IsGenerated(file) { - // continue - // } + if generate.ExcludeFile(g.Pkg, file, "enumgen.go", "typegen.go") { + continue + } var terr error ast.Inspect(file, func(n ast.Node) bool { if terr != nil { diff --git a/enums/enumgen/testdata/enumgen.go b/enums/enumgen/testdata/enumgen.go index 33366ee1a4..4826cd43fd 100644 --- a/enums/enumgen/testdata/enumgen.go +++ b/enums/enumgen/testdata/enumgen.go @@ -1,4 +1,4 @@ -// Code generated by "enumgen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build1497819794/b657/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. +// Code generated by "enumgen.test -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. package testdata diff --git a/enums/enumgen/typegen.go b/enums/enumgen/typegen.go index eae66358d7..ffa0a70002 100644 --- a/enums/enumgen/typegen.go +++ b/enums/enumgen/typegen.go @@ -7,5 +7,3 @@ import ( ) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/enums/enumgen.Config", IDName: "config", Doc: "Config contains the configuration information\nused by enumgen", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Dir", Doc: "the source directory to run enumgen on (can be set to multiple through paths like ./...)"}, {Name: "Output", Doc: "the output file location relative to the package on which enumgen is being called"}, {Name: "Transform", Doc: "if specified, the enum item transformation method (upper, lower, snake, SNAKE, kebab, KEBAB,\ncamel, lower-camel, title, sentence, first, first-upper, or first-lower)"}, {Name: "TrimPrefix", Doc: "if specified, a comma-separated list of prefixes to trim from each item"}, {Name: "AddPrefix", Doc: "if specified, the prefix to add to each item"}, {Name: "LineComment", Doc: "whether to use line comment text as printed text when present"}, {Name: "AcceptLower", Doc: "whether to accept lowercase versions of enum names in SetString"}, {Name: "IsValid", Doc: "whether to generate a method returning whether a value is\na valid option for its enum type; this must also be set for\nany base enum type being extended"}, {Name: "Text", Doc: "whether to generate text marshaling methods"}, {Name: "SQL", Doc: "whether to generate methods that implement the SQL Scanner and Valuer interfaces"}, {Name: "GQL", Doc: "whether to generate GraphQL marshaling methods for gqlgen"}, {Name: "Extend", Doc: "whether to allow enums to extend other enums; this should be on in almost all circumstances,\nbut can be turned off for specific enum types that extend non-enum types"}}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/enums/enumgen.Generate", Doc: "Generate generates enum methods, using the\nconfiguration information, loading the packages from the\nconfiguration source directory, and writing the result\nto the configuration output file.\n\nIt is a simple entry point to enumgen that does all\nof the steps; for more specific functionality, create\na new [Generator] with [NewGenerator] and call methods on it.", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) diff --git a/filetree/enumgen.go b/filetree/enumgen.go index 9d2068dcb7..f01aa7ef8b 100644 --- a/filetree/enumgen.go +++ b/filetree/enumgen.go @@ -13,7 +13,7 @@ const dirFlagsN dirFlags = 3 var _dirFlagsValueMap = map[string]dirFlags{`IsOpen`: 0, `SortByName`: 1, `SortByModTime`: 2} -var _dirFlagsDescMap = map[dirFlags]string{0: `dirIsOpen means directory is open -- else closed`, 1: `dirSortByName means sort the directory entries by name. this is mutex with other sorts -- keeping option open for non-binary sort choices.`, 2: `dirSortByModTime means sort the directory entries by modification time`} +var _dirFlagsDescMap = map[dirFlags]string{0: `dirIsOpen means directory is open -- else closed`, 1: `dirSortByName means sort the directory entries by name. this overrides SortByModTime default on Tree if set.`, 2: `dirSortByModTime means sort the directory entries by modification time.`} var _dirFlagsMap = map[dirFlags]string{0: `IsOpen`, 1: `SortByName`, 2: `SortByModTime`} diff --git a/filetree/typegen.go b/filetree/typegen.go index 376b56a57c..52030218eb 100644 --- a/filetree/typegen.go +++ b/filetree/typegen.go @@ -3,12 +3,14 @@ package filetree import ( + "io/fs" + "cogentcore.org/core/base/vcs" "cogentcore.org/core/tree" "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.Filer", IDName: "filer", Doc: "Filer is an interface for file tree file actions that all [Node]s satisfy.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "AsFileNode", Doc: "AsFileNode returns the [Node]", Returns: []string{"Node"}}, {Name: "RenameFiles", Doc: "RenameFiles renames any selected files."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.Filer", IDName: "filer", Doc: "Filer is an interface for file tree file actions that all [Node]s satisfy.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "AsFileNode", Doc: "AsFileNode returns the [Node]", Returns: []string{"Node"}}, {Name: "RenameFiles", Doc: "RenameFiles renames any selected files."}, {Name: "GetFileInfo", Doc: "GetFileInfo updates the .Info for this file", Returns: []string{"error"}}, {Name: "OpenFile", Doc: "OpenFile opens the file for node. This is called by OpenFilesDefault", Returns: []string{"error"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.Node", IDName: "node", Doc: "Node represents a file in the file system, as a [core.Tree] node.\nThe name of the node is the name of the file.\nFolders have children containing further nodes.", Directives: []types.Directive{{Tool: "core", Directive: "embedder"}}, Methods: []types.Method{{Name: "Cut", Doc: "Cut copies the selected files to the clipboard and then deletes them.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "Paste", Doc: "Paste inserts files from the clipboard.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "OpenFilesDefault", Doc: "OpenFilesDefault opens selected files with default app for that file type (os defined).\nruns open on Mac, xdg-open on Linux, and start on Windows", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "duplicateFiles", Doc: "duplicateFiles makes a copy of selected files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "deleteFiles", Doc: "deletes any selected files or directories. If any directory is selected,\nall files and subdirectories in that directory are also deleted.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "RenameFiles", Doc: "renames any selected files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "RenameFile", Doc: "RenameFile renames file to new name", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"newpath"}, Returns: []string{"error"}}, {Name: "newFiles", Doc: "newFiles makes a new file in selected directory", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "addToVCS"}}, {Name: "newFile", Doc: "newFile makes a new file in this directory node", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "addToVCS"}}, {Name: "newFolders", Doc: "makes a new folder in the given selected directory", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"foldername"}}, {Name: "newFolder", Doc: "newFolder makes a new folder (directory) in this directory node", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"foldername"}}, {Name: "showFileInfo", Doc: "Shows file information about selected file(s)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "sortBys", Doc: "sortBys determines how to sort the selected files in the directory.\nDefault is alpha by name, optionally can be sorted by modification time.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"modTime"}}, {Name: "openAll", Doc: "openAll opens all directories under this one", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "removeFromExterns", Doc: "removeFromExterns removes file from list of external files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "addToVCSSelected", Doc: "addToVCSSelected adds selected files to version control system", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "deleteFromVCSSelected", Doc: "deleteFromVCSSelected removes selected files from version control system", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "commitToVCSSelected", Doc: "commitToVCSSelected commits to version control system based on last selected file", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "revertVCSSelected", Doc: "revertVCSSelected removes selected files from version control system", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "diffVCSSelected", Doc: "diffVCSSelected shows the diffs between two versions of selected files, given by the\nrevision specifiers -- if empty, defaults to A = current HEAD, B = current WC file.\n-1, -2 etc also work as universal ways of specifying prior revisions.\nDiffs are shown in a DiffEditorDialog.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rev_a", "rev_b"}}, {Name: "logVCSSelected", Doc: "logVCSSelected shows the VCS log of commits for selected files.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "blameVCSSelected", Doc: "blameVCSSelected shows the VCS blame report for this file, reporting for each line\nthe revision and author of the last change.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Tree"}}, Fields: []types.Field{{Name: "Filepath", Doc: "Filepath is the full path to this file."}, {Name: "Info", Doc: "Info is the full standard file info about this file."}, {Name: "Buffer", Doc: "Buffer is the file buffer for editing this file."}, {Name: "FileRoot", Doc: "FileRoot is the root [Tree] of the tree, which has global state."}, {Name: "DirRepo", Doc: "DirRepo is the version control system repository for this directory,\nonly non-nil if this is the highest-level directory in the tree under vcs control."}, {Name: "repoFiles", Doc: "repoFiles has the version control system repository file status,\nproviding a much faster way to get file status, vs. the repo.Status\ncall which is exceptionally slow."}}}) @@ -35,7 +37,7 @@ func AsNode(n tree.Node) *Node { // AsNode satisfies the [NodeEmbedder] interface func (t *Node) AsNode() *Node { return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.Tree", IDName: "tree", Doc: "Tree is the root widget of a file tree representing files in a given directory\n(and subdirectories thereof), and has some overall management state for how to\nview things.", Embeds: []types.Field{{Name: "Node"}}, Fields: []types.Field{{Name: "externalFiles", Doc: "externalFiles are external files outside the root path of the tree.\nThey are stored in terms of their absolute paths. These are shown\nin the first sub-node if present; use [Tree.AddExternalFile] to add one."}, {Name: "Dirs", Doc: "records state of directories within the tree (encoded using paths relative to root),\ne.g., open (have been opened by the user) -- can persist this to restore prior view of a tree"}, {Name: "DirsOnTop", Doc: "if true, then all directories are placed at the top of the tree.\nOtherwise everything is mixed."}, {Name: "FileNodeType", Doc: "type of node to create; defaults to [Node] but can use custom node types"}, {Name: "inOpenAll", Doc: "if true, we are in midst of an OpenAll call; nodes should open all dirs"}, {Name: "watcher", Doc: "change notify for all dirs"}, {Name: "doneWatcher", Doc: "channel to close watcher watcher"}, {Name: "watchedPaths", Doc: "map of paths that have been added to watcher; only active if bool = true"}, {Name: "lastWatchUpdate", Doc: "last path updated by watcher"}, {Name: "lastWatchTime", Doc: "timestamp of last update"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.Tree", IDName: "tree", Doc: "Tree is the root widget of a file tree representing files in a given directory\n(and subdirectories thereof), and has some overall management state for how to\nview things.", Embeds: []types.Field{{Name: "Node"}}, Fields: []types.Field{{Name: "externalFiles", Doc: "externalFiles are external files outside the root path of the tree.\nThey are stored in terms of their absolute paths. These are shown\nin the first sub-node if present; use [Tree.AddExternalFile] to add one."}, {Name: "Dirs", Doc: "Dirs records state of directories within the tree (encoded using paths relative to root),\ne.g., open (have been opened by the user) -- can persist this to restore prior view of a tree"}, {Name: "DirsOnTop", Doc: "DirsOnTop indicates whether all directories are placed at the top of the tree.\nOtherwise everything is mixed. This is the default."}, {Name: "SortByModTime", Doc: "SortByModTime causes files to be sorted by modification time by default.\nOtherwise it is a per-directory option."}, {Name: "FileNodeType", Doc: "FileNodeType is the type of node to create; defaults to [Node] but can use custom node types"}, {Name: "FilterFunc", Doc: "FilterFunc, if set, determines whether to include the given node in the tree.\nreturn true to include, false to not. This applies to files and directories alike."}, {Name: "FS", Doc: "FS is the file system we are browsing, if it is an FS (nil = os filesystem)"}, {Name: "inOpenAll", Doc: "inOpenAll indicates whether we are in midst of an OpenAll call; nodes should open all dirs."}, {Name: "watcher", Doc: "watcher does change notify for all dirs"}, {Name: "doneWatcher", Doc: "doneWatcher is channel to close watcher watcher"}, {Name: "watchedPaths", Doc: "watchedPaths is map of paths that have been added to watcher; only active if bool = true"}, {Name: "lastWatchUpdate", Doc: "lastWatchUpdate is last path updated by watcher"}, {Name: "lastWatchTime", Doc: "lastWatchTime is timestamp of last update"}}}) // NewTree returns a new [Tree] with the given optional parent: // Tree is the root widget of a file tree representing files in a given directory @@ -44,14 +46,31 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.Tree", IDN func NewTree(parent ...tree.Node) *Tree { return tree.New[Tree](parent...) } // SetDirsOnTop sets the [Tree.DirsOnTop]: -// if true, then all directories are placed at the top of the tree. -// Otherwise everything is mixed. +// DirsOnTop indicates whether all directories are placed at the top of the tree. +// Otherwise everything is mixed. This is the default. func (t *Tree) SetDirsOnTop(v bool) *Tree { t.DirsOnTop = v; return t } +// SetSortByModTime sets the [Tree.SortByModTime]: +// SortByModTime causes files to be sorted by modification time by default. +// Otherwise it is a per-directory option. +func (t *Tree) SetSortByModTime(v bool) *Tree { t.SortByModTime = v; return t } + // SetFileNodeType sets the [Tree.FileNodeType]: -// type of node to create; defaults to [Node] but can use custom node types +// FileNodeType is the type of node to create; defaults to [Node] but can use custom node types func (t *Tree) SetFileNodeType(v *types.Type) *Tree { t.FileNodeType = v; return t } +// SetFilterFunc sets the [Tree.FilterFunc]: +// FilterFunc, if set, determines whether to include the given node in the tree. +// return true to include, false to not. This applies to files and directories alike. +func (t *Tree) SetFilterFunc(v func(path string, info fs.FileInfo) bool) *Tree { + t.FilterFunc = v + return t +} + +// SetFS sets the [Tree.FS]: +// FS is the file system we are browsing, if it is an FS (nil = os filesystem) +func (t *Tree) SetFS(v fs.FS) *Tree { t.FS = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/filetree.VCSLog", IDName: "vcs-log", Doc: "VCSLog is a widget that represents VCS log data.", Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "Log", Doc: "current log"}, {Name: "File", Doc: "file that this is a log of -- if blank then it is entire repository"}, {Name: "Since", Doc: "date expression for how long ago to include log entries from"}, {Name: "Repo", Doc: "version control system repository"}, {Name: "revisionA", Doc: "revision A -- defaults to HEAD"}, {Name: "revisionB", Doc: "revision B -- blank means current working copy"}, {Name: "setA", Doc: "double-click will set the A revision -- else B"}, {Name: "arev"}, {Name: "brev"}, {Name: "atf"}, {Name: "btf"}}}) // NewVCSLog returns a new [VCSLog] with the given optional parent: diff --git a/goal/interpreter/typegen.go b/goal/interpreter/typegen.go index b8836bd6be..2060f498c6 100644 --- a/goal/interpreter/typegen.go +++ b/goal/interpreter/typegen.go @@ -14,7 +14,7 @@ var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Ru var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Interactive", Doc: "Interactive runs an interactive shell that allows the user to input goal.", Args: []string{"c", "in"}, Returns: []string{"error"}}) -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Build", Doc: "Build builds the specified input goal file, or all .goal files in the current\ndirectory if no input is specified, to corresponding .go file name(s).\nIf the file does not already contain a \"package\" specification, then\n\"package main; func main()...\" wrappers are added, which allows the same\ncode to be used in interactive and Go compiled modes.", Args: []string{"c"}, Returns: []string{"error"}}) +var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.Build", Doc: "Build builds the specified input goal file, or all .goal files in the current\ndirectory if no input is specified, to corresponding .go file name(s).\nIf the file does not already contain a \"package\" specification, then\n\"package main; func main()...\" wrappers are added, which allows the same\ncode to be used in interactive and Go compiled modes.\ngo build is run after this.", Args: []string{"c"}, Returns: []string{"error"}}) var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/goal/interpreter.init"}) diff --git a/plot/plotcore/typegen.go b/plot/plotcore/typegen.go index 2dbff3ebbd..9db4fb21c9 100644 --- a/plot/plotcore/typegen.go +++ b/plot/plotcore/typegen.go @@ -9,9 +9,9 @@ import ( var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotOptions", IDName: "plot-options", Doc: "PlotOptions are options for the overall plot.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Title", Doc: "optional title at top of plot"}, {Name: "Type", Doc: "type of plot to generate. For a Bar plot, items are plotted ordinally by row and the XAxis is optional"}, {Name: "Lines", Doc: "whether to plot lines"}, {Name: "Points", Doc: "whether to plot points with symbols"}, {Name: "LineWidth", Doc: "width of lines"}, {Name: "PointSize", Doc: "size of points"}, {Name: "PointShape", Doc: "the shape used to draw points"}, {Name: "BarWidth", Doc: "width of bars for bar plot, as fraction of available space (1 = no gaps)"}, {Name: "NegativeXDraw", Doc: "if true, draw lines that connect points with a negative X-axis direction;\notherwise there is a break in the line.\ndefault is false, so that repeated series of data across the X axis\nare plotted separately."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "XAxis", Doc: "what column to use for the common X axis. if empty or not found,\nthe row number is used. This optional for Bar plots, if present and\nLegend is also present, then an extra space will be put between X values."}, {Name: "Legend", Doc: "optional column for adding a separate colored / styled line or bar\naccording to this value, and acts just like a separate Y variable,\ncrossed with Y variables."}, {Name: "LegendPosition", Doc: "position of the Legend"}, {Name: "XAxisRotation", Doc: "rotation of the X Axis labels, in degrees"}, {Name: "XAxisLabel", Doc: "optional label to use for XAxis instead of column name"}, {Name: "YAxisLabel", Doc: "optional label to use for YAxis -- if empty, first column name is used"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.ColumnOptions", IDName: "column-options", Doc: "ColumnOptions are options for plotting one column of data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "On", Doc: "whether to plot this column"}, {Name: "Column", Doc: "name of column being plotting"}, {Name: "Lines", Doc: "whether to plot lines; uses the overall plot option if unset"}, {Name: "Points", Doc: "whether to plot points with symbols; uses the overall plot option if unset"}, {Name: "LineWidth", Doc: "the width of lines; uses the overall plot option if unset"}, {Name: "PointSize", Doc: "the size of points; uses the overall plot option if unset"}, {Name: "PointShape", Doc: "the shape used to draw points; uses the overall plot option if unset"}, {Name: "Range", Doc: "effective range of data to plot -- either end can be fixed"}, {Name: "FullRange", Doc: "full actual range of data -- only valid if specifically computed"}, {Name: "Color", Doc: "color to use when plotting the line / column"}, {Name: "NTicks", Doc: "desired number of ticks"}, {Name: "Label", Doc: "if specified, this is an alternative label to use when plotting"}, {Name: "TensorIndex", Doc: "if column has n-dimensional tensor cells in each row, this is the index within each cell to plot -- use -1 to plot *all* indexes as separate lines"}, {Name: "ErrColumn", Doc: "specifies a column containing error bars for this column"}, {Name: "IsString", Doc: "if true this is a string column -- plots as labels"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.ColumnOptions", IDName: "column-options", Doc: "ColumnOptions are options for plotting one column of data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "On", Doc: "whether to plot this column"}, {Name: "Column", Doc: "name of column being plotted"}, {Name: "Lines", Doc: "whether to plot lines; uses the overall plot option if unset"}, {Name: "Points", Doc: "whether to plot points with symbols; uses the overall plot option if unset"}, {Name: "LineWidth", Doc: "the width of lines; uses the overall plot option if unset"}, {Name: "PointSize", Doc: "the size of points; uses the overall plot option if unset"}, {Name: "PointShape", Doc: "the shape used to draw points; uses the overall plot option if unset"}, {Name: "Range", Doc: "effective range of data to plot -- either end can be fixed"}, {Name: "FullRange", Doc: "full actual range of data -- only valid if specifically computed"}, {Name: "Color", Doc: "color to use when plotting the line / column"}, {Name: "NTicks", Doc: "desired number of ticks"}, {Name: "Label", Doc: "if specified, this is an alternative label to use when plotting"}, {Name: "TensorIndex", Doc: "if column has n-dimensional tensor cells in each row, this is the index within each cell to plot -- use -1 to plot *all* indexes as separate lines"}, {Name: "ErrColumn", Doc: "specifies a column containing error bars for this column"}, {Name: "IsString", Doc: "if true this is a string column -- plots as labels"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plot", IDName: "plot", Doc: "Plot is a widget that renders a [plot.Plot] object.\nIf it is not [states.ReadOnly], the user can pan and zoom the graph.\nSee [PlotEditor] for an interactive interface for selecting columns to view.", Embeds: []types.Field{{Name: "WidgetBase"}}, Fields: []types.Field{{Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Plot", Doc: "Plot is the Plot to display in this widget"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plot", IDName: "plot", Doc: "Plot is a widget that renders a [plot.Plot] object.\nIf it is not [states.ReadOnly], the user can pan and zoom the graph.\nSee [PlotEditor] for an interactive interface for selecting columns to view.", Embeds: []types.Field{{Name: "WidgetBase"}}, Fields: []types.Field{{Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Plot", Doc: "Plot is the Plot to display in this widget"}, {Name: "SetRangesFunc", Doc: "SetRangesFunc, if set, is called to adjust the data ranges\nafter the point when these ranges are updated based on the plot data."}}}) // NewPlot returns a new [Plot] with the given optional parent: // Plot is a widget that renders a [plot.Plot] object. @@ -26,6 +26,11 @@ func NewPlot(parent ...tree.Node) *Plot { return tree.New[Plot](parent...) } // documents or other cases where the overall plot size will be small. func (t *Plot) SetScale(v float32) *Plot { t.Scale = v; return t } +// SetSetRangesFunc sets the [Plot.SetRangesFunc]: +// SetRangesFunc, if set, is called to adjust the data ranges +// after the point when these ranges are updated based on the plot data. +func (t *Plot) SetSetRangesFunc(v func()) *Plot { t.SetRangesFunc = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Table] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "Options", Doc: "Options are the overall plot options."}, {Name: "Columns", Doc: "Columns are the options for each column of the table."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) // NewPlotEditor returns a new [PlotEditor] with the given optional parent: diff --git a/plot/typegen.go b/plot/typegen.go index cc812fd88e..c076e347ab 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -56,7 +56,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nSome standard implementations of Plotter can be found in plotters.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DataRanger", IDName: "data-ranger", Doc: "DataRanger wraps the DataRange method.", Methods: []types.Method{{Name: "DataRange", Doc: "DataRange returns the range of X and Y values.", Returns: []string{"xmin", "xmax", "ymin", "ymax"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DataRanger", IDName: "data-ranger", Doc: "DataRanger wraps the DataRange method.", Methods: []types.Method{{Name: "DataRange", Doc: "DataRange returns the range of X and Y values.", Args: []string{"pt"}, Returns: []string{"xmin", "xmax", "ymin", "ymax"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TextStyle", IDName: "text-style", Doc: "TextStyle specifies styling parameters for Text elements", Embeds: []types.Field{{Name: "FontRender"}}, Fields: []types.Field{{Name: "Align", Doc: "how to align text along the relevant dimension for the text element"}, {Name: "Padding", Doc: "Padding is used in a case-dependent manner to add space around text elements"}, {Name: "Rotation", Doc: "rotation of the text, in Degrees"}}}) diff --git a/spell/dict/typegen.go b/spell/dict/typegen.go index 7fcd3f4f0a..af1751e4be 100644 --- a/spell/dict/typegen.go +++ b/spell/dict/typegen.go @@ -6,7 +6,7 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "main.Config", IDName: "config", Doc: "Config is the configuration information for the cosh cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "InputA", Doc: "InputA is the first input dictionary file"}, {Name: "InputB", Doc: "InputB is the second input dictionary file"}, {Name: "Output", Doc: "Output is the output file for merge command"}}}) +var _ = types.AddType(&types.Type{Name: "main.Config", IDName: "config", Doc: "Config is the configuration information for the dict cli.", Directives: []types.Directive{{Tool: "go", Directive: "generate", Args: []string{"core", "generate", "-add-types", "-add-funcs"}}}, Fields: []types.Field{{Name: "InputA", Doc: "InputA is the first input dictionary file"}, {Name: "InputB", Doc: "InputB is the second input dictionary file"}, {Name: "Output", Doc: "Output is the output file for merge command"}}}) var _ = types.AddFunc(&types.Func{Name: "main.Compare", Doc: "Compare compares two dictionaries", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}}, Args: []string{"c"}, Returns: []string{"error"}}) diff --git a/styles/typegen.go b/styles/typegen.go index 15bfced6d0..e056b35a6d 100644 --- a/styles/typegen.go +++ b/styles/typegen.go @@ -10,7 +10,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Border", IDN var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Shadow", IDName: "shadow", Doc: "style parameters for shadows", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "OffsetX", Doc: "OffsetX is th horizontal offset of the shadow.\nPositive moves it right, negative moves it left."}, {Name: "OffsetY", Doc: "OffsetY is the vertical offset of the shadow.\nPositive moves it down, negative moves it up."}, {Name: "Blur", Doc: "Blur specifies the blur radius of the shadow.\nHigher numbers make it more blurry."}, {Name: "Spread", Doc: "Spread specifies the spread radius of the shadow.\nPositive numbers increase the size of the shadow,\nand negative numbers decrease the size."}, {Name: "Color", Doc: "Color specifies the color of the shadow."}, {Name: "Inset", Doc: "Inset specifies whether the shadow is inset within the\nbox instead of outset outside of the box.\nTODO: implement."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Font", IDName: "font", Doc: "Font contains all font styling information.\nMost of font information is inherited.\nFont does not include all information needed\nfor rendering -- see [FontRender] for that.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Size", Doc: "size of font to render (inherited); converted to points when getting font to use"}, {Name: "Family", Doc: "font family (inherited): ordered list of comma-separated names from more general to more specific to use; use split on , to parse"}, {Name: "Style", Doc: "style (inherited): normal, italic, etc"}, {Name: "Weight", Doc: "weight (inherited): normal, bold, etc"}, {Name: "Stretch", Doc: "font stretch / condense options (inherited)"}, {Name: "Variant", Doc: "normal or small caps (inherited)"}, {Name: "Decoration", Doc: "underline, line-through, etc (not inherited)"}, {Name: "Shift", Doc: "super / sub script (not inherited)"}, {Name: "Face", Doc: "full font information including enhanced metrics and actual font codes for drawing text; this is a pointer into FontLibrary of loaded fonts"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Font", IDName: "font", Doc: "Font contains all font styling information.\nMost of font information is inherited.\nFont does not include all information needed\nfor rendering -- see [FontRender] for that.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Size", Doc: "size of font to render (inherited); converted to points when getting font to use"}, {Name: "Family", Doc: "font family (inherited): ordered list of comma-separated names from more general to more specific to use; use split on , to parse"}, {Name: "Style", Doc: "style (inherited): normal, italic, etc"}, {Name: "Weight", Doc: "weight (inherited): normal, bold, etc"}, {Name: "Stretch", Doc: "font stretch / condense options (inherited)"}, {Name: "Variant", Doc: "normal or small caps (inherited)"}, {Name: "Decoration", Doc: "Decoration contains the bit flag [TextDecorations]\n(underline, line-through, etc). It must be set using\n[Font.SetDecoration] since it contains bit flags.\nIt is not inherited."}, {Name: "Shift", Doc: "super / sub script (not inherited)"}, {Name: "Face", Doc: "full font information including enhanced metrics and actual font codes for drawing text; this is a pointer into FontLibrary of loaded fonts"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.FontRender", IDName: "font-render", Doc: "FontRender contains all font styling information\nthat is needed for SVG text rendering. It is passed to\nPaint and Style functions. It should typically not be\nused by end-user code -- see [Font] for that.\nIt stores all values as pointers so that they correspond\nto the values of the style object it was derived from.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Embeds: []types.Field{{Name: "Font"}}, Fields: []types.Field{{Name: "Color", Doc: "text color (inherited)"}, {Name: "Background", Doc: "background color (not inherited, transparent by default)"}, {Name: "Opacity", Doc: "alpha value between 0 and 1 to apply to the foreground and background of this element and all of its children"}}}) @@ -30,7 +30,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.SideFloats", var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.SideColors", IDName: "side-colors", Doc: "SideColors contains color values for each side/corner of a box", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Embeds: []types.Field{{Name: "Sides"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Style", IDName: "style", Doc: "Style contains all of the style properties used for GUI widgets.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "State", Doc: "State holds style-relevant state flags, for convenient styling access,\ngiven that styles typically depend on element states."}, {Name: "Abilities", Doc: "Abilities specifies the abilities of this element, which determine\nwhich kinds of states the element can express.\nThis is used by the system/events system. Putting this info next\nto the State info makes it easy to configure and manage."}, {Name: "Cursor", Doc: "the cursor to switch to upon hovering over the element (inherited)"}, {Name: "Padding", Doc: "Padding is the transparent space around central content of box,\nwhich is _included_ in the size of the standard box rendering."}, {Name: "Margin", Doc: "Margin is the outer-most transparent space around box element,\nwhich is _excluded_ from standard box rendering."}, {Name: "Display", Doc: "Display controls how items are displayed, in terms of layout"}, {Name: "Direction", Doc: "Direction specifies the way in which elements are laid out, or\nthe dimension on which an element is longer / travels in."}, {Name: "Wrap", Doc: "Wrap causes elements to wrap around in the CrossAxis dimension\nto fit within sizing constraints (on by default)."}, {Name: "Justify", Doc: "Justify specifies the distribution of elements along the main axis,\ni.e., the same as Direction, for Flex Display. For Grid, the main axis is\ngiven by the writing direction (e.g., Row-wise for latin based languages)."}, {Name: "Align", Doc: "Align specifies the cross-axis alignment of elements, orthogonal to the\nmain Direction axis. For Grid, the cross-axis is orthogonal to the\nwriting direction (e.g., Column-wise for latin based languages)."}, {Name: "Min", Doc: "Min is the minimum size of the actual content, exclusive of additional space\nfrom padding, border, margin; 0 = default is sum of Min for all content\n(which _includes_ space for all sub-elements).\nThis is equivalent to the Basis for the CSS flex styling model."}, {Name: "Max", Doc: "Max is the maximum size of the actual content, exclusive of additional space\nfrom padding, border, margin; 0 = default provides no Max size constraint"}, {Name: "Grow", Doc: "Grow is the proportional amount that the element can grow (stretch)\nif there is more space available. 0 = default = no growth.\nExtra available space is allocated as: Grow / sum (all Grow).\nImportant: grow elements absorb available space and thus are not\nsubject to alignment (Center, End)."}, {Name: "GrowWrap", Doc: "GrowWrap is a special case for Text elements where it grows initially\nin the horizontal axis to allow for longer, word wrapped text to fill\nthe available space, but then it does not grow thereafter, so that alignment\noperations still work (Grow elements do not align because they absorb all\navailable space)."}, {Name: "RenderBox", Doc: "RenderBox determines whether to render the standard box model for the element.\nThis is typically necessary for most elements and helps prevent text, border,\nand box shadow from rendering over themselves. Therefore, it should be kept at\nits default value of true in most circumstances, but it can be set to false\nwhen the element is fully managed by something that is guaranteed to render the\nappropriate background color and/or border for the element."}, {Name: "FillMargin", Doc: "FillMargin determines is whether to fill the margin with\nthe surrounding background color before rendering the element itself.\nThis is typically necessary to prevent text, border, and box shadow from\nrendering over themselves. Therefore, it should be kept at its default value\nof true in most circumstances, but it can be set to false when the element\nis fully managed by something that is guaranteed to render the\nappropriate background color for the element. It is irrelevant if RenderBox\nis false."}, {Name: "Overflow", Doc: "Overflow determines how to handle overflowing content in a layout.\nDefault is OverflowVisible. Set to OverflowAuto to enable scrollbars."}, {Name: "Gap", Doc: "For layouts, extra space added between elements in the layout."}, {Name: "Columns", Doc: "For grid layouts, the number of columns to use.\nIf > 0, number of rows is computed as N elements / Columns.\nUsed as a constraint in layout if individual elements\ndo not specify their row, column positions"}, {Name: "ObjectFit", Doc: "If this object is a replaced object (image, video, etc)\nor has a background image, ObjectFit specifies the way\nin which the replaced object should be fit into the element."}, {Name: "ObjectPosition", Doc: "If this object is a replaced object (image, video, etc)\nor has a background image, ObjectPosition specifies the\nX,Y position of the object within the space allocated for\nthe object (see ObjectFit)."}, {Name: "Border", Doc: "Border is a rendered border around the element."}, {Name: "MaxBorder", Doc: "MaxBorder is the largest border that will ever be rendered\naround the element, the size of which is used for computing\nthe effective margin to allocate for the element."}, {Name: "BoxShadow", Doc: "BoxShadow is the box shadows to render around box (can have multiple)"}, {Name: "MaxBoxShadow", Doc: "MaxBoxShadow contains the largest shadows that will ever be rendered\naround the element, the size of which are used for computing the\neffective margin to allocate for the element."}, {Name: "Color", Doc: "Color specifies the text / content color, and it is inherited."}, {Name: "Background", Doc: "Background specifies the background of the element. It is not inherited,\nand it is nil (transparent) by default."}, {Name: "Opacity", Doc: "alpha value between 0 and 1 to apply to the foreground and background of this element and all of its children"}, {Name: "StateLayer", Doc: "StateLayer, if above zero, indicates to create a state layer over the element with this much opacity (on a scale of 0-1) and the\ncolor Color (or StateColor if it defined). It is automatically set based on State, but can be overridden in stylers."}, {Name: "StateColor", Doc: "StateColor, if not nil, is the color to use for the StateLayer instead of Color. If you want to disable state layers\nfor an element, do not use this; instead, set StateLayer to 0."}, {Name: "ActualBackground", Doc: "ActualBackground is the computed actual background rendered for the element,\ntaking into account its Background, Opacity, StateLayer, and parent\nActualBackground. It is automatically computed and should not be set manually."}, {Name: "VirtualKeyboard", Doc: "VirtualKeyboard is the virtual keyboard to display, if any,\non mobile platforms when this element is focused. It is not\nused if the element is read only."}, {Name: "Pos", Doc: "position is only used for Layout = Nil cases"}, {Name: "ZIndex", Doc: "ordering factor for rendering depth -- lower numbers rendered first.\nSort children according to this factor"}, {Name: "Row", Doc: "specifies the row that this element should appear within a grid layout"}, {Name: "Col", Doc: "specifies the column that this element should appear within a grid layout"}, {Name: "RowSpan", Doc: "specifies the number of sequential rows that this element should occupy\nwithin a grid layout (todo: not currently supported)"}, {Name: "ColSpan", Doc: "specifies the number of sequential columns that this element should occupy\nwithin a grid layout"}, {Name: "ScrollbarWidth", Doc: "ScrollbarWidth is the width of layout scrollbars. It defaults\nto [DefaultScrollbarWidth], and it is inherited."}, {Name: "Font", Doc: "font styling parameters"}, {Name: "Text", Doc: "text styling parameters"}, {Name: "UnitContext", Doc: "unit context: parameters necessary for anchoring relative units"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Style", IDName: "style", Doc: "Style contains all of the style properties used for GUI widgets.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "State", Doc: "State holds style-relevant state flags, for convenient styling access,\ngiven that styles typically depend on element states."}, {Name: "Abilities", Doc: "Abilities specifies the abilities of this element, which determine\nwhich kinds of states the element can express.\nThis is used by the system/events system. Putting this info next\nto the State info makes it easy to configure and manage."}, {Name: "Cursor", Doc: "the cursor to switch to upon hovering over the element (inherited)"}, {Name: "Padding", Doc: "Padding is the transparent space around central content of box,\nwhich is _included_ in the size of the standard box rendering."}, {Name: "Margin", Doc: "Margin is the outer-most transparent space around box element,\nwhich is _excluded_ from standard box rendering."}, {Name: "Display", Doc: "Display controls how items are displayed, in terms of layout"}, {Name: "Direction", Doc: "Direction specifies the way in which elements are laid out, or\nthe dimension on which an element is longer / travels in."}, {Name: "Wrap", Doc: "Wrap causes elements to wrap around in the CrossAxis dimension\nto fit within sizing constraints."}, {Name: "Justify", Doc: "Justify specifies the distribution of elements along the main axis,\ni.e., the same as Direction, for Flex Display. For Grid, the main axis is\ngiven by the writing direction (e.g., Row-wise for latin based languages)."}, {Name: "Align", Doc: "Align specifies the cross-axis alignment of elements, orthogonal to the\nmain Direction axis. For Grid, the cross-axis is orthogonal to the\nwriting direction (e.g., Column-wise for latin based languages)."}, {Name: "Min", Doc: "Min is the minimum size of the actual content, exclusive of additional space\nfrom padding, border, margin; 0 = default is sum of Min for all content\n(which _includes_ space for all sub-elements).\nThis is equivalent to the Basis for the CSS flex styling model."}, {Name: "Max", Doc: "Max is the maximum size of the actual content, exclusive of additional space\nfrom padding, border, margin; 0 = default provides no Max size constraint"}, {Name: "Grow", Doc: "Grow is the proportional amount that the element can grow (stretch)\nif there is more space available. 0 = default = no growth.\nExtra available space is allocated as: Grow / sum (all Grow).\nImportant: grow elements absorb available space and thus are not\nsubject to alignment (Center, End)."}, {Name: "GrowWrap", Doc: "GrowWrap is a special case for Text elements where it grows initially\nin the horizontal axis to allow for longer, word wrapped text to fill\nthe available space, but then it does not grow thereafter, so that alignment\noperations still work (Grow elements do not align because they absorb all\navailable space)."}, {Name: "RenderBox", Doc: "RenderBox determines whether to render the standard box model for the element.\nThis is typically necessary for most elements and helps prevent text, border,\nand box shadow from rendering over themselves. Therefore, it should be kept at\nits default value of true in most circumstances, but it can be set to false\nwhen the element is fully managed by something that is guaranteed to render the\nappropriate background color and/or border for the element."}, {Name: "FillMargin", Doc: "FillMargin determines is whether to fill the margin with\nthe surrounding background color before rendering the element itself.\nThis is typically necessary to prevent text, border, and box shadow from\nrendering over themselves. Therefore, it should be kept at its default value\nof true in most circumstances, but it can be set to false when the element\nis fully managed by something that is guaranteed to render the\nappropriate background color for the element. It is irrelevant if RenderBox\nis false."}, {Name: "Overflow", Doc: "Overflow determines how to handle overflowing content in a layout.\nDefault is OverflowVisible. Set to OverflowAuto to enable scrollbars."}, {Name: "Gap", Doc: "For layouts, extra space added between elements in the layout."}, {Name: "Columns", Doc: "For grid layouts, the number of columns to use.\nIf > 0, number of rows is computed as N elements / Columns.\nUsed as a constraint in layout if individual elements\ndo not specify their row, column positions"}, {Name: "ObjectFit", Doc: "If this object is a replaced object (image, video, etc)\nor has a background image, ObjectFit specifies the way\nin which the replaced object should be fit into the element."}, {Name: "ObjectPosition", Doc: "If this object is a replaced object (image, video, etc)\nor has a background image, ObjectPosition specifies the\nX,Y position of the object within the space allocated for\nthe object (see ObjectFit)."}, {Name: "Border", Doc: "Border is a rendered border around the element."}, {Name: "MaxBorder", Doc: "MaxBorder is the largest border that will ever be rendered\naround the element, the size of which is used for computing\nthe effective margin to allocate for the element."}, {Name: "BoxShadow", Doc: "BoxShadow is the box shadows to render around box (can have multiple)"}, {Name: "MaxBoxShadow", Doc: "MaxBoxShadow contains the largest shadows that will ever be rendered\naround the element, the size of which are used for computing the\neffective margin to allocate for the element."}, {Name: "Color", Doc: "Color specifies the text / content color, and it is inherited."}, {Name: "Background", Doc: "Background specifies the background of the element. It is not inherited,\nand it is nil (transparent) by default."}, {Name: "Opacity", Doc: "alpha value between 0 and 1 to apply to the foreground and background of this element and all of its children"}, {Name: "StateLayer", Doc: "StateLayer, if above zero, indicates to create a state layer over the element with this much opacity (on a scale of 0-1) and the\ncolor Color (or StateColor if it defined). It is automatically set based on State, but can be overridden in stylers."}, {Name: "StateColor", Doc: "StateColor, if not nil, is the color to use for the StateLayer instead of Color. If you want to disable state layers\nfor an element, do not use this; instead, set StateLayer to 0."}, {Name: "ActualBackground", Doc: "ActualBackground is the computed actual background rendered for the element,\ntaking into account its Background, Opacity, StateLayer, and parent\nActualBackground. It is automatically computed and should not be set manually."}, {Name: "VirtualKeyboard", Doc: "VirtualKeyboard is the virtual keyboard to display, if any,\non mobile platforms when this element is focused. It is not\nused if the element is read only."}, {Name: "Pos", Doc: "Pos is used for the position of the widget if the parent frame\nhas [Style.Display] = [NoLayout]."}, {Name: "ZIndex", Doc: "ordering factor for rendering depth -- lower numbers rendered first.\nSort children according to this factor"}, {Name: "Row", Doc: "specifies the row that this element should appear within a grid layout"}, {Name: "Col", Doc: "specifies the column that this element should appear within a grid layout"}, {Name: "RowSpan", Doc: "specifies the number of sequential rows that this element should occupy\nwithin a grid layout (todo: not currently supported)"}, {Name: "ColSpan", Doc: "specifies the number of sequential columns that this element should occupy\nwithin a grid layout"}, {Name: "ScrollbarWidth", Doc: "ScrollbarWidth is the width of layout scrollbars. It defaults\nto [DefaultScrollbarWidth], and it is inherited."}, {Name: "Font", Doc: "font styling parameters"}, {Name: "Text", Doc: "text styling parameters"}, {Name: "UnitContext", Doc: "unit context: parameters necessary for anchoring relative units"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/styles.Text", IDName: "text", Doc: "Text is used for layout-level (widget, html-style) text styling --\nFontStyle contains all the lower-level text rendering info used in SVG --\nmost of these are inherited", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Align", Doc: "how to align text, horizontally (inherited).\nThis *only* applies to the text within its containing element,\nand is typically relevant only for multi-line text:\nfor single-line text, if element does not have a specified size\nthat is different from the text size, then this has *no effect*."}, {Name: "AlignV", Doc: "vertical alignment of text (inherited).\nThis *only* applies to the text within its containing element:\nif that element does not have a specified size\nthat is different from the text size, then this has *no effect*."}, {Name: "Anchor", Doc: "for svg rendering only (inherited):\ndetermines the alignment relative to text position coordinate.\nFor RTL start is right, not left, and start is top for TB"}, {Name: "LetterSpacing", Doc: "spacing between characters and lines"}, {Name: "WordSpacing", Doc: "extra space to add between words (inherited)"}, {Name: "LineHeight", Doc: "specified height of a line of text (inherited); text is centered within the overall lineheight;\nthe standard way to specify line height is in terms of em"}, {Name: "WhiteSpace", Doc: "WhiteSpace (not inherited) specifies how white space is processed,\nand how lines are wrapped. If set to WhiteSpaceNormal (default) lines are wrapped.\nSee info about interactions with Grow.X setting for this and the NoWrap case."}, {Name: "UnicodeBidi", Doc: "determines how to treat unicode bidirectional information (inherited)"}, {Name: "Direction", Doc: "bidi-override or embed -- applies to all text elements (inherited)"}, {Name: "WritingMode", Doc: "overall writing mode -- only for text elements, not span (inherited)"}, {Name: "OrientationVert", Doc: "for TBRL writing mode (only), determines orientation of alphabetic characters (inherited);\n90 is default (rotated); 0 means keep upright"}, {Name: "OrientationHoriz", Doc: "for horizontal LR/RL writing mode (only), determines orientation of all characters (inherited);\n0 is default (upright)"}, {Name: "Indent", Doc: "how much to indent the first line in a paragraph (inherited)"}, {Name: "ParaSpacing", Doc: "extra spacing between paragraphs (inherited); copied from [Style.Margin] per CSS spec\nif that is non-zero, else can be set directly with para-spacing"}, {Name: "TabSize", Doc: "tab size, in number of characters (inherited)"}}}) diff --git a/tensor/table/typegen.go b/tensor/table/typegen.go index fa644e2e9e..9754001a97 100644 --- a/tensor/table/typegen.go +++ b/tensor/table/typegen.go @@ -6,4 +6,4 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of Tensor columns aligned by a common outermost row dimension.\nUse the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a\n[tensor.Indexed] view of the column, using the shared [Table.Indexes] of the Table.\nThus, a coordinated sorting and filtered view of the column data is automatically\navailable for any of the tensor package functions that use [tensor.Indexed] as the one\ncommon data representation for all operations.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumns", Doc: "SortColumns sorts the indexes into our Table according to values in\ngiven column names, using either ascending or descending order,\n(use [tensor.Ascending] or [tensor.Descending] for self-documentation,\nand optionally using a stable sort.\nOnly valid for 1-dimensional columns.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"ascending", "stable", "columns"}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values in column compared to given\nstring. Includes rows with matching values unless exclude is set.\nIf contains, only checks if row contains string; if ignoreCase, ignores case.\nUse the named const args [tensor.Include], [tensor.Exclude], [tensor.Contains],\n[tensor.Equals], [tensor.IgnoreCase], [tensor.UseCase] for greater clarity.\nOnly valid for 1-dimensional columns (uses first cell from higher dimensions).\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"columnName", "str", "exclude", "contains", "ignoreCase"}, Returns: []string{"error"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}, Returns: []string{"Table"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns.\nIf rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0.\nIf indexes are in place and rows are added, indexes for the new rows are added.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "Columns has the list of column tensor data for this table.\nDifferent tables can provide different indexed views onto the same Columns."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nThese indexes are shared into the `tensor.Indexed` Column values\nto provide a coordinated indexed view into the underlying data."}, {Name: "Meta", Doc: "Meta is misc metadata for the table. Use lower-case key names\nfollowing the struct tag convention:\n\t- name string = name of table\n\t- doc string = documentation, description\n\t- read-only bool = gui is read-only\n\t- precision int = n for precision to write out floats in csv."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/table.Table", IDName: "table", Doc: "Table is a table of Tensor columns aligned by a common outermost row dimension.\nUse the [Table.Column] (by name) and [Table.ColumnIndex] methods to obtain a\n[tensor.Rows] view of the column, using the shared [Table.Indexes] of the Table.\nThus, a coordinated sorting and filtered view of the column data is automatically\navailable for any of the tensor package functions that use [tensor.Tensor] as the one\ncommon data representation for all operations.\nTensor Columns are always raw value types and support SubSpace operations on cells.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Sequential", Doc: "Sequential sets Indexes to nil, resulting in sequential row-wise access into tensor.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "SortColumn", Doc: "SortColumn sorts the indexes into our Table according to values in\ngiven column, using either ascending or descending order,\n(use [tensor.Ascending] or [tensor.Descending] for self-documentation).\nUses first cell of higher dimensional data.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"columnName", "ascending"}, Returns: []string{"error"}}, {Name: "SortColumns", Doc: "SortColumns sorts the indexes into our Table according to values in\ngiven column names, using either ascending or descending order,\n(use [tensor.Ascending] or [tensor.Descending] for self-documentation,\nand optionally using a stable sort.\nUses first cell of higher dimensional data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"ascending", "stable", "columns"}}, {Name: "FilterString", Doc: "FilterString filters the indexes using string values in column compared to given\nstring. Includes rows with matching values unless the Exclude option is set.\nIf Contains option is set, it only checks if row contains string;\nif IgnoreCase, ignores case, otherwise filtering is case sensitive.\nUses first cell from higher dimensions.\nReturns error if column name not found.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"columnName", "str", "opts"}, Returns: []string{"error"}}, {Name: "SaveCSV", Doc: "SaveCSV writes a table to a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg).\nIf headers = true then generate column headers that capture the type\nand tensor cell geometry of the columns, enabling full reloading\nof exactly the same table format and data (recommended).\nOtherwise, only the data is written.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim", "headers"}, Returns: []string{"error"}}, {Name: "OpenCSV", Doc: "OpenCSV reads a table from a comma-separated-values (CSV) file\n(where comma = any delimiter, specified in the delim arg),\nusing the Go standard encoding/csv reader conforming to the official CSV standard.\nIf the table does not currently have any columns, the first row of the file\nis assumed to be headers, and columns are constructed therefrom.\nIf the file was saved from table with headers, then these have full configuration\ninformation for tensor type and dimensionality.\nIf the table DOES have existing columns, then those are used robustly\nfor whatever information fits from each row of the file.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}, Returns: []string{"error"}}, {Name: "AddRows", Doc: "AddRows adds n rows to end of underlying Table, and to the indexes in this view.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"n"}, Returns: []string{"Table"}}, {Name: "SetNumRows", Doc: "SetNumRows sets the number of rows in the table, across all columns.\nIf rows = 0 then effective number of rows in tensors is 1, as this dim cannot be 0.\nIf indexes are in place and rows are added, indexes for the new rows are added.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"rows"}, Returns: []string{"Table"}}}, Fields: []types.Field{{Name: "Columns", Doc: "Columns has the list of column tensor data for this table.\nDifferent tables can provide different indexed views onto the same Columns."}, {Name: "Indexes", Doc: "Indexes are the indexes into Tensor rows, with nil = sequential.\nOnly set if order is different from default sequential order.\nThese indexes are shared into the `tensor.Rows` Column values\nto provide a coordinated indexed view into the underlying data."}, {Name: "Meta", Doc: "Meta is misc metadata for the table. Use lower-case key names\nfollowing the struct tag convention:\n\t- name string = name of table\n\t- doc string = documentation, description\n\t- read-only bool = gui is read-only\n\t- precision int = n for precision to write out floats in csv."}}}) diff --git a/texteditor/highlighting/typegen.go b/texteditor/highlighting/typegen.go index 2566f3a9a0..ac1180c6c2 100644 --- a/texteditor/highlighting/typegen.go +++ b/texteditor/highlighting/typegen.go @@ -10,7 +10,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor/highligh var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor/highlighting.Trilean", IDName: "trilean", Doc: "Trilean value for StyleEntry value inheritance."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor/highlighting.StyleEntry", IDName: "style-entry", Doc: "StyleEntry is one value in the map of highlight style values", Fields: []types.Field{{Name: "Color", Doc: "text color"}, {Name: "Background", Doc: "background color"}, {Name: "Border", Doc: "border color? not sure what this is -- not really used"}, {Name: "Bold", Doc: "bold font"}, {Name: "Italic", Doc: "italic font"}, {Name: "Underline", Doc: "underline"}, {Name: "NoInherit", Doc: "don't inherit these settings from sub-category or category levels -- otherwise everything with a Pass is inherited"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor/highlighting.StyleEntry", IDName: "style-entry", Doc: "StyleEntry is one value in the map of highlight style values", Fields: []types.Field{{Name: "Color", Doc: "Color is the text color."}, {Name: "Background", Doc: "Background color.\nIn general it is not good to use this because it obscures highlighting."}, {Name: "Border", Doc: "Border color? not sure what this is -- not really used."}, {Name: "Bold", Doc: "Bold font."}, {Name: "Italic", Doc: "Italic font."}, {Name: "Underline", Doc: "Underline."}, {Name: "NoInherit", Doc: "NoInherit indicates to not inherit these settings from sub-category or category levels.\nOtherwise everything with a Pass is inherited."}, {Name: "themeColor", Doc: "themeColor is the theme-adjusted text color."}, {Name: "themeBackground", Doc: "themeBackground is the theme-adjusted background color."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor/highlighting.Style", IDName: "style", Doc: "Style is a full style map of styles for different token.Tokens tag values"}) diff --git a/texteditor/typegen.go b/texteditor/typegen.go index 22d09b6999..d603903f29 100644 --- a/texteditor/typegen.go +++ b/texteditor/typegen.go @@ -15,7 +15,7 @@ import ( var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor.Buffer", IDName: "buffer", Doc: "Buffer is a buffer of text, which can be viewed by [Editor](s).\nIt holds the raw text lines (in original string and rune formats,\nand marked-up from syntax highlighting), and sends signals for making\nedits to the text and coordinating those edits across multiple views.\nEditors always only view a single buffer, so they directly call methods\non the buffer to drive updates, which are then broadcast.\nIt also has methods for loading and saving buffers to files.\nUnlike GUI widgets, its methods generally send events, without an\nexplicit Event suffix.\nInternally, the buffer represents new lines using \\n = LF, but saving\nand loading can deal with Windows/DOS CRLF format.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "Open", Doc: "Open loads the given file into the buffer.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename"}, Returns: []string{"error"}}, {Name: "Revert", Doc: "Revert re-opens text from the current file,\nif the filename is set; returns false if not.\nIt uses an optimized diff-based update to preserve\nexisting formatting, making it very fast if not very different.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Returns: []string{"bool"}}, {Name: "SaveAs", Doc: "SaveAs saves the current text into given file; does an editDone first to save edits\nand checks for an existing file; if it does exist then prompts to overwrite or not.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename"}}, {Name: "Save", Doc: "Save saves the current text into the current filename associated with this buffer.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Returns: []string{"error"}}}, Embeds: []types.Field{{Name: "Lines"}}, Fields: []types.Field{{Name: "Filename", Doc: "Filename is the filename of the file that was last loaded or saved.\nIt is used when highlighting code."}, {Name: "Autosave", Doc: "Autosave specifies whether the file should be automatically\nsaved after changes are made."}, {Name: "Info", Doc: "Info is the full information about the current file."}, {Name: "LineColors", Doc: "LineColors are the colors to use for rendering circles\nnext to the line numbers of certain lines."}, {Name: "editors", Doc: "editors are the editors that are currently viewing this buffer."}, {Name: "posHistory", Doc: "posHistory is the history of cursor positions.\nIt can be used to move back through them."}, {Name: "Complete", Doc: "Complete is the functions and data for text completion."}, {Name: "spell", Doc: "spell is the functions and data for spelling correction."}, {Name: "currentEditor", Doc: "currentEditor is the current text editor, such as the one that initiated the\nComplete or Correct process. The cursor position in this view is updated, and\nit is reset to nil after usage."}, {Name: "listeners", Doc: "listeners is used for sending standard system events.\nChange is sent for BufferDone, BufferInsert, and BufferDelete."}, {Name: "autoSaving", Doc: "autoSaving is used in atomically safe way to protect autosaving"}, {Name: "notSaved", Doc: "notSaved indicates if the text has been changed (edited) relative to the\noriginal, since last Save. This can be true even when changed flag is\nfalse, because changed is cleared on EditDone, e.g., when texteditor\nis being monitored for OnChange and user does Control+Enter.\nUse IsNotSaved() method to query state."}, {Name: "fileModOK", Doc: "fileModOK have already asked about fact that file has changed since being\nopened, user is ok"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor.DiffEditor", IDName: "diff-editor", Doc: "DiffEditor presents two side-by-side [Editor]s showing the differences\nbetween two files (represented as lines of strings).", Methods: []types.Method{{Name: "saveFileA", Doc: "saveFileA saves the current state of file A to given filename", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "saveFileB", Doc: "saveFileB saves the current state of file B to given filename", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FileA", Doc: "first file name being compared"}, {Name: "FileB", Doc: "second file name being compared"}, {Name: "RevisionA", Doc: "revision for first file, if relevant"}, {Name: "RevisionB", Doc: "revision for second file, if relevant"}, {Name: "bufferA", Doc: "[Buffer] for A showing the aligned edit view"}, {Name: "bufferB", Doc: "[Buffer] for B showing the aligned edit view"}, {Name: "alignD", Doc: "aligned diffs records diff for aligned lines"}, {Name: "diffs", Doc: "diffs applied"}, {Name: "inInputEvent"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/texteditor.DiffEditor", IDName: "diff-editor", Doc: "DiffEditor presents two side-by-side [Editor]s showing the differences\nbetween two files (represented as lines of strings).", Methods: []types.Method{{Name: "saveFileA", Doc: "saveFileA saves the current state of file A to given filename", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "saveFileB", Doc: "saveFileB saves the current state of file B to given filename", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FileA", Doc: "first file name being compared"}, {Name: "FileB", Doc: "second file name being compared"}, {Name: "RevisionA", Doc: "revision for first file, if relevant"}, {Name: "RevisionB", Doc: "revision for second file, if relevant"}, {Name: "bufferA", Doc: "[Buffer] for A showing the aligned edit view"}, {Name: "bufferB", Doc: "[Buffer] for B showing the aligned edit view"}, {Name: "alignD", Doc: "aligned diffs records diff for aligned lines"}, {Name: "diffs", Doc: "diffs applied"}, {Name: "inInputEvent"}, {Name: "toolbar"}}}) // NewDiffEditor returns a new [DiffEditor] with the given optional parent: // DiffEditor presents two side-by-side [Editor]s showing the differences diff --git a/types/typegen/generator.go b/types/typegen/generator.go index 4c7e371adf..733c95809b 100644 --- a/types/typegen/generator.go +++ b/types/typegen/generator.go @@ -80,7 +80,7 @@ func (g *Generator) Find() error { return err } g.Types = []*Type{} - err = generate.Inspect(g.Pkg, g.Inspect) + err = generate.Inspect(g.Pkg, g.Inspect, "enumgen.go", "typegen.go") if err != nil { return fmt.Errorf("error while inspecting: %w", err) } diff --git a/types/typegen/testdata/typegen.go b/types/typegen/testdata/typegen.go index e14a692b4a..14ab9d7984 100644 --- a/types/typegen/testdata/typegen.go +++ b/types/typegen/testdata/typegen.go @@ -1,4 +1,4 @@ -// Code generated by "typegen.test -test.testlogfile=/var/folders/x1/r8shprmj7j71zbw3qvgl9dqc0000gq/T/go-build1497819794/b982/testlog.txt -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. +// Code generated by "typegen.test -test.paniconexit0 -test.timeout=10m0s -test.v=true"; DO NOT EDIT. package testdata diff --git a/types/typegen/typegen_gen.go b/types/typegen/typegen_gen.go index 1e087d2d29..41753e4872 100644 --- a/types/typegen/typegen_gen.go +++ b/types/typegen/typegen_gen.go @@ -7,5 +7,3 @@ import ( ) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/types/typegen.Config", IDName: "config", Doc: "Config contains the configuration information\nused by typegen", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Dir", Doc: "the source directory to run typegen on (can be set to multiple through paths like ./...)"}, {Name: "Output", Doc: "the output file location relative to the package on which typegen is being called"}, {Name: "AddTypes", Doc: "whether to add types to typegen by default"}, {Name: "AddMethods", Doc: "whether to add methods to typegen by default"}, {Name: "AddFuncs", Doc: "whether to add functions to typegen by default"}, {Name: "InterfaceConfigs", Doc: "An ordered map of configs keyed by fully qualified interface type names; if a type implements the interface, the config will be applied to it.\nThe configs are applied in sequential ascending order, which means that\nthe last config overrides the other ones, so the most specific\ninterfaces should typically be put last.\nNote: the package typegen is run on must explicitly reference this interface at some point for this to work; adding a simple\n`var _ MyInterface = (*MyType)(nil)` statement to check for interface implementation is an easy way to accomplish that.\nNote: typegen will still succeed if it can not find one of the interfaces specified here in order to allow it to work generically across multiple directories; you can use the -v flag to get log warnings about this if you suspect that it is not finding interfaces when it should."}, {Name: "Setters", Doc: "Whether to generate chaining `Set*` methods for each exported field of each type (eg: \"SetText\" for field \"Text\").\nIf this is set to true, then you can add `set:\"-\"` struct tags to individual fields\nto prevent Set methods being generated for them."}, {Name: "Templates", Doc: "a slice of templates to execute on each type being added; the template data is of the type typegen.Type"}}}) - -var _ = types.AddFunc(&types.Func{Name: "cogentcore.org/core/types/typegen.Generate", Doc: "Generate generates typegen type info, using the\nconfiguration information, loading the packages from the\nconfiguration source directory, and writing the result\nto the configuration output file.\n\nIt is a simple entry point to typegen that does all\nof the steps; for more specific functionality, create\na new [Generator] with [NewGenerator] and call methods on it.", Directives: []types.Directive{{Tool: "cli", Directive: "cmd", Args: []string{"-root"}}, {Tool: "types", Directive: "add"}}, Args: []string{"cfg"}, Returns: []string{"error"}}) diff --git a/xyz/typegen.go b/xyz/typegen.go index 801ba84539..66e8ac870c 100644 --- a/xyz/typegen.go +++ b/xyz/typegen.go @@ -62,8 +62,6 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.LightColors", I var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.Lines", IDName: "lines", Doc: "Lines are lines rendered as long thin boxes defined by points\nand width parameters. The Mesh must be drawn in the XY plane (i.e., use Z = 0\nor a constant unless specifically relevant to have full 3D variation).\nRotate the solid to put into other planes.", Embeds: []types.Field{{Name: "MeshBase"}}, Fields: []types.Field{{Name: "Points", Doc: "line points (must be 2 or more)"}, {Name: "Width", Doc: "line width, Y = height perpendicular to line direction, and X = depth"}, {Name: "Colors", Doc: "optional colors for each point -- actual color interpolates between"}, {Name: "Closed", Doc: "if true, connect the first and last points to form a closed shape"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.Line", IDName: "line", Doc: "Line is a Solid that is used for line elements.\nType is need to trigger more precise event handling.", Directives: []types.Directive{{Tool: "core", Directive: "no-new"}}, Embeds: []types.Field{{Name: "Solid"}}}) - var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.Material", IDName: "material", Doc: "Material describes the material properties of a surface (colors, shininess, texture)\ni.e., phong lighting parameters.\nMain color is used for both ambient and diffuse color, and alpha component\nis used for opacity. The Emissive color is only for glowing objects.\nThe Specular color is always white (multiplied by light color).\nTextures are stored on the Scene and accessed by name", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Color", Doc: "Color is the main color of surface, used for both ambient and diffuse color in standard Phong model -- alpha component determines transparency -- note that transparent objects require more complex rendering"}, {Name: "Emissive", Doc: "Emissive is the color that surface emits independent of any lighting -- i.e., glow -- can be used for marking lights with an object"}, {Name: "Shiny", Doc: "Shiny is the specular shininess factor -- how focally vs. broad the surface shines back directional light -- this is an exponential factor, with 0 = very broad diffuse reflection, and higher values (typically max of 128 or so but can go higher) having a smaller more focal specular reflection. Also set Reflective factor to change overall shininess effect."}, {Name: "Reflective", Doc: "Reflective is the specular reflectiveness factor -- how much it shines back directional light. The specular reflection color is always white * the incoming light."}, {Name: "Bright", Doc: "Bright is an overall multiplier on final computed color value -- can be used to tune the overall brightness of various surfaces relative to each other for a given set of lighting parameters"}, {Name: "TextureName", Doc: "TextureName is the name of the texture to provide color for the surface."}, {Name: "Tiling", Doc: "Tiling is the texture tiling parameters: repeat and offset."}, {Name: "CullBack", Doc: "CullBack indicates to cull the back-facing surfaces."}, {Name: "CullFront", Doc: "CullFront indicates to cull the front-facing surfaces."}, {Name: "Texture", Doc: "Texture is the cached [Texture] object set based on [Material.TextureName]."}}}) // SetColor sets the [Material.Color]: @@ -124,7 +122,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.GenMesh", IDNam var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.Node", IDName: "node", Doc: "Node is the common interface for all xyz 3D tree nodes.\n[Solid] and [Group] are the two main types of nodes,\nwhich both extend [NodeBase] for the core functionality.", Methods: []types.Method{{Name: "AsNodeBase", Doc: "AsNodeBase returns the [NodeBase] for our node, which gives\naccess to all the base-level data structures and methods\nwithout requiring interface methods.", Returns: []string{"NodeBase"}}, {Name: "IsSolid", Doc: "IsSolid returns true if this is an [Solid] node (otherwise a [Group]).", Returns: []string{"bool"}}, {Name: "AsSolid", Doc: "AsSolid returns the node as a [Solid] (nil if not).", Returns: []string{"Solid"}}, {Name: "Validate", Doc: "Validate checks that scene element is valid.", Returns: []string{"error"}}, {Name: "UpdateWorldMatrix", Doc: "UpdateWorldMatrix updates this node's local and world matrix based on parent's world matrix.", Args: []string{"parWorld"}}, {Name: "UpdateMeshBBox", Doc: "UpdateMeshBBox updates the Mesh-based BBox info for all nodes.\ngroups aggregate over elements. It is called from WalkPost traversal."}, {Name: "IsVisible", Doc: "IsVisible provides the definitive answer as to whether a given node\nis currently visible. It is only entirely valid after a render pass\nfor widgets in a visible window, but it checks the window and viewport\nfor their visibility status as well, which is available always.\nNon-visible nodes are automatically not rendered and not connected to\nwindow events. The Invisible flag is one key element of the IsVisible\ncalculus; it is set by e.g., TabView for invisible tabs, and is also\nset if a widget is entirely out of render range. But again, use\nIsVisible as the main end-user method.\nFor robustness, it recursively calls the parent; this is typically\na short path; propagating the Invisible flag properly can be\nvery challenging without mistakenly overwriting invisibility at various\nlevels.", Returns: []string{"bool"}}, {Name: "IsTransparent", Doc: "IsTransparent returns true if solid has transparent color.", Returns: []string{"bool"}}, {Name: "Config", Doc: "Config configures the node."}, {Name: "RenderClass", Doc: "RenderClass returns the class of rendering for this solid.\nIt is used for organizing the ordering of rendering.", Returns: []string{"RenderClasses"}}, {Name: "PreRender", Doc: "PreRender is called by Scene Render to upload\nall the object data to the Phong renderer."}, {Name: "Render", Doc: "Render is called by Scene Render to actually render.", Args: []string{"rp"}}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.NodeBase", IDName: "node-base", Doc: "NodeBase is the basic 3D tree node, which has the full transform information\nrelative to parent, and computed bounding boxes, etc.\nIt implements the [Node] interface and contains the core functionality\ncommon to all 3D nodes.", Embeds: []types.Field{{Name: "NodeBase"}}, Fields: []types.Field{{Name: "Invisible", Doc: "Invisible is whether this node is invisible."}, {Name: "Pose", Doc: "Pose is the complete specification of position and orientation."}, {Name: "Scene", Doc: "Scene is the cached [Scene]."}, {Name: "MeshBBox", Doc: "mesh-based local bounding box (aggregated for groups)"}, {Name: "WorldBBox", Doc: "world coordinates bounding box"}, {Name: "NDCBBox", Doc: "normalized display coordinates bounding box, used for frustrum clipping"}, {Name: "BBox", Doc: "raw original bounding box for the widget within its parent Scene.\nThis is prior to intersecting with Frame bounds."}, {Name: "SceneBBox", Doc: "2D bounding box for region occupied within Scene Frame that we render onto.\nThis is BBox intersected with Frame bounds."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/xyz.NodeBase", IDName: "node-base", Doc: "NodeBase is the basic 3D tree node, which has the full transform information\nrelative to parent, and computed bounding boxes, etc.\nIt implements the [Node] interface and contains the core functionality\ncommon to all 3D nodes.", Embeds: []types.Field{{Name: "NodeBase"}}, Fields: []types.Field{{Name: "Invisible", Doc: "Invisible is whether this node is invisible."}, {Name: "Pose", Doc: "Pose is the complete specification of position and orientation."}, {Name: "Scene", Doc: "Scene is the cached [Scene]."}, {Name: "MeshBBox", Doc: "mesh-based local bounding box (aggregated for groups)"}, {Name: "WorldBBox", Doc: "world coordinates bounding box"}, {Name: "NDCBBox", Doc: "normalized display coordinates bounding box, used for frustrum clipping"}, {Name: "BBox", Doc: "raw original bounding box for the widget within its parent Scene.\nThis is prior to intersecting with Frame bounds."}, {Name: "SceneBBox", Doc: "2D bounding box for region occupied within Scene Frame that we render onto.\nThis is BBox intersected with Frame bounds."}, {Name: "isLinear", Doc: "isLinear indicates that this element contains a line-like shape,\nwhich engages a more selective event processing logic to determine\nif the node was selected based on a mouse click point."}}}) // NewNodeBase returns a new [NodeBase] with the given optional parent: // NodeBase is the basic 3D tree node, which has the full transform information From ecf906cc4e006ed303296f3a83c8abd41b4a4a48 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 24 Oct 2024 17:00:53 -0700 Subject: [PATCH 247/311] in process logic for commands / builtins to be able to do pipe output -- pipe logic looks good now; just need the args to work. --- base/exec/stdio.go | 11 ++--- goal/builtins.go | 27 +++++++++++- goal/exec.go | 73 +++++++++++++++++++++++++------- goal/goal.go | 17 ++++++-- goal/transpile/transpile_test.go | 2 +- 5 files changed, 103 insertions(+), 27 deletions(-) diff --git a/base/exec/stdio.go b/base/exec/stdio.go index f871bd6284..57a4d9e0a8 100644 --- a/base/exec/stdio.go +++ b/base/exec/stdio.go @@ -56,9 +56,13 @@ func (st *StdIO) Set(o *StdIO) *StdIO { func (st *StdIO) SetToOS() *StdIO { cur := &StdIO{} cur.SetFromOS() + if sif, ok := st.In.(*os.File); ok { + os.Stdin = sif + } else { + fmt.Printf("In is not an *os.File: %#v\n", st.In) + } os.Stdout = st.Out.(*os.File) os.Stderr = st.Err.(*os.File) - os.Stdin = st.In.(*os.File) return cur } @@ -98,13 +102,10 @@ func IsPipe(rw any) bool { if rw == nil { return false } - w, ok := rw.(io.Writer) + _, ok := rw.(io.Writer) if !ok { return false } - if w == os.Stdout { - return false - } of, ok := rw.(*os.File) if !ok { return false diff --git a/goal/builtins.go b/goal/builtins.go index 7960a547a2..d44946c833 100644 --- a/goal/builtins.go +++ b/goal/builtins.go @@ -16,6 +16,8 @@ import ( "cogentcore.org/core/base/exec" "cogentcore.org/core/base/logx" "cogentcore.org/core/base/sshclient" + "cogentcore.org/core/core" + "cogentcore.org/core/system" "github.com/mitchellh/go-homedir" ) @@ -27,6 +29,7 @@ func (gl *Goal) InstallBuiltins() { gl.Builtins["jobs"] = gl.JobsCmd gl.Builtins["kill"] = gl.Kill gl.Builtins["set"] = gl.Set + gl.Builtins["unset"] = gl.Unset gl.Builtins["add-path"] = gl.AddPath gl.Builtins["which"] = gl.Which gl.Builtins["source"] = gl.Source @@ -78,7 +81,23 @@ func (gl *Goal) Set(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 2 { return fmt.Errorf("expected two arguments, got %d", len(args)) } - return os.Setenv(args[0], args[1]) + err := os.Setenv(args[0], args[1]) + if core.TheApp.Platform() == system.MacOS { + gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", args[0], args[1]) + } + return err +} + +// Unset un-sets the given environment variable. +func (gl *Goal) Unset(cmdIO *exec.CmdIO, args ...string) error { + if len(args) != 1 { + return fmt.Errorf("expected one argument, got %d", len(args)) + } + err := os.Unsetenv(args[0]) + if core.TheApp.Platform() == system.MacOS { + gl.Config.RunIO(cmdIO, "/bin/launchctl", "unsetenv", args[0]) + } + return err } // JobsCmd is the builtin jobs command @@ -135,7 +154,11 @@ func (gl *Goal) AddPath(cmdIO *exec.CmdIO, args ...string) error { } path = path + ":" + arg } - return os.Setenv("PATH", path) + err := os.Setenv("PATH", path) + if core.TheApp.Platform() == system.MacOS { + gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", "PATH", path) + } + return err } // Which reports the executable associated with the given command. diff --git a/goal/exec.go b/goal/exec.go index 0b746cc225..ce63cf1e8f 100644 --- a/goal/exec.go +++ b/goal/exec.go @@ -11,6 +11,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "cogentcore.org/core/base/exec" "cogentcore.org/core/base/reflectx" @@ -56,25 +57,28 @@ func (gl *Goal) Exec(errOk, start, output bool, cmd any, args ...any) string { } } else { ran := false - ran, out = gl.RunBuiltinOrCommand(cmdIO, errOk, output, scmd, sargs...) + ran, out = gl.RunBuiltinOrCommand(cmdIO, errOk, start, output, scmd, sargs...) if !ran { gl.isCommand.Push(false) switch { case start: + fmt.Fprintf(gl.debugTrace, "start exe %s in: %#v out: %#v %v\n ", scmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) err = gl.Config.StartIO(cmdIO, scmd, sargs...) - gl.Jobs.Push(cmdIO) + job := &Job{CmdIO: cmdIO} + gl.Jobs.Push(job) go func() { if !cmdIO.OutIsPipe() { fmt.Printf("[%d] %s\n", len(gl.Jobs), cmdIO.String()) } cmdIO.Cmd.Wait() cmdIO.PopToStart() - gl.DeleteJob(cmdIO) + gl.DeleteJob(job) }() case output: cmdIO.PushOut(nil) out, err = gl.Config.OutputIO(cmdIO, scmd, sargs...) default: + fmt.Fprintf(gl.debugTrace, "run exe %s in: %#v out: %#v %v\n ", scmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) err = gl.Config.RunIO(cmdIO, scmd, sargs...) } if !errOk { @@ -91,7 +95,7 @@ func (gl *Goal) Exec(errOk, start, output bool, cmd any, args ...any) string { // RunBuiltinOrCommand runs a builtin or a command, returning true if it ran, // and the output string if running in output mode. -func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, output bool, cmd string, args ...string) (bool, string) { +func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, start, output bool, cmd string, args ...string) (bool, string) { out := "" cmdFun, hasCmd := gl.Commands[cmd] bltFun, hasBlt := gl.Builtins[cmd] @@ -105,12 +109,50 @@ func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, output bool, cmd s gl.isCommand.Push(true) } + fmt.Fprintf(gl.debugTrace, "cmd %s push: %#v out: %#v %v\n", cmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) + // note: we need to set both os. and wrapper versions, so it works the same // in compiled vs. interpreted mode oldsh := gl.Config.StdIO.Set(&cmdIO.StdIO) oldwrap := gl.StdIOWrappers.SetWrappers(&cmdIO.StdIO) oldstd := cmdIO.SetToOS() - if output { + fmt.Fprintf(gl.debugTrace, "%s oldstd in: %#v out: %#v\n", cmd, oldstd.In, oldstd.Out) + + done := func() { + if hasCmd { + gl.isCommand.Pop() + gl.commandArgs.Pop() + } + fmt.Fprintf(gl.debugTrace, "%s restore %#v\n", cmd, oldstd.In) + oldstd.SetToOS() + gl.StdIOWrappers.SetWrappers(oldwrap) + gl.Config.StdIO = *oldsh + } + + switch { + case start: + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + if !cmdIO.OutIsPipe() { + fmt.Printf("[%d] %s\n", len(gl.Jobs), cmd) + } + if hasCmd { + cmdFun(args...) + } else { + gl.AddError(bltFun(cmdIO, args...)) + } + wg.Done() + }() + fmt.Fprintf(gl.debugTrace, "%s push: %#v out: %#v %v\n", cmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) + job := &Job{CmdIO: cmdIO} + gl.Jobs.Push(job) + go func() { + wg.Wait() + gl.DeleteJob(job) + done() + }() + case output: obuf := &bytes.Buffer{} // os.Stdout = obuf // needs a file gl.Config.StdIO.Out = obuf @@ -122,22 +164,15 @@ func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, output bool, cmd s gl.AddError(bltFun(cmdIO, args...)) } out = strings.TrimSuffix(obuf.String(), "\n") - } else { + done() + default: if hasCmd { cmdFun(args...) } else { gl.AddError(bltFun(cmdIO, args...)) } + done() } - - if hasCmd { - gl.isCommand.Pop() - gl.commandArgs.Pop() - } - oldstd.SetToOS() - gl.StdIOWrappers.SetWrappers(oldwrap) - gl.Config.StdIO = *oldsh - return true, out } @@ -159,8 +194,13 @@ func (gl *Goal) HandleArgErr(errok bool, err error) error { func (gl *Goal) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) (*sshclient.Client, string, []string) { if len(gl.Jobs) > 0 { jb := gl.Jobs.Peek() - if jb.OutIsPipe() { + fmt.Fprintln(gl.debugTrace, "job:", len(gl.Jobs)-1) + if jb.OutIsPipe() && !jb.GotPipe { + jb.GotPipe = true + fmt.Fprintf(gl.debugTrace, "out: %#v in pipe: %#v\n", jb.Out, jb.PipeIn.Peek()) cmdIO.PushIn(jb.PipeIn.Peek()) + } else { + fmt.Fprintf(gl.debugTrace, "not pipe out: %#v\n", jb.Out) } } scmd := reflectx.ToString(cmd) @@ -301,6 +341,7 @@ func (gl *Goal) OutToPipe(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, s // todo: what to do here? sargs = slices.Delete(sargs, i, i+1) cmdIO.PushOutPipe() + fmt.Fprintf(gl.debugTrace, "pushed pipe in: %#v out: %#v %v\n", cmdIO.PipeIn.Peek(), cmdIO.Out, cmdIO.OutIsPipe()) if errf { cmdIO.PushErr(cmdIO.Out) } diff --git a/goal/goal.go b/goal/goal.go index 7e57e493ad..322c0c382e 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -55,7 +55,7 @@ type Goal struct { // Jobs is a stack of commands running in the background // (via Start instead of Run) - Jobs stack.Stack[*exec.CmdIO] + Jobs stack.Stack[*Job] // Cancel, while the interpreter is running, can be called // to stop the code interpreting. @@ -90,6 +90,9 @@ type Goal struct { // isCommand is a stack of bools indicating whether the _immediate_ run context // is a command, which affects the way that args are processed. isCommand stack.Stack[bool] + + // debugTrace is a file written to for debugging + debugTrace *os.File } // NewGoal returns a new [Goal] with default options. @@ -107,6 +110,7 @@ func NewGoal() *Goal { gl.SSHClients = make(map[string]*sshclient.Client) gl.Commands = make(map[string]func(args ...string)) gl.InstallBuiltins() + gl.debugTrace, _ = os.Create("goal.debug") return gl } @@ -319,8 +323,8 @@ func (gl *Goal) RunCommands(cmds []any) error { } // DeleteJob deletes the given job and returns true if successful, -func (gl *Goal) DeleteJob(cmdIO *exec.CmdIO) bool { - idx := slices.Index(gl.Jobs, cmdIO) +func (gl *Goal) DeleteJob(job *Job) bool { + idx := slices.Index(gl.Jobs, job) if idx >= 0 { gl.Jobs = slices.Delete(gl.Jobs, idx, idx+1) return true @@ -350,3 +354,10 @@ func (gl *Goal) JobIDExpand(args []string) int { } return exp } + +// Job represents a job that has been started and we're waiting for it to finish. +type Job struct { + *exec.CmdIO + IsExec bool + GotPipe bool +} diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 902c7ffebb..5fd2465533 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -218,7 +218,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"Layers[NeuronIxs[NrnLayIndex, ni]].GatherSpikes(&Ctx[0], ni, di)", `Layers[NeuronIxs.Value(int(NrnLayIndex), int(ni))].GatherSpikes( & Ctx[0], ni, di)`}, + {"ls | grep ev", `goal.Start("ls", "|"); goal.Run("grep", "ev")`}, } st := NewState() st.MathRecord = false From 215e3dfd8e4706aab9802b8cf3288d339046d7e7 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 25 Oct 2024 00:44:34 -0700 Subject: [PATCH 248/311] pipe working for builtin but not commands -- args fixed -- unclear what exactly is not working -- but not going to continue at this point. --- base/exec/stdio.go | 3 +++ goal/builtins.go | 9 ++++----- goal/exec.go | 44 ++++++++++++++++++++++++-------------------- goal/goal.go | 2 +- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/base/exec/stdio.go b/base/exec/stdio.go index 57a4d9e0a8..3c04bebaa4 100644 --- a/base/exec/stdio.go +++ b/base/exec/stdio.go @@ -248,6 +248,9 @@ func (st *StdIOState) PopToStart() { for len(st.InStack) > st.InStart { st.PopIn() } + for len(st.PipeIn) > 0 { + CloseReader(st.PipeIn.Pop()) + } } // ErrIsInOut returns true if the given Err writer is also present diff --git a/goal/builtins.go b/goal/builtins.go index d44946c833..f9fc8e2d92 100644 --- a/goal/builtins.go +++ b/goal/builtins.go @@ -10,14 +10,13 @@ import ( "log/slog" "os" "path/filepath" + "runtime" "strconv" "strings" "cogentcore.org/core/base/exec" "cogentcore.org/core/base/logx" "cogentcore.org/core/base/sshclient" - "cogentcore.org/core/core" - "cogentcore.org/core/system" "github.com/mitchellh/go-homedir" ) @@ -82,7 +81,7 @@ func (gl *Goal) Set(cmdIO *exec.CmdIO, args ...string) error { return fmt.Errorf("expected two arguments, got %d", len(args)) } err := os.Setenv(args[0], args[1]) - if core.TheApp.Platform() == system.MacOS { + if runtime.GOOS == "DARWIN" { gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", args[0], args[1]) } return err @@ -94,7 +93,7 @@ func (gl *Goal) Unset(cmdIO *exec.CmdIO, args ...string) error { return fmt.Errorf("expected one argument, got %d", len(args)) } err := os.Unsetenv(args[0]) - if core.TheApp.Platform() == system.MacOS { + if runtime.GOOS == "DARWIN" { gl.Config.RunIO(cmdIO, "/bin/launchctl", "unsetenv", args[0]) } return err @@ -155,7 +154,7 @@ func (gl *Goal) AddPath(cmdIO *exec.CmdIO, args ...string) error { path = path + ":" + arg } err := os.Setenv("PATH", path) - if core.TheApp.Platform() == system.MacOS { + if runtime.GOOS == "DARWIN" { gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", "PATH", path) } return err diff --git a/goal/exec.go b/goal/exec.go index ce63cf1e8f..57b768c334 100644 --- a/goal/exec.go +++ b/goal/exec.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "sync" + "time" "cogentcore.org/core/base/exec" "cogentcore.org/core/base/reflectx" @@ -62,7 +63,7 @@ func (gl *Goal) Exec(errOk, start, output bool, cmd any, args ...any) string { gl.isCommand.Push(false) switch { case start: - fmt.Fprintf(gl.debugTrace, "start exe %s in: %#v out: %#v %v\n ", scmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) + // fmt.Fprintf(gl.debugTrace, "start exe %s in: %#v out: %#v %v\n ", scmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) err = gl.Config.StartIO(cmdIO, scmd, sargs...) job := &Job{CmdIO: cmdIO} gl.Jobs.Push(job) @@ -78,7 +79,7 @@ func (gl *Goal) Exec(errOk, start, output bool, cmd any, args ...any) string { cmdIO.PushOut(nil) out, err = gl.Config.OutputIO(cmdIO, scmd, sargs...) default: - fmt.Fprintf(gl.debugTrace, "run exe %s in: %#v out: %#v %v\n ", scmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) + // fmt.Fprintf(gl.debugTrace, "run exe %s in: %#v out: %#v %v\n ", scmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) err = gl.Config.RunIO(cmdIO, scmd, sargs...) } if !errOk { @@ -109,21 +110,21 @@ func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, start, output bool gl.isCommand.Push(true) } - fmt.Fprintf(gl.debugTrace, "cmd %s push: %#v out: %#v %v\n", cmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) - // note: we need to set both os. and wrapper versions, so it works the same // in compiled vs. interpreted mode - oldsh := gl.Config.StdIO.Set(&cmdIO.StdIO) - oldwrap := gl.StdIOWrappers.SetWrappers(&cmdIO.StdIO) - oldstd := cmdIO.SetToOS() - fmt.Fprintf(gl.debugTrace, "%s oldstd in: %#v out: %#v\n", cmd, oldstd.In, oldstd.Out) + var oldsh, oldwrap, oldstd *exec.StdIO + save := func() { + oldsh = gl.Config.StdIO.Set(&cmdIO.StdIO) + oldwrap = gl.StdIOWrappers.SetWrappers(&cmdIO.StdIO) + oldstd = cmdIO.SetToOS() + } done := func() { if hasCmd { gl.isCommand.Pop() gl.commandArgs.Pop() } - fmt.Fprintf(gl.debugTrace, "%s restore %#v\n", cmd, oldstd.In) + // fmt.Fprintf(gl.debugTrace, "%s restore %#v\n", cmd, oldstd.In) oldstd.SetToOS() gl.StdIOWrappers.SetWrappers(oldwrap) gl.Config.StdIO = *oldsh @@ -138,21 +139,30 @@ func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, start, output bool fmt.Printf("[%d] %s\n", len(gl.Jobs), cmd) } if hasCmd { + oldwrap = gl.StdIOWrappers.SetWrappers(&cmdIO.StdIO) + // oldstd = cmdIO.SetToOS() + // fmt.Fprintf(gl.debugTrace, "%s oldstd in: %#v out: %#v\n", cmd, oldstd.In, oldstd.Out) cmdFun(args...) + // oldstd.SetToOS() + gl.StdIOWrappers.SetWrappers(oldwrap) + gl.isCommand.Pop() + gl.commandArgs.Pop() } else { gl.AddError(bltFun(cmdIO, args...)) } + time.Sleep(time.Millisecond) wg.Done() }() - fmt.Fprintf(gl.debugTrace, "%s push: %#v out: %#v %v\n", cmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) + // fmt.Fprintf(gl.debugTrace, "%s push: %#v out: %#v %v\n", cmd, cmdIO.In, cmdIO.Out, cmdIO.OutIsPipe()) job := &Job{CmdIO: cmdIO} gl.Jobs.Push(job) go func() { wg.Wait() + cmdIO.PopToStart() gl.DeleteJob(job) - done() }() case output: + save() obuf := &bytes.Buffer{} // os.Stdout = obuf // needs a file gl.Config.StdIO.Out = obuf @@ -166,6 +176,7 @@ func (gl *Goal) RunBuiltinOrCommand(cmdIO *exec.CmdIO, errOk, start, output bool out = strings.TrimSuffix(obuf.String(), "\n") done() default: + save() if hasCmd { cmdFun(args...) } else { @@ -194,18 +205,14 @@ func (gl *Goal) HandleArgErr(errok bool, err error) error { func (gl *Goal) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) (*sshclient.Client, string, []string) { if len(gl.Jobs) > 0 { jb := gl.Jobs.Peek() - fmt.Fprintln(gl.debugTrace, "job:", len(gl.Jobs)-1) if jb.OutIsPipe() && !jb.GotPipe { jb.GotPipe = true - fmt.Fprintf(gl.debugTrace, "out: %#v in pipe: %#v\n", jb.Out, jb.PipeIn.Peek()) cmdIO.PushIn(jb.PipeIn.Peek()) - } else { - fmt.Fprintf(gl.debugTrace, "not pipe out: %#v\n", jb.Out) } } scmd := reflectx.ToString(cmd) cl := gl.ActiveSSH() - isCmd := gl.isCommand.Peek() + // isCmd := gl.isCommand.Peek() sargs := make([]string, 0, len(args)) var err error for _, a := range args { @@ -252,7 +259,7 @@ func (gl *Goal) ExecArgs(cmdIO *exec.CmdIO, errOk bool, cmd any, args ...any) (* sargs = gl.OutToFile(cl, cmdIO, errOk, sargs, i) case s[0] == '|': sargs = gl.OutToPipe(cl, cmdIO, errOk, sargs, i) - case cl == nil && isCmd && strings.HasPrefix(s, "args"): + case cl == nil && strings.HasPrefix(s, "args"): sargs = gl.CmdArgs(errOk, sargs, i) i-- // back up because we consume this one } @@ -338,14 +345,11 @@ func (gl *Goal) OutToPipe(cl *sshclient.Client, cmdIO *exec.CmdIO, errOk bool, s if sn > 1 && s[1] == '&' { errf = true } - // todo: what to do here? sargs = slices.Delete(sargs, i, i+1) cmdIO.PushOutPipe() - fmt.Fprintf(gl.debugTrace, "pushed pipe in: %#v out: %#v %v\n", cmdIO.PipeIn.Peek(), cmdIO.Out, cmdIO.OutIsPipe()) if errf { cmdIO.PushErr(cmdIO.Out) } - // sh.HandleArgErr(errok, err) return sargs } diff --git a/goal/goal.go b/goal/goal.go index 322c0c382e..7c8f94a64c 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -110,7 +110,7 @@ func NewGoal() *Goal { gl.SSHClients = make(map[string]*sshclient.Client) gl.Commands = make(map[string]func(args ...string)) gl.InstallBuiltins() - gl.debugTrace, _ = os.Create("goal.debug") + // gl.debugTrace, _ = os.Create("goal.debug") // debugging return gl } From 7ceb537639f01c30f884e7ab292138d21ebaf04f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 25 Oct 2024 03:25:37 -0700 Subject: [PATCH 249/311] Set builtin automatically does home path expansion and dedupe, as does path (key for when it is set automatically in launchctl) --- base/stringsx/stringsx.go | 16 ++++++++++++ goal/builtins.go | 55 +++++++++++++++++++++++++++++---------- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/base/stringsx/stringsx.go b/base/stringsx/stringsx.go index 79dc0941ca..84450ffad8 100644 --- a/base/stringsx/stringsx.go +++ b/base/stringsx/stringsx.go @@ -8,6 +8,7 @@ package stringsx import ( "bytes" + "slices" "strings" ) @@ -88,3 +89,18 @@ func InsertFirstUnique(strs *[]string, str string, max int) { (*strs)[0] = str } } + +// DedupeList removes duplicates from given string list, +// preserving the order. +func DedupeList(strs []string) []string { + n := len(strs) + for i := n - 1; i >= 0; i-- { + p := strs[i] + for j, s := range strs { + if p == s && i != j { + strs = slices.Delete(strs, i, i+1) + } + } + } + return strs +} diff --git a/goal/builtins.go b/goal/builtins.go index f9fc8e2d92..6903aca5d4 100644 --- a/goal/builtins.go +++ b/goal/builtins.go @@ -14,9 +14,11 @@ import ( "strconv" "strings" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/exec" "cogentcore.org/core/base/logx" "cogentcore.org/core/base/sshclient" + "cogentcore.org/core/base/stringsx" "github.com/mitchellh/go-homedir" ) @@ -80,9 +82,15 @@ func (gl *Goal) Set(cmdIO *exec.CmdIO, args ...string) error { if len(args) != 2 { return fmt.Errorf("expected two arguments, got %d", len(args)) } - err := os.Setenv(args[0], args[1]) - if runtime.GOOS == "DARWIN" { - gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", args[0], args[1]) + val := args[1] + if strings.Count(val, ":") > 1 { + vl := stringsx.DedupeList(strings.Split(val, ":")) + vl = AddHomeExpand([]string{}, vl...) + val = strings.Join(vl, ":") + } + err := os.Setenv(args[0], val) + if runtime.GOOS == "darwin" { + gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", args[0], val) } return err } @@ -93,7 +101,7 @@ func (gl *Goal) Unset(cmdIO *exec.CmdIO, args ...string) error { return fmt.Errorf("expected one argument, got %d", len(args)) } err := os.Unsetenv(args[0]) - if runtime.GOOS == "DARWIN" { + if runtime.GOOS == "darwin" { gl.Config.RunIO(cmdIO, "/bin/launchctl", "unsetenv", args[0]) } return err @@ -140,23 +148,42 @@ func (gl *Goal) Fg(cmdIO *exec.CmdIO, args ...string) error { return nil } +// AddHomeExpand adds given strings to the given list of strings, +// expanding any ~ symbols with the home directory, +// and returns the updated list. +func AddHomeExpand(list []string, adds ...string) []string { + for _, add := range adds { + add, err := homedir.Expand(add) + errors.Log(err) + has := false + for _, s := range list { + if s == add { + has = true + } + } + if !has { + list = append(list, add) + } + } + return list +} + // AddPath adds the given path(s) to $PATH. func (gl *Goal) AddPath(cmdIO *exec.CmdIO, args ...string) error { if len(args) == 0 { return fmt.Errorf("goal add-path expected at least one argument") } path := os.Getenv("PATH") - for _, arg := range args { - arg, err := homedir.Expand(arg) - if err != nil { - return err - } - path = path + ":" + arg - } + ps := strings.Split(path, ":") + ps = stringsx.DedupeList(ps) + ps = AddHomeExpand(ps, args...) + path = strings.Join(ps, ":") err := os.Setenv("PATH", path) - if runtime.GOOS == "DARWIN" { - gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", "PATH", path) - } + // if runtime.GOOS == "darwin" { + // this is what would be required to work: + // sudo launchctl config user path $PATH -- the following does not work: + // gl.Config.RunIO(cmdIO, "/bin/launchctl", "setenv", "PATH", path) + // } return err } From 433561dc61f64453cfbed5247f09969ac2247244 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 25 Oct 2024 04:14:14 -0700 Subject: [PATCH 250/311] goal: a bit more robustness for crashing but still not dealing with stdio case which doesn't allow the ctrl+c to get through --- goal/builtins.go | 2 +- goal/goal.go | 9 +++++++++ goal/interpreter/interpreter.go | 3 ++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/goal/builtins.go b/goal/builtins.go index 6903aca5d4..8f3abd023a 100644 --- a/goal/builtins.go +++ b/goal/builtins.go @@ -83,7 +83,7 @@ func (gl *Goal) Set(cmdIO *exec.CmdIO, args ...string) error { return fmt.Errorf("expected two arguments, got %d", len(args)) } val := args[1] - if strings.Count(val, ":") > 1 { + if strings.Count(val, ":") > 1 || strings.Contains(val, "~") { vl := stringsx.DedupeList(strings.Split(val, ":")) vl = AddHomeExpand([]string{}, vl...) val = strings.Join(vl, ":") diff --git a/goal/goal.go b/goal/goal.go index 7c8f94a64c..2e7e67a87c 100644 --- a/goal/goal.go +++ b/goal/goal.go @@ -322,6 +322,15 @@ func (gl *Goal) RunCommands(cmds []any) error { return nil } +// DeleteAllJobs deletes any existing jobs, closing stdio. +func (gl *Goal) DeleteAllJobs() { + n := len(gl.Jobs) + for i := n - 1; i >= 0; i-- { + jb := gl.Jobs.Pop() + jb.CmdIO.PopToStart() + } +} + // DeleteJob deletes the given job and returns true if successful, func (gl *Goal) DeleteJob(job *Job) bool { idx := slices.Index(gl.Jobs, job) diff --git a/goal/interpreter/interpreter.go b/goal/interpreter/interpreter.go index 9cca117257..d30080c924 100644 --- a/goal/interpreter/interpreter.go +++ b/goal/interpreter/interpreter.go @@ -140,6 +140,7 @@ func (in *Interpreter) RunCode() (reflect.Value, error) { if err != nil { cancelled := errors.Is(err, context.Canceled) // fmt.Println("cancelled:", cancelled) + in.Goal.DeleteAllJobs() in.Goal.RestoreOrigStdIO() in.Goal.TrState.ResetDepth() if !cancelled { @@ -169,7 +170,7 @@ func (in *Interpreter) RunConfig() error { // It is called automatically in another goroutine in [NewInterpreter]. func (in *Interpreter) MonitorSignals() { c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) + signal.Notify(c, os.Interrupt, syscall.SIGTERM, syscall.SIGINT, syscall.SIGSEGV) for { <-c in.Goal.CancelExecution() From d10c1864c4035670cb750b870050cab019cd8cda Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 31 Oct 2024 01:00:44 -0700 Subject: [PATCH 251/311] goal: & of tensor -> ValuePtr -- needed for atomic exprs --- goal/gosl/examples/basic/atomic.go | 18 ++++++ goal/gosl/examples/basic/atomic.goal | 16 +++++ goal/gosl/examples/basic/compute.go | 6 ++ goal/gosl/examples/basic/compute.goal | 5 ++ goal/gosl/examples/basic/gosl.go | 58 +++++++++++++++++++ goal/gosl/examples/basic/shaders/Compute.wgsl | 8 +++ goal/transpile/transpile.go | 12 +++- goal/transpile/transpile_test.go | 3 +- tensor/base.go | 4 ++ 9 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 goal/gosl/examples/basic/atomic.go create mode 100644 goal/gosl/examples/basic/atomic.goal diff --git a/goal/gosl/examples/basic/atomic.go b/goal/gosl/examples/basic/atomic.go new file mode 100644 index 0000000000..681f1ba127 --- /dev/null +++ b/goal/gosl/examples/basic/atomic.go @@ -0,0 +1,18 @@ +// Code generated by "goal build"; DO NOT EDIT. +//line atomic.goal:1 +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "sync/atomic" + +//gosl:start + +// Atomic does an atomic computation on the data. +func Atomic(i uint32) { //gosl:kernel + atomic.AddInt32(IntData.ValuePtr(int(i), int(Integ)), 1) +} + +//gosl:end diff --git a/goal/gosl/examples/basic/atomic.goal b/goal/gosl/examples/basic/atomic.goal new file mode 100644 index 0000000000..b4e7f97858 --- /dev/null +++ b/goal/gosl/examples/basic/atomic.goal @@ -0,0 +1,16 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "sync/atomic" + +//gosl:start + +// Atomic does an atomic computation on the data. +func Atomic(i uint32) { //gosl:kernel + atomic.AddInt32(&IntData[i, Integ], 1) +} + +//gosl:end diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index f071751cad..dcd21b27ae 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -26,6 +26,12 @@ var ( // //gosl:dims 2 Data *tensor.Float32 + + // IntData is the int data on which the computation operates. + // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + // + //gosl:dims 2 + IntData *tensor.Int32 ) const ( diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal index 9cc4ec19b4..11bb70f398 100644 --- a/goal/gosl/examples/basic/compute.goal +++ b/goal/gosl/examples/basic/compute.goal @@ -22,6 +22,11 @@ var ( // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. //gosl:dims 2 Data *tensor.Float32 + + // IntData is the int data on which the computation operates. + // 2D: outer index is data, inner index is: Raw, Integ, Exp vars. + //gosl:dims 2 + IntData *tensor.Int32 ) const ( diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index 2752aa041b..62a14a3e89 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -27,6 +27,7 @@ type GPUVars int32 //enums:enum const ( ParamsVar GPUVars = 0 DataVar GPUVars = 1 + IntDataVar GPUVars = 2 ) // GPUInit initializes the GPU compute system, @@ -37,6 +38,7 @@ func GPUInit() { { sy := gpu.NewComputeSystem(gp, "Default") GPUSystem = sy + gpu.NewComputePipelineShaderFS(shaders, "shaders/Atomic.wgsl", sy) gpu.NewComputePipelineShaderFS(shaders, "shaders/Compute.wgsl", sy) vars := sy.Vars() { @@ -45,6 +47,7 @@ func GPUInit() { _ = vr vr = sgp.AddStruct("Params", int(unsafe.Sizeof(ParamStruct{})), 1, gpu.ComputeShader) vr = sgp.Add("Data", gpu.Float32, 1, gpu.ComputeShader) + vr = sgp.Add("IntData", gpu.Int32, 1, gpu.ComputeShader) sgp.SetNValues(1) } sy.Config() @@ -58,6 +61,51 @@ func GPURelease() { ComputeGPU.Release() } +// RunAtomic runs the Atomic kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// Can call multiple Run* kernels in a row, which are then all launched +// in the same command submission on the GPU, which is by far the most efficient. +// MUST call RunDone (with optional vars to sync) after all Run calls. +// Alternatively, a single-shot RunOneAtomic call does Run and Done for a +// single run-and-sync case. +func RunAtomic(n int) { + if UseGPU { + RunAtomicGPU(n) + } else { + RunAtomicCPU(n) + } +} + +// RunAtomicGPU runs the Atomic kernel on the GPU. See [RunAtomic] for more info. +func RunAtomicGPU(n int) { + sy := GPUSystem + pl := sy.ComputePipelines["Atomic"] + ce, _ := sy.BeginComputePass() + pl.Dispatch1D(ce, n, 64) +} + +// RunAtomicCPU runs the Atomic kernel on the CPU. +func RunAtomicCPU(n int) { + // todo: need threaded api -- not tensor + for i := range n { + Atomic(uint32(i)) + } +} + +// RunOneAtomic runs the Atomic kernel with given number of elements, +// on either the CPU or GPU depending on the UseGPU variable. +// This version then calls RunDone with the given variables to sync +// after the Run, for a single-shot Run-and-Done call. If multiple kernels +// can be run in sequence, it is much more efficient to do multiple Run* +// calls followed by a RunDone call. +func RunOneAtomic(n int, syncVars ...GPUVars) { + if UseGPU { + RunAtomicGPU(n) + RunDone(syncVars...) + } else { + RunAtomicCPU(n) + } +} // RunCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched @@ -131,6 +179,9 @@ func ToGPU(vars ...GPUVars) { case DataVar: v, _ := syVars.ValueByIndex(0, "Data", 0) gpu.SetValueFrom(v, Data.Values) + case IntDataVar: + v, _ := syVars.ValueByIndex(0, "IntData", 0) + gpu.SetValueFrom(v, IntData.Values) } } } @@ -147,6 +198,9 @@ func ReadFromGPU(vars ...GPUVars) { case DataVar: v, _ := syVars.ValueByIndex(0, "Data", 0) v.GPUToRead(sy.CommandEncoder) + case IntDataVar: + v, _ := syVars.ValueByIndex(0, "IntData", 0) + v.GPUToRead(sy.CommandEncoder) } } } @@ -165,6 +219,10 @@ func SyncFromGPU(vars ...GPUVars) { v, _ := syVars.ValueByIndex(0, "Data", 0) v.ReadSync() gpu.ReadToBytes(v, Data.Values) + case IntDataVar: + v, _ := syVars.ValueByIndex(0, "IntData", 0) + v.ReadSync() + gpu.ReadToBytes(v, IntData.Values) } } } diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 469b475ce0..1e834084ef 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -6,6 +6,8 @@ var Params: array; @group(0) @binding(1) var Data: array; +@group(0) @binding(2) +var IntData: array; @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { @@ -16,6 +18,10 @@ fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; } +fn IndexI322D(s0: i32, s1: i32, i0: u32, i1: u32) -> u32 { + return u32(2) + u32(s0) * i0 + u32(s1) * i1; +} + ///////////// import: "compute.go" const Raw: i32 = 0; @@ -39,6 +45,8 @@ fn Compute(i: u32) { //gosl:kernel Params[0] = params; } +///////////// import: "atomic.go" + ///////////// import: "math32-fastexp.go" fn FastExp(x: f32) -> f32 { if (x <= -88.02969) { // this doesn't add anything and -exp is main use-case anyway diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 13bcb2c4d0..5849628b5e 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -7,6 +7,7 @@ package transpile import ( "fmt" "go/token" + "slices" "strings" "cogentcore.org/core/base/logx" @@ -355,6 +356,12 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, code string, gtoks *Tokens, i if len(commas) == 0 { // not multidim return -1 } + isPtr := false + if idIdx > 0 && toks[idIdx-1].Tok == token.AND { + isPtr = true + lgt := len(*gtoks) + *gtoks = slices.Delete(*gtoks, lgt-2, lgt-1) // get rid of & + } // now we need to determine if it is a Set based on what happens after rb isSet := false stok := token.ILLEGAL @@ -372,7 +379,10 @@ func (st *State) TranspileGoNDimIndex(toks Tokens, code string, gtoks *Tokens, i } } fun := "Value" - if isSet { + if isPtr { + fun = "ValuePtr" + isSet = false + } else if isSet { fun = "Set" switch stok { case token.ADD_ASSIGN: diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 5fd2465533..77006ef627 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -131,6 +131,7 @@ func TestTranspile(t *testing.T) { {"ls -la >> test.out", `goal.Run("ls", "-la", ">>", "test.out")`}, {"ls -la >& test.out", `goal.Run("ls", "-la", ">&", "test.out")`}, {"ls -la >>& test.out", `goal.Run("ls", "-la", ">>&", "test.out")`}, + {"ls | grep ev", `goal.Start("ls", "|"); goal.Run("grep", "ev")`}, {"@1 ls -la", `goal.Run("@1", "ls", "-la")`}, {"git switch main", `goal.Run("git", "switch", "main")`}, {"git checkout 123abc", `goal.Run("git", "checkout", "123abc")`}, @@ -218,7 +219,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"ls | grep ev", `goal.Start("ls", "|"); goal.Run("grep", "ev")`}, + {"&Data[idx, Integ] = integ", `Data.ValuePtr(int(idx), int(Integ)) = integ`}, } st := NewState() st.MathRecord = false diff --git a/tensor/base.go b/tensor/base.go index 4a6a2d95b2..bd47054963 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -68,6 +68,10 @@ func (tsr *Base[T]) Value(i ...int) T { return tsr.Values[tsr.shape.IndexTo1D(i...)] } +func (tsr *Base[T]) ValuePtr(i ...int) *T { + return &tsr.Values[tsr.shape.IndexTo1D(i...)] +} + func (tsr *Base[T]) Set(val T, i ...int) { tsr.Values[tsr.shape.IndexTo1D(i...)] = val } From e02a82b267521c99d9c2da9101b0c6d6e2410a0e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 31 Oct 2024 02:31:30 -0700 Subject: [PATCH 252/311] full atomic support in place --- goal/gosl/examples/basic/main.go | 18 +- goal/gosl/examples/basic/shaders/Atomic.wgsl | 209 +++++++++++++++++++ goal/gosl/gotosl/callgraph.go | 20 +- goal/gosl/gotosl/genkernel.go | 8 +- goal/gosl/gotosl/nodes.go | 16 ++ goal/gosl/gotosl/printer.go | 9 +- goal/gosl/gotosl/translate.go | 11 +- 7 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 goal/gosl/examples/basic/shaders/Atomic.wgsl diff --git a/goal/gosl/examples/basic/main.go b/goal/gosl/examples/basic/main.go index b02a39e2bb..0fa0ca103a 100644 --- a/goal/gosl/examples/basic/main.go +++ b/goal/gosl/examples/basic/main.go @@ -37,10 +37,16 @@ func main() { sltensor.SetShapeSizes(Data, n, 3) // critically, makes GPU compatible Header with strides nt := Data.Len() + IntData = tensor.NewInt32() + sltensor.SetShapeSizes(IntData, n, 3) // critically, makes GPU compatible Header with strides + for i := range nt { Data.Set1D(rand.Float32(), i) } + sid := tensor.NewInt32() + sltensor.SetShapeSizes(sid, n, 3) // critically, makes GPU compatible Header with strides + sd := tensor.NewFloat32() sltensor.SetShapeSizes(sd, n, 3) for i := range nt { @@ -51,29 +57,37 @@ func main() { cpuTmr.Start() UseGPU = false + RunOneAtomic(n) RunOneCompute(n) cpuTmr.Stop() cd := Data + cid := IntData Data = sd + IntData = sid gpuFullTmr := timer.Time{} gpuFullTmr.Start() - ToGPU(ParamsVar, DataVar) + ToGPU(ParamsVar, DataVar, IntDataVar) gpuTmr := timer.Time{} gpuTmr.Start() UseGPU = true + RunAtomic(n) RunCompute(n) gpuTmr.Stop() - RunDone(DataVar) + RunDone(DataVar, IntDataVar) gpuFullTmr.Stop() mx := min(n, 5) + for i := 0; i < mx; i++ { + fmt.Printf("%d\t CPU IntData: %d\t GPU: %d\n", i, cid.Value(1, Integ), sid.Value(i, Integ)) + } + fmt.Println() for i := 0; i < mx; i++ { d := cd.Value(i, Exp) - sd.Value(i, Exp) fmt.Printf("%d\t Raw: %6.4g\t Integ: %6.4g\t Exp: %6.4g\tTrg: %6.4g\tDiff: %g\n", i, sd.Value(i, Raw), sd.Value(i, Integ), sd.Value(i, Exp), cd.Value(i, Exp), d) diff --git a/goal/gosl/examples/basic/shaders/Atomic.wgsl b/goal/gosl/examples/basic/shaders/Atomic.wgsl new file mode 100644 index 0000000000..a9d9d5b6e3 --- /dev/null +++ b/goal/gosl/examples/basic/shaders/Atomic.wgsl @@ -0,0 +1,209 @@ +// Code generated by "gosl"; DO NOT EDIT +// kernel: Atomic + +// // Params are the parameters for the computation. // // gosl:read-only +@group(0) @binding(0) +var Params: array; +@group(0) @binding(1) +var Data: array; +@group(0) @binding(2) +var IntData: array>; + +@compute @workgroup_size(64, 1, 1) +fn main(@builtin(global_invocation_id) idx: vec3) { + Atomic(idx.x); +} + +fn IndexF322D(s0: f32, s1: f32, i0: u32, i1: u32) -> u32 { + return u32(2) + bitcast(s0) * i0 + bitcast(s1) * i1; +} + +fn IndexI322D(s0: i32, s1: i32, i0: u32, i1: u32) -> u32 { + return u32(2) + u32(s0) * i0 + u32(s1) * i1; +} + + +///////////// import: "compute.go" +const Raw: i32 = 0; +const Integ: i32 = 1; +const Exp: i32 = 2; +struct ParamStruct { + Tau: f32, + Dt: f32, + pad: f32, + pad1: f32, +} + +///////////// import: "atomic.go" +fn Atomic(i: u32) { //gosl:kernel + atomicAdd(&IntData[IndexI322D(IntData[0], IntData[1], u32(i),u32(Integ))], 1); +} + +///////////// import: "math32-fastexp.go" + +///////////// import: "slrand.wgsl" +fn Philox2x32round(counter: su64, key: u32) -> su64 { + let mul = Uint32Mul64(u32(0xD256D193), counter.x); + var ctr: su64; + ctr.x = mul.y ^ key ^ counter.y; + ctr.y = mul.x; + return ctr; +} +fn Philox2x32bumpkey(key: u32) -> u32 { + return key + u32(0x9E3779B9); +} +fn Philox2x32(counter: su64, key: u32) -> vec2 { + var ctr = Philox2x32round(counter, key); // 1 + var ky = Philox2x32bumpkey(key); + ctr = Philox2x32round(ctr, ky); // 2 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 3 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 4 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 5 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 6 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 7 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 8 + ky = Philox2x32bumpkey(ky); + ctr = Philox2x32round(ctr, ky); // 9 + ky = Philox2x32bumpkey(ky); + return Philox2x32round(ctr, ky); // 10 +} +fn RandUint32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Philox2x32(Uint64Add32(counter, funcIndex), key); +} +fn RandUint32(counter: su64, funcIndex: u32, key: u32) -> u32 { + return Philox2x32(Uint64Add32(counter, funcIndex), key).x; +} +fn RandFloat32Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} +fn RandFloat32(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32(RandUint32(counter, funcIndex, key)); +} +fn RandFloat32Range11Vec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + return Uint32ToFloat32Vec2(RandUint32Vec2(counter, funcIndex, key)); +} +fn RandFloat32Range11(counter: su64, funcIndex: u32, key: u32) -> f32 { + return Uint32ToFloat32Range11(RandUint32(counter, funcIndex, key)); +} +fn RandBoolP(counter: su64, funcIndex: u32, key: u32, p: f32) -> bool { + return (RandFloat32(counter, funcIndex, key) < p); +} +fn sincospi(x: f32) -> vec2 { + let PIf = 3.1415926535897932; + var r: vec2; + r.x = cos(PIf*x); + r.y = sin(PIf*x); + return r; +} +fn RandFloat32NormVec2(counter: su64, funcIndex: u32, key: u32) -> vec2 { + let ur = RandUint32Vec2(counter, funcIndex, key); + var f = sincospi(Uint32ToFloat32Range11(ur.x)); + let r = sqrt(-2.0 * log(Uint32ToFloat32(ur.y))); // guaranteed to avoid 0. + return f * r; +} +fn RandFloat32Norm(counter: su64, funcIndex: u32, key: u32) -> f32 { + return RandFloat32Vec2(counter, funcIndex, key).x; +} +fn RandUint32N(counter: su64, funcIndex: u32, key: u32, n: u32) -> u32 { + let v = RandFloat32(counter, funcIndex, key); + return u32(v * f32(n)); +} +struct RandCounter { + Counter: su64, + HiSeed: u32, + pad: u32, +} +fn RandCounter_Reset(ct: ptr) { + (*ct).Counter.x = u32(0); + (*ct).Counter.y = (*ct).HiSeed; +} +fn RandCounter_Seed(ct: ptr, seed: u32) { + (*ct).HiSeed = seed; + RandCounter_Reset(ct); +} +fn RandCounter_Add(ct: ptr, inc: u32) { + (*ct).Counter = Uint64Add32((*ct).Counter, inc); +} + +///////////// import: "sltype.wgsl" +alias su64 = vec2; +fn Uint32Mul64(a: u32, b: u32) -> su64 { + let LOMASK = (((u32(1))<<16)-1); + var r: su64; + r.x = a * b; /* full low multiply */ + let ahi = a >> 16; + let alo = a & LOMASK; + let bhi = b >> 16; + let blo = b & LOMASK; + let ahbl = ahi * blo; + let albh = alo * bhi; + let ahbl_albh = ((ahbl&LOMASK) + (albh&LOMASK)); + var hit = ahi*bhi + (ahbl>>16) + (albh>>16); + hit += ahbl_albh >> 16; /* carry from the sum of lo(ahbl) + lo(albh) ) */ + /* carry from the sum with alo*blo */ + if ((r.x >> u32(16)) < (ahbl_albh&LOMASK)) { + hit += u32(1); + } + r.y = hit; + return r; +} +/* +fn Uint32Mul64(a: u32, b: u32) -> su64 { + return su64(a) * su64(b); +} +*/ +fn Uint64Add32(a: su64, b: u32) -> su64 { + if (b == 0) { + return a; + } + var s = a; + if (s.x > u32(0xffffffff) - b) { + s.y++; + s.x = (b - 1) - (u32(0xffffffff) - s.x); + } else { + s.x += b; + } + return s; +} +fn Uint64Incr(a: su64) -> su64 { + var s = a; + if(s.x == 0xffffffff) { + s.y++; + s.x = u32(0); + } else { + s.x++; + } + return s; +} +fn Uint32ToFloat32(val: u32) -> f32 { + let factor = f32(1.0) / (f32(u32(0xffffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + var f = f32(val) * factor + halffactor; + if (f == 1.0) { // exclude 1 + return bitcast(0x3F7FFFFF); + } + return f; +} +fn Uint32ToFloat32Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32(val.x); + r.y = Uint32ToFloat32(val.y); + return r; +} +fn Uint32ToFloat32Range11(val: u32) -> f32 { + let factor = f32(1.0) / (f32(i32(0x7fffffff)) + f32(1.0)); + let halffactor = f32(0.5) * factor; + return (f32(val) * factor + halffactor); +} +fn Uint32ToFloat32Range11Vec2(val: vec2) -> vec2 { + var r: vec2; + r.x = Uint32ToFloat32Range11(val.x); + r.y = Uint32ToFloat32Range11(val.y); + return r; +} \ No newline at end of file diff --git a/goal/gosl/gotosl/callgraph.go b/goal/gosl/gotosl/callgraph.go index d2eeee0ef5..f686ac02e6 100644 --- a/goal/gosl/gotosl/callgraph.go +++ b/goal/gosl/gotosl/callgraph.go @@ -13,8 +13,9 @@ import ( // Function represents the call graph of functions type Function struct { - Name string - Funcs map[string]*Function + Name string + Funcs map[string]*Function + Atomics map[string]*Var // variables that have atomic operations in this function } func NewFunction(name string) *Function { @@ -60,6 +61,21 @@ func (st *State) AllFuncs(name string) map[string]*Function { return all } +// AtomicVars returns all the variables marked as atomic +// within the list of functions. +func (st *State) AtomicVars(funcs map[string]*Function) map[string]*Var { + avars := make(map[string]*Var) + for _, fn := range funcs { + if fn.Atomics == nil { + continue + } + for vn, v := range fn.Atomics { + avars[vn] = v + } + } + return avars +} + func (st *State) PrintFuncGraph() { funs := maps.Keys(st.FuncGraph) sort.Strings(funs) diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index da33fdf680..77412d153d 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -12,7 +12,7 @@ import ( // GenKernelHeader returns the novel generated WGSL kernel code // for given kernel, which goes at the top of the resulting file. -func (st *State) GenKernelHeader(sy *System, kn *Kernel) string { +func (st *State) GenKernelHeader(sy *System, kn *Kernel, avars map[string]*Var) string { var b strings.Builder b.WriteString("// Code generated by \"gosl\"; DO NOT EDIT\n") b.WriteString("// kernel: " + kn.Name + "\n\n") @@ -31,7 +31,11 @@ func (st *State) GenKernelHeader(sy *System, kn *Kernel) string { } b.WriteString(fmt.Sprintf("@group(%d) @binding(%d)\n", gi, vi)) b.WriteString(fmt.Sprintf("var<%s> %s: ", str, vr.Name)) - b.WriteString(fmt.Sprintf("array<%s>;\n", vr.SLType())) + if _, ok := avars[vr.Name]; ok { + b.WriteString(fmt.Sprintf("array>;\n", vr.SLType())) + } else { + b.WriteString(fmt.Sprintf("array<%s>;\n", vr.SLType())) + } } } diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 45291eced7..ce34e9ad37 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1738,6 +1738,20 @@ func (p *printer) tensorMethod(x *ast.CallExpr, vr *Var, methName string) { if strings.HasPrefix(methName, "Set") { stArg = 1 } + if strings.HasSuffix(methName, "Ptr") { + p.print(token.AND) + if p.curMethIsAtomic { + gv := p.GoToSL.GlobalVar(vr.Name) + if gv != nil { + if p.curFunc != nil { + if p.curFunc.Atomics == nil { + p.curFunc.Atomics = make(map[string]*Var) + } + p.curFunc.Atomics[vr.Name] = vr + } + } + } + } p.print(vr.Name, token.LBRACK) p.print(vr.IndexFunc(), token.LPAREN) nd := vr.TensorDims @@ -1854,6 +1868,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { // fmt.Println(pathIsPackage, recvType, methName, recvPath) if pathIsPackage { if recvType == "atomic" { + p.curMethIsAtomic = true switch methName { case "AddInt32": p.print("atomicAdd") @@ -1892,6 +1907,7 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { } p.setPos(x.Rparen) p.print(token.RPAREN) + p.curMethIsAtomic = false p.assignRwArgs(rwargs) // gosl: assign temp var back to global var } diff --git a/goal/gosl/gotosl/printer.go b/goal/gosl/gotosl/printer.go index dc2a591f19..2c1cf86be8 100644 --- a/goal/gosl/gotosl/printer.go +++ b/goal/gosl/gotosl/printer.go @@ -108,10 +108,11 @@ type printer struct { // current arguments to function that are pointers and thus need dereferencing // when accessing fields - curPtrArgs []*ast.Ident - curFunc *Function - curMethRecv *ast.Field // current method receiver, also included in curPtrArgs if ptr - curReturnType *ast.Ident + curPtrArgs []*ast.Ident + curFunc *Function + curMethRecv *ast.Field // current method receiver, also included in curPtrArgs if ptr + curReturnType *ast.Ident + curMethIsAtomic bool // current method an atomic.* function -- marks arg as atomic } func (p *printer) internalError(msg ...any) { diff --git a/goal/gosl/gotosl/translate.go b/goal/gosl/gotosl/translate.go index 04e741f41b..42e1af88de 100644 --- a/goal/gosl/gotosl/translate.go +++ b/goal/gosl/gotosl/translate.go @@ -123,15 +123,16 @@ func (st *State) TranslateDir(pf string) error { sort.Strings(kns) for _, knm := range kns { kn := sy.Kernels[knm] - // if st.Config.Debug { - fmt.Printf("###################################\nTranslating Kernel file: %s\n", kn.Name) - // } - hdr := st.GenKernelHeader(sy, kn) - lines := bytes.Split([]byte(hdr), []byte("\n")) st.KernelFuncs = st.AllFuncs(kn.Name) if st.KernelFuncs == nil { continue } + avars := st.AtomicVars(st.KernelFuncs) + // if st.Config.Debug { + fmt.Printf("###################################\nTranslating Kernel file: %s\n", kn.Name) + // } + hdr := st.GenKernelHeader(sy, kn, avars) + lines := bytes.Split([]byte(hdr), []byte("\n")) for fn := range st.GoVarsFiles { // do varsFiles first!! lines = doKernelFile(fn, lines) } From 2b46ead63b6e93281a25d045ebed9c295516ef12 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 1 Nov 2024 11:37:55 -0700 Subject: [PATCH 253/311] goal: add atomic test case and const slice test case, and rename atomiccounters -> atomicx with MaxInt32 impl. --- .../atomicx_test.go} | 12 +++- .../atomiccounter.go => atomicx/counter.go} | 4 +- base/atomicx/max.go | 15 +++++ goal/gosl/examples/basic/compute.go | 4 ++ goal/gosl/examples/basic/compute.goal | 4 ++ goal/gosl/examples/basic/gosl.go | 60 +++++++++---------- goal/gosl/examples/basic/shaders/Atomic.wgsl | 2 + goal/gosl/examples/basic/shaders/Compute.wgsl | 2 + goal/gosl/gotosl/nodes.go | 8 ++- 9 files changed, 75 insertions(+), 36 deletions(-) rename base/{atomiccounter/atomiccounter_test.go => atomicx/atomicx_test.go} (77%) rename base/{atomiccounter/atomiccounter.go => atomicx/counter.go} (92%) create mode 100644 base/atomicx/max.go diff --git a/base/atomiccounter/atomiccounter_test.go b/base/atomicx/atomicx_test.go similarity index 77% rename from base/atomiccounter/atomiccounter_test.go rename to base/atomicx/atomicx_test.go index 327700d2eb..b8f4b89223 100644 --- a/base/atomiccounter/atomiccounter_test.go +++ b/base/atomicx/atomicx_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package atomiccounter +package atomicx import ( "testing" @@ -26,3 +26,13 @@ func TestCounter(t *testing.T) { c.Set(2) assert.Equal(t, int64(2), c.Value()) } + +func TestMax(t *testing.T) { + a := int32(10) + MaxInt32(&a, 5) + assert.Equal(t, a, int32(10)) + MaxInt32(&a, 10) + assert.Equal(t, a, int32(10)) + MaxInt32(&a, 11) + assert.Equal(t, a, int32(11)) +} diff --git a/base/atomiccounter/atomiccounter.go b/base/atomicx/counter.go similarity index 92% rename from base/atomiccounter/atomiccounter.go rename to base/atomicx/counter.go index 9846df1c74..e8aa6e0562 100644 --- a/base/atomiccounter/atomiccounter.go +++ b/base/atomicx/counter.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package atomiccounter implements a basic atomic int64 counter. -package atomiccounter +// Package atomicx implements misc atomic functions. +package atomicx import ( "sync/atomic" diff --git a/base/atomicx/max.go b/base/atomicx/max.go new file mode 100644 index 0000000000..763e7c6e16 --- /dev/null +++ b/base/atomicx/max.go @@ -0,0 +1,15 @@ +// Copyright (c) 2018, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomicx + +import "sync/atomic" + +// MaxInt32 performs an atomic Max operation: a = max(a, b) +func MaxInt32(a *int32, b int32) { + old := atomic.LoadInt32(a) + for old < b && !atomic.CompareAndSwapInt32(a, old, b) { + old = atomic.LoadInt32(a) + } +} diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index dcd21b27ae..32206951ff 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -38,8 +38,12 @@ const ( Raw int = iota Integ Exp + NVars ) +// testSlice is a global array. +var testSlice = [NVars]int{Raw, Integ, Exp} + // ParamStruct has the test params type ParamStruct struct { diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal index 11bb70f398..4fdabc28bf 100644 --- a/goal/gosl/examples/basic/compute.goal +++ b/goal/gosl/examples/basic/compute.goal @@ -33,8 +33,12 @@ const ( Raw int = iota Integ Exp + NVars ) +// testSlice is a global array. +var testSlice = [NVars]int{Raw, Integ, Exp} + // ParamStruct has the test params type ParamStruct struct { diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index 62a14a3e89..b8be9175b3 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -61,94 +61,94 @@ func GPURelease() { ComputeGPU.Release() } -// RunAtomic runs the Atomic kernel with given number of elements, +// RunCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched // in the same command submission on the GPU, which is by far the most efficient. // MUST call RunDone (with optional vars to sync) after all Run calls. -// Alternatively, a single-shot RunOneAtomic call does Run and Done for a +// Alternatively, a single-shot RunOneCompute call does Run and Done for a // single run-and-sync case. -func RunAtomic(n int) { +func RunCompute(n int) { if UseGPU { - RunAtomicGPU(n) + RunComputeGPU(n) } else { - RunAtomicCPU(n) + RunComputeCPU(n) } } -// RunAtomicGPU runs the Atomic kernel on the GPU. See [RunAtomic] for more info. -func RunAtomicGPU(n int) { +// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. +func RunComputeGPU(n int) { sy := GPUSystem - pl := sy.ComputePipelines["Atomic"] + pl := sy.ComputePipelines["Compute"] ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, 64) } -// RunAtomicCPU runs the Atomic kernel on the CPU. -func RunAtomicCPU(n int) { +// RunComputeCPU runs the Compute kernel on the CPU. +func RunComputeCPU(n int) { // todo: need threaded api -- not tensor for i := range n { - Atomic(uint32(i)) + Compute(uint32(i)) } } -// RunOneAtomic runs the Atomic kernel with given number of elements, +// RunOneCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // This version then calls RunDone with the given variables to sync // after the Run, for a single-shot Run-and-Done call. If multiple kernels // can be run in sequence, it is much more efficient to do multiple Run* // calls followed by a RunDone call. -func RunOneAtomic(n int, syncVars ...GPUVars) { +func RunOneCompute(n int, syncVars ...GPUVars) { if UseGPU { - RunAtomicGPU(n) + RunComputeGPU(n) RunDone(syncVars...) } else { - RunAtomicCPU(n) + RunComputeCPU(n) } } -// RunCompute runs the Compute kernel with given number of elements, +// RunAtomic runs the Atomic kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched // in the same command submission on the GPU, which is by far the most efficient. // MUST call RunDone (with optional vars to sync) after all Run calls. -// Alternatively, a single-shot RunOneCompute call does Run and Done for a +// Alternatively, a single-shot RunOneAtomic call does Run and Done for a // single run-and-sync case. -func RunCompute(n int) { +func RunAtomic(n int) { if UseGPU { - RunComputeGPU(n) + RunAtomicGPU(n) } else { - RunComputeCPU(n) + RunAtomicCPU(n) } } -// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. -func RunComputeGPU(n int) { +// RunAtomicGPU runs the Atomic kernel on the GPU. See [RunAtomic] for more info. +func RunAtomicGPU(n int) { sy := GPUSystem - pl := sy.ComputePipelines["Compute"] + pl := sy.ComputePipelines["Atomic"] ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, 64) } -// RunComputeCPU runs the Compute kernel on the CPU. -func RunComputeCPU(n int) { +// RunAtomicCPU runs the Atomic kernel on the CPU. +func RunAtomicCPU(n int) { // todo: need threaded api -- not tensor for i := range n { - Compute(uint32(i)) + Atomic(uint32(i)) } } -// RunOneCompute runs the Compute kernel with given number of elements, +// RunOneAtomic runs the Atomic kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // This version then calls RunDone with the given variables to sync // after the Run, for a single-shot Run-and-Done call. If multiple kernels // can be run in sequence, it is much more efficient to do multiple Run* // calls followed by a RunDone call. -func RunOneCompute(n int, syncVars ...GPUVars) { +func RunOneAtomic(n int, syncVars ...GPUVars) { if UseGPU { - RunComputeGPU(n) + RunAtomicGPU(n) RunDone(syncVars...) } else { - RunComputeCPU(n) + RunAtomicCPU(n) } } // RunDone must be called after Run* calls to start compute kernels. diff --git a/goal/gosl/examples/basic/shaders/Atomic.wgsl b/goal/gosl/examples/basic/shaders/Atomic.wgsl index a9d9d5b6e3..8db12825af 100644 --- a/goal/gosl/examples/basic/shaders/Atomic.wgsl +++ b/goal/gosl/examples/basic/shaders/Atomic.wgsl @@ -27,6 +27,8 @@ fn IndexI322D(s0: i32, s1: i32, i0: u32, i1: u32) -> u32 { const Raw: i32 = 0; const Integ: i32 = 1; const Exp: i32 = 2; +const NVars: i32 = 3; +var testSlice = [NVars]i32{Raw, Integ, Exp}; struct ParamStruct { Tau: f32, Dt: f32, diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 1e834084ef..6667c6a3a7 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -27,6 +27,8 @@ fn IndexI322D(s0: i32, s1: i32, i0: u32, i1: u32) -> u32 { const Raw: i32 = 0; const Integ: i32 = 1; const Exp: i32 = 2; +const NVars: i32 = 3; +var testSlice = [NVars]i32{Raw, Integ, Exp}; struct ParamStruct { Tau: f32, Dt: f32, diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index ce34e9ad37..0835fc8214 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1867,11 +1867,13 @@ func (p *printer) methodExpr(x *ast.CallExpr, depth int) { } // fmt.Println(pathIsPackage, recvType, methName, recvPath) if pathIsPackage { - if recvType == "atomic" { + if recvType == "atomic" || recvType == "atomicx" { p.curMethIsAtomic = true - switch methName { - case "AddInt32": + switch { + case strings.HasPrefix(methName, "Add"): p.print("atomicAdd") + case strings.HasPrefix(methName, "Max"): + p.print("atomicMax") } } else { p.print(recvType + "." + methName) From cd0f47484af88594363e3e5300cca7ccbb88c123 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 1 Nov 2024 12:48:53 -0700 Subject: [PATCH 254/311] add Editable to texteditor context menu when in read only mode --- texteditor/events.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/texteditor/events.go b/texteditor/events.go index 46f70d5b54..1f86f3ee51 100644 --- a/texteditor/events.go +++ b/texteditor/events.go @@ -729,5 +729,13 @@ func (ed *Editor) contextMenu(m *core.Scene) { OnClick(func(e events.Event) { ed.Clear() }) + core.NewButton(m).SetText("Editable").SetIcon(icons.Edit). + OnClick(func(e events.Event) { + ed.SetReadOnly(false) + if ed.Buffer != nil { + ed.Buffer.Info.Generated = false // another reason it is !editable + } + ed.Update() + }) } } From 1fbfb705747062a79bdafac7d695607399722da7 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 1 Nov 2024 15:56:10 -0700 Subject: [PATCH 255/311] gosl properly handles global array vars as consts. --- goal/gosl/examples/basic/compute.go | 3 - goal/gosl/examples/basic/compute.goal | 3 - goal/gosl/examples/basic/gosl.go | 60 +++++++++---------- goal/gosl/examples/basic/shaders/Atomic.wgsl | 1 - goal/gosl/examples/basic/shaders/Compute.wgsl | 1 - goal/gosl/gotosl/nodes.go | 40 +++++++++---- goal/gosl/gotosl/testdata/Compute.golden | 1 + goal/gosl/gotosl/testdata/basic.go | 3 + 8 files changed, 63 insertions(+), 49 deletions(-) diff --git a/goal/gosl/examples/basic/compute.go b/goal/gosl/examples/basic/compute.go index 32206951ff..1714f9ec0d 100644 --- a/goal/gosl/examples/basic/compute.go +++ b/goal/gosl/examples/basic/compute.go @@ -41,9 +41,6 @@ const ( NVars ) -// testSlice is a global array. -var testSlice = [NVars]int{Raw, Integ, Exp} - // ParamStruct has the test params type ParamStruct struct { diff --git a/goal/gosl/examples/basic/compute.goal b/goal/gosl/examples/basic/compute.goal index 4fdabc28bf..27297ef6a6 100644 --- a/goal/gosl/examples/basic/compute.goal +++ b/goal/gosl/examples/basic/compute.goal @@ -36,9 +36,6 @@ const ( NVars ) -// testSlice is a global array. -var testSlice = [NVars]int{Raw, Integ, Exp} - // ParamStruct has the test params type ParamStruct struct { diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index b8be9175b3..62a14a3e89 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -61,94 +61,94 @@ func GPURelease() { ComputeGPU.Release() } -// RunCompute runs the Compute kernel with given number of elements, +// RunAtomic runs the Atomic kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched // in the same command submission on the GPU, which is by far the most efficient. // MUST call RunDone (with optional vars to sync) after all Run calls. -// Alternatively, a single-shot RunOneCompute call does Run and Done for a +// Alternatively, a single-shot RunOneAtomic call does Run and Done for a // single run-and-sync case. -func RunCompute(n int) { +func RunAtomic(n int) { if UseGPU { - RunComputeGPU(n) + RunAtomicGPU(n) } else { - RunComputeCPU(n) + RunAtomicCPU(n) } } -// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. -func RunComputeGPU(n int) { +// RunAtomicGPU runs the Atomic kernel on the GPU. See [RunAtomic] for more info. +func RunAtomicGPU(n int) { sy := GPUSystem - pl := sy.ComputePipelines["Compute"] + pl := sy.ComputePipelines["Atomic"] ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, 64) } -// RunComputeCPU runs the Compute kernel on the CPU. -func RunComputeCPU(n int) { +// RunAtomicCPU runs the Atomic kernel on the CPU. +func RunAtomicCPU(n int) { // todo: need threaded api -- not tensor for i := range n { - Compute(uint32(i)) + Atomic(uint32(i)) } } -// RunOneCompute runs the Compute kernel with given number of elements, +// RunOneAtomic runs the Atomic kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // This version then calls RunDone with the given variables to sync // after the Run, for a single-shot Run-and-Done call. If multiple kernels // can be run in sequence, it is much more efficient to do multiple Run* // calls followed by a RunDone call. -func RunOneCompute(n int, syncVars ...GPUVars) { +func RunOneAtomic(n int, syncVars ...GPUVars) { if UseGPU { - RunComputeGPU(n) + RunAtomicGPU(n) RunDone(syncVars...) } else { - RunComputeCPU(n) + RunAtomicCPU(n) } } -// RunAtomic runs the Atomic kernel with given number of elements, +// RunCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // Can call multiple Run* kernels in a row, which are then all launched // in the same command submission on the GPU, which is by far the most efficient. // MUST call RunDone (with optional vars to sync) after all Run calls. -// Alternatively, a single-shot RunOneAtomic call does Run and Done for a +// Alternatively, a single-shot RunOneCompute call does Run and Done for a // single run-and-sync case. -func RunAtomic(n int) { +func RunCompute(n int) { if UseGPU { - RunAtomicGPU(n) + RunComputeGPU(n) } else { - RunAtomicCPU(n) + RunComputeCPU(n) } } -// RunAtomicGPU runs the Atomic kernel on the GPU. See [RunAtomic] for more info. -func RunAtomicGPU(n int) { +// RunComputeGPU runs the Compute kernel on the GPU. See [RunCompute] for more info. +func RunComputeGPU(n int) { sy := GPUSystem - pl := sy.ComputePipelines["Atomic"] + pl := sy.ComputePipelines["Compute"] ce, _ := sy.BeginComputePass() pl.Dispatch1D(ce, n, 64) } -// RunAtomicCPU runs the Atomic kernel on the CPU. -func RunAtomicCPU(n int) { +// RunComputeCPU runs the Compute kernel on the CPU. +func RunComputeCPU(n int) { // todo: need threaded api -- not tensor for i := range n { - Atomic(uint32(i)) + Compute(uint32(i)) } } -// RunOneAtomic runs the Atomic kernel with given number of elements, +// RunOneCompute runs the Compute kernel with given number of elements, // on either the CPU or GPU depending on the UseGPU variable. // This version then calls RunDone with the given variables to sync // after the Run, for a single-shot Run-and-Done call. If multiple kernels // can be run in sequence, it is much more efficient to do multiple Run* // calls followed by a RunDone call. -func RunOneAtomic(n int, syncVars ...GPUVars) { +func RunOneCompute(n int, syncVars ...GPUVars) { if UseGPU { - RunAtomicGPU(n) + RunComputeGPU(n) RunDone(syncVars...) } else { - RunAtomicCPU(n) + RunComputeCPU(n) } } // RunDone must be called after Run* calls to start compute kernels. diff --git a/goal/gosl/examples/basic/shaders/Atomic.wgsl b/goal/gosl/examples/basic/shaders/Atomic.wgsl index 8db12825af..4da119c151 100644 --- a/goal/gosl/examples/basic/shaders/Atomic.wgsl +++ b/goal/gosl/examples/basic/shaders/Atomic.wgsl @@ -28,7 +28,6 @@ const Raw: i32 = 0; const Integ: i32 = 1; const Exp: i32 = 2; const NVars: i32 = 3; -var testSlice = [NVars]i32{Raw, Integ, Exp}; struct ParamStruct { Tau: f32, Dt: f32, diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index 6667c6a3a7..c7cc6c8618 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -28,7 +28,6 @@ const Raw: i32 = 0; const Integ: i32 = 1; const Exp: i32 = 2; const NVars: i32 = 3; -var testSlice = [NVars]i32{Raw, Integ, Exp}; struct ParamStruct { Tau: f32, Dt: f32, diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 0835fc8214..3f46beb7b1 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -1360,12 +1360,18 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { case *ast.CompositeLit: // composite literal elements that are composite literals themselves may have the type omitted + lb := token.LBRACE + rb := token.RBRACE + if _, isAry := x.Type.(*ast.ArrayType); isAry { + lb = token.LPAREN + rb = token.RPAREN + } if x.Type != nil { p.expr1(x.Type, token.HighestPrec, depth) } p.level++ p.setPos(x.Lbrace) - p.print(token.LBRACE) + p.print(lb) p.exprList(x.Lbrace, x.Elts, 1, commaTerm, x.Rbrace, x.Incomplete) // do not insert extra line break following a /*-style comment // before the closing '}' as it might break the code if there @@ -1380,7 +1386,7 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { // the proper level of indentation p.print(indent, unindent, mode) p.setPos(x.Rbrace) - p.print(token.RBRACE, mode) + p.print(rb, mode) p.level-- case *ast.Ellipsis: @@ -1390,12 +1396,13 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int) { } case *ast.ArrayType: - p.print(token.LBRACK) - if x.Len != nil { - p.expr(x.Len) - } - p.print(token.RBRACK) - p.expr(x.Elt) + p.print("array") + // p.print(token.LBRACK) + // if x.Len != nil { + // p.expr(x.Len) + // } + // p.print(token.RBRACK) + // p.expr(x.Elt) case *ast.StructType: // p.print(token.STRUCT) @@ -2745,13 +2752,19 @@ func (p *printer) genDecl(d *ast.GenDecl) { p.valueSpec(vs, keepType[i], d.Tok, firstSpec, isIota, i) } } else { + tok := d.Tok + if p.curFunc == nil && tok == token.VAR { // only system vars are supported at global scope + // could add further comment-directive logic + // to specify or scope if needed + tok = token.CONST + } var line int for i, s := range d.Specs { if i > 0 { p.linebreak(p.lineFor(s.Pos()), 1, ignore, p.linesFrom(line) > 0) } p.recordLine(&line) - p.spec(s, n, false, d.Tok) + p.spec(s, n, false, tok) } } // p.print(unindent, formfeed) @@ -2759,8 +2772,12 @@ func (p *printer) genDecl(d *ast.GenDecl) { // p.setPos(d.Rparen) // p.print(token.RPAREN) } else if len(d.Specs) > 0 { + tok := d.Tok + if p.curFunc == nil && tok == token.VAR { // only system vars are supported at global scope + tok = token.CONST + } // single declaration - p.spec(d.Specs[0], 1, true, d.Tok) + p.spec(d.Specs[0], 1, true, tok) } } @@ -2942,10 +2959,11 @@ func (p *printer) funcDecl(d *ast.FuncDecl) { if p.GoToSL.GetFuncGraph { p.curFunc = p.GoToSL.RecycleFunc(fname) } else { - _, ok := p.GoToSL.KernelFuncs[fname] + fn, ok := p.GoToSL.KernelFuncs[fname] if !ok { return } + p.curFunc = fn } p.setComment(d.Doc) p.setPos(d.Pos()) diff --git a/goal/gosl/gotosl/testdata/Compute.golden b/goal/gosl/gotosl/testdata/Compute.golden index 4f7caade21..9f60a056c0 100644 --- a/goal/gosl/gotosl/testdata/Compute.golden +++ b/goal/gosl/gotosl/testdata/Compute.golden @@ -24,6 +24,7 @@ const NoEvalMode: Modes = 0; const AllModes: Modes = 1; const Train: Modes = 2; const Test: Modes = 3; +const testSlice = array(NoEvalMode, AllModes, Train, Test); struct DataStruct { Raw: f32, Integ: f32, diff --git a/goal/gosl/gotosl/testdata/basic.go b/goal/gosl/gotosl/testdata/basic.go index 6eb20ac6be..ba4b425107 100644 --- a/goal/gosl/gotosl/testdata/basic.go +++ b/goal/gosl/gotosl/testdata/basic.go @@ -69,6 +69,9 @@ const ( Test ) +// testSlice is a global array: will be const = array(...); +var testSlice = [NVars]Modes{NoEvalMode, AllModes, Train, Test} + // DataStruct has the test data type DataStruct struct { From afc747f3b4f11439190245d55d178f4c356a1eb1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 1 Nov 2024 16:25:03 -0700 Subject: [PATCH 256/311] gosl: enumgen exports the N const with //gosl:start, end wrapper -- needed for gpu code --- enums/enumgen/methods.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/enums/enumgen/methods.go b/enums/enumgen/methods.go index b7cc783130..b8b633e4cb 100644 --- a/enums/enumgen/methods.go +++ b/enums/enumgen/methods.go @@ -45,8 +45,10 @@ func (i {{.Name}}) {{if .IsBitFlag}} BitIndexString {{else}} String {{end}} () s `)) var NConstantTmpl = template.Must(template.New("StringNConstant").Parse( - `//{{.Name}}N is the highest valid value for type {{.Name}}, plus one. + `//gosl:start +//{{.Name}}N is the highest valid value for type {{.Name}}, plus one. const {{.Name}}N {{.Name}} = {{.MaxValueP1}} +//gosl:end `)) var SetStringMethodTmpl = template.Must(template.New("SetStringMethod").Parse( From 8a575187bc61c553832f78bf09f21463b8c7beba Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 2 Nov 2024 12:39:45 -0700 Subject: [PATCH 257/311] gosl: for range expr translation --- goal/gosl/gotosl/genkernel.go | 3 ++- goal/gosl/gotosl/nodes.go | 33 +++++++++++++++--------- goal/gosl/gotosl/testdata/Compute.golden | 5 ++++ goal/gosl/gotosl/testdata/basic.go | 4 +++ 4 files changed, 32 insertions(+), 13 deletions(-) diff --git a/goal/gosl/gotosl/genkernel.go b/goal/gosl/gotosl/genkernel.go index 77412d153d..e3e1b94e20 100644 --- a/goal/gosl/gotosl/genkernel.go +++ b/goal/gosl/gotosl/genkernel.go @@ -39,7 +39,8 @@ func (st *State) GenKernelHeader(sy *System, kn *Kernel, avars map[string]*Var) } } - b.WriteString("\n") + b.WriteString("\nalias GPUVars = i32;\n\n") // gets included when iteratively processing enumgen.go + b.WriteString("@compute @workgroup_size(64, 1, 1)\n") // todo: conditional on different index dimensionality b.WriteString("fn main(@builtin(global_invocation_id) idx: vec3) {\n") diff --git a/goal/gosl/gotosl/nodes.go b/goal/gosl/gotosl/nodes.go index 3f46beb7b1..29be2a2744 100644 --- a/goal/gosl/gotosl/nodes.go +++ b/goal/gosl/gotosl/nodes.go @@ -2333,22 +2333,31 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, nosemi bool) { p.block(s.Body, 1) case *ast.RangeStmt: + // gosl: only supporting the for i := range 10 kind of range loop p.print(token.FOR, blank) if s.Key != nil { + p.print(token.LPAREN, "var", blank) p.expr(s.Key) - if s.Value != nil { - // use position of value following the comma as - // comma position for correct comment placement - p.setPos(s.Value.Pos()) - p.print(token.COMMA, blank) - p.expr(s.Value) - } - p.print(blank) - p.setPos(s.TokPos) - p.print(s.Tok, blank) + p.print(token.ASSIGN, "0", token.SEMICOLON, blank) + p.expr(s.Key) + p.print(token.LSS) + p.expr(stripParens(s.X)) + p.print(token.SEMICOLON, blank) + p.expr(s.Key) + p.print(token.INC, token.RPAREN) + // if s.Value != nil { + // // use position of value following the comma as + // // comma position for correct comment placement + // p.setPos(s.Value.Pos()) + // p.print(token.COMMA, blank) + // p.expr(s.Value) + // } + // p.print(blank) + // p.setPos(s.TokPos) + // p.print(s.Tok, blank) } - p.print(token.RANGE, blank) - p.expr(stripParens(s.X)) + // p.print(token.RANGE, blank) + // p.expr(stripParens(s.X)) p.print(blank) p.block(s.Body, 1) diff --git a/goal/gosl/gotosl/testdata/Compute.golden b/goal/gosl/gotosl/testdata/Compute.golden index 9f60a056c0..6183713ad9 100644 --- a/goal/gosl/gotosl/testdata/Compute.golden +++ b/goal/gosl/gotosl/testdata/Compute.golden @@ -7,6 +7,8 @@ var Params: array; @group(0) @binding(1) var Data: array; +alias GPUVars = i32; + @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { Compute(idx.x); @@ -87,6 +89,9 @@ fn ParamStruct_AnotherMeth(ps: ptr, ds: ptr Date: Mon, 4 Nov 2024 03:02:18 -0800 Subject: [PATCH 258/311] GPUInit is safe to multiple calls; README docs updates --- goal/gosl/README.md | 23 ++++++++++++----------- goal/gosl/gotosl/gengpu.go | 4 ++++ gpu/device.go | 2 ++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/goal/gosl/README.md b/goal/gosl/README.md index 3a2b5585f6..abe319ac97 100644 --- a/goal/gosl/README.md +++ b/goal/gosl/README.md @@ -2,7 +2,7 @@ `gosl` implements _Go as a shader language_ for GPU compute shaders (using [WebGPU](https://www.w3.org/TR/webgpu/)), **enabling standard Go code to run on the GPU**. -`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader, using the [gpu](../../gpu) GPU compute shader system. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go language. See the [GPU](../GPU.md) documentation for an overview of issues in GPU computation. The `goal build` command automatically runs `gosl`, which has no effect if there are no `//gosl:` tags in a given directory. +`gosl` converts Go code to WGSL which can then be loaded directly into a WebGPU compute shader, using the [gpu](../../gpu) GPU compute shader system. It operates within the overall [Goal](../README.md) framework of an augmented version of the Go language. See the [GPU](../GPU.md) documentation for an overview of issues in GPU computation. The relevant regions of Go code to be run on the GPU are tagged using the `//gosl:start` and `//gosl:end` comment directives, and this code must only use basic expressions and concrete types that will compile correctly in a GPU shader (see [Restrictions](#restrictions) below). Method functions and pass-by-reference pointer arguments to `struct` types are supported and incur no additional compute cost due to inlining (see notes below for more detail). @@ -10,14 +10,14 @@ See [examples/basic](examples/basic) and [rand](examples/rand) for complete work Although `gosl` is typically run via the `goal build` command, you can also run `gosl` directly. Here's how to install the standalone `gosl` command: ```bash -$ go install cogentcore.org/core/gpu/gosl@latest +$ go install cogentcore.org/core/goal/gosl@latest ``` # Usage There are two critical elements for GPU-enabled code: -1. One or more [Kernel](#kernels) compute functions that take an _index_ argument and perform computations for that specific index of data, _in parallel_. On the GPU, each such kernel is implemented by its own separate compute shader code, and one of the main functions of `gosl` is to generate this code from the Go sources, in the automatically-created `shaders/` directory. +1. One or more [Kernel](#kernels) compute functions that take an _index_ argument and perform computations for that specific index of data, _in parallel_. **GPU computation is effectively just a parallel `for` loop**. On the GPU, each such kernel is implemented by its own separate compute shader code, and one of the main functions of `gosl` is to generate this code from the Go sources, in the automatically created `shaders/` directory. 2. [Global variables](#global-variables) on which the kernel functions _exclusively_ operate: all relevant data must be specifically copied from the CPU to the GPU and back. As explained in the [GPU](../GPU.md) docs, each GPU compute shader is effectively a _standalone_ program operating on these global variables. To replicate this environment on the CPU, so the code is transferrable, we need to make these variables global in the CPU (Go) environment as well. @@ -41,7 +41,7 @@ In the CPU mode, the kernel is effectively run in a `for` loop like this: Compute(uint32(i)) } ``` -A parallel goroutine-based mechanism is actually used, but conceptually this is what it does, on both the CPU and the GPU: **GPU computation is effectively just a parallel for loop**. +A parallel goroutine-based mechanism is actually used, but conceptually this is what it does, on both the CPU and the GPU. To reiterate: **GPU computation is effectively just a parallel for loop**. ## Global variables @@ -62,8 +62,10 @@ var ( ``` All such variables must be either: -1. A `slice` of basic GPU-compatible data values such as `float32`, or GPU-alignment compatible `struct` types, such as `ParamStruct` in the above example. -2. A `tensor` of a GPU-compatible data type (`float32`, `uint32`, or `int32`), with the number of dimensions indicated by the `//gosl:dims ` tag as shown above. +1. A `slice` of GPU-alignment compatible `struct` types, such as `ParamStruct` in the above example. +2. A `tensor` of a GPU-compatible elemental data type (`float32`, `uint32`, or `int32`), with the number of dimensions indicated by the `//gosl:dims ` tag as shown above. + +You can also just declare a slice of elemental GPU-compatible data values such as `float32`, but it is generally preferable to use the tensor instead. ### Tensor data @@ -91,7 +93,6 @@ var ( // Path-level parameters Paths []PathParam - // Unit state values //gosl:group Units Units tensor.Float32 @@ -182,11 +183,9 @@ In general shader code should be simple mathematical expressions and data types, * Cannot use multiple return values, or multiple assignment of variables in a single `=` expression. -* *Can* use multiple variable names with the same type (e.g., `min, max float32`) -- this will be properly converted to the more redundant C form with the type repeated. +* *Can* use multiple variable names with the same type (e.g., `min, max float32`) -- this will be properly converted to the more redundant form with the type repeated, for WGSL. -* `switch` `case` statements are _purely_ self-contained -- no `fallthrough` allowed! does support multiple items per `case` however. - -* TODO: WGSL does not do multi-pass compiling, so all dependent types must be specified *before* being used in other ones, and this also precludes referencing the *current* type within itself. todo: can you just use a forward declaration? +* `switch` `case` statements are _purely_ self-contained -- no `fallthrough` allowed! does support multiple items per `case` however. Every `switch` _must_ have a `default` case. * WGSL does specify that new variables are initialized to 0, like Go, but also somehow discourages that use-case. It is safer to initialize directly: ```Go @@ -194,6 +193,8 @@ In general shader code should be simple mathematical expressions and data types, var val float32 // ok but generally avoid ``` +* A local variable to a global `struct` array variable (e.g., `par := &Params[i]`) can only be created as a function argument. There are special access restrictions that make it impossible to do otherwise. + ## Other language features * [tour-of-wgsl](https://google.github.io/tour-of-wgsl/types/pointers/passing_pointers/) is a good reference to explain things more directly than the spec. diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index 97f39f3fea..92e88f32ab 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -80,7 +80,11 @@ const ( initf := ` // GPUInit initializes the GPU compute system, // configuring system(s), variables and kernels. +// It is safe to call multiple times: detects if already run. func GPUInit() { + if ComputeGPU != nil { + return + } gp := gpu.NewComputeGPU() ComputeGPU = gp ` diff --git a/gpu/device.go b/gpu/device.go index def58d4219..a8cb21d2e1 100644 --- a/gpu/device.go +++ b/gpu/device.go @@ -33,6 +33,8 @@ func NewDevice(gpu *GPU) (*Device, error) { limits.MaxStorageBufferBindingSize = min(gpu.Limits.Limits.MaxStorageBufferBindingSize, maxv) // note: this limit is not working properly: limits.MaxBufferSize = min(gpu.Limits.Limits.MaxBufferSize, maxv) + // limits.MaxBindGroups = gpu.Limits.Limits.MaxBindGroups // note: no point in changing -- web constraint + desc := wgpu.DeviceDescriptor{ RequiredLimits: &wgpu.RequiredLimits{ Limits: limits, From febe06cca0c58a646ce58530270d8484ce8d274c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 5 Nov 2024 21:47:31 -0800 Subject: [PATCH 259/311] plot styler implementation -- seems reasonable --- plot/draw.go | 3 + plot/plotcore/xyplot.go | 10 +- plot/plots/barchart.go | 21 +++- plot/plots/errbars.go | 36 ++++-- plot/plots/labels.go | 11 ++ plot/plots/line.go | 67 ++++++----- plot/plots/plot_test.go | 240 ++++++++++++++++++++++------------------ plot/plots/scatter.go | 50 +++++---- plot/plots/table.go | 10 +- plot/plotter.go | 16 +++ 10 files changed, 285 insertions(+), 179 deletions(-) diff --git a/plot/draw.go b/plot/draw.go index 5d73bb95c6..608fa4e8f9 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -54,6 +54,9 @@ func (pt *Plot) SVGToFile(filename string) error { // drawConfig configures everything for drawing func (pt *Plot) drawConfig() { pt.Resize(pt.Size) // ensure + for _, plt := range pt.Plotters { + plt.ApplyStyle() + } pt.Legend.TextStyle.openFont(pt) pt.Paint.ToDots() } diff --git a/plot/plotcore/xyplot.go b/plot/plotcore/xyplot.go index 7d99de5334..14db52f215 100644 --- a/plot/plotcore/xyplot.go +++ b/plot/plotcore/xyplot.go @@ -119,8 +119,8 @@ func (pl *PlotEditor) genPlotXY() { lns, _ = plots.NewLine(xy) } if lns != nil { - lns.LineStyle.Width.Pt(float32(cp.LineWidth.Or(pl.Options.LineWidth))) - lns.LineStyle.Color = clr + lns.Line.Width.Pt(float32(cp.LineWidth.Or(pl.Options.LineWidth))) + lns.Line.Color = clr lns.NegativeXDraw = pl.Options.NegativeXDraw plt.Add(lns) if pts != nil { @@ -130,8 +130,8 @@ func (pl *PlotEditor) genPlotXY() { } } if pts != nil { - pts.LineStyle.Color = clr - pts.LineStyle.Width.Pt(float32(cp.LineWidth.Or(pl.Options.LineWidth))) + pts.Line.Color = clr + pts.Line.Width.Pt(float32(cp.LineWidth.Or(pl.Options.LineWidth))) pts.PointSize.Pt(float32(cp.PointSize.Or(pl.Options.PointSize))) pts.PointShape = cp.PointShape.Or(pl.Options.PointShape) plt.Add(pts) @@ -144,7 +144,7 @@ func (pl *PlotEditor) genPlotXY() { if ec >= 0 { xy.errColumn = ec eb, _ := plots.NewYErrorBars(xy) - eb.LineStyle.Color = clr + eb.Line.Color = clr plt.Add(eb) } } diff --git a/plot/plots/barchart.go b/plot/plots/barchart.go index 0b20ce7b00..7a55182afe 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/barchart.go @@ -64,9 +64,9 @@ type BarChart struct { // Color is the fill color of the bars. Color image.Image - // LineStyle is the style of the line connecting the points. + // Line is the style of the line connecting the points. // Use zero width to disable lines. - LineStyle plot.LineStyle + Line plot.LineStyle // Horizontal dictates whether the bars should be in the vertical // (default) or horizontal direction. If Horizontal is true, all @@ -76,6 +76,8 @@ type BarChart struct { // stackedOn is the bar chart upon which this bar chart is stacked. StackedOn *BarChart + + stylers plot.Stylers } // NewBarChart returns a new bar chart with a single bar for each value. @@ -108,7 +110,16 @@ func (b *BarChart) Defaults() { b.Width = .8 b.Pad = 1 b.Color = colors.Scheme.OnSurface - b.LineStyle.Defaults() + b.Line.Defaults() +} + +func (b *BarChart) Styler(f func(s *BarChart)) *BarChart { + b.stylers.Add(func(p plot.Plotter) { f(p.(*BarChart)) }) + return b +} + +func (b *BarChart) ApplyStyle() { + b.stylers.Run(b) } func (b *BarChart) XYData() (data plot.XYer, pixels plot.XYer) { @@ -148,7 +159,7 @@ func (b *BarChart) StackOn(on *BarChart) { func (b *BarChart) Plot(plt *plot.Plot) { pc := plt.Paint pc.FillStyle.Color = b.Color - b.LineStyle.SetStroke(plt) + b.Line.SetStroke(plt) nv := len(b.Values) b.XYs = make(plot.XYs, nv) @@ -233,7 +244,7 @@ func (b *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { func (b *BarChart) Thumbnail(plt *plot.Plot) { pc := plt.Paint pc.FillStyle.Color = b.Color - b.LineStyle.SetStroke(plt) + b.Line.SetStroke(plt) ptb := pc.Bounds pc.DrawRectangle(float32(ptb.Min.X), float32(ptb.Min.Y), float32(ptb.Size().X), float32(ptb.Size().Y)) pc.FillStrokeClear() diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 09e305e017..9eb8fe2985 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -58,15 +58,17 @@ type YErrorBars struct { // representing the high, center value of the error bar. PXYs plot.XYs - // LineStyle is the style used to draw the error bars. - LineStyle plot.LineStyle + // Line is the style used to draw the error bars. + Line plot.LineStyle // CapWidth is the width of the caps drawn at the top of each error bar. CapWidth units.Value + + stylers plot.Stylers } func (eb *YErrorBars) Defaults() { - eb.LineStyle.Defaults() + eb.Line.Defaults() eb.CapWidth.Dp(10) } @@ -100,6 +102,14 @@ func NewYErrorBars(yerrs interface { return eb, nil } +// Styler adds a style function to set style parameters. +func (e *YErrorBars) Styler(f func(s *YErrorBars)) *YErrorBars { + e.stylers.Add(func(p plot.Plotter) { f(p.(*YErrorBars)) }) + return e +} + +func (e *YErrorBars) ApplyStyle() { e.stylers.Run(e) } + func (e *YErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { data = e.XYs pixels = e.PXYs @@ -115,7 +125,7 @@ func (e *YErrorBars) Plot(plt *plot.Plot) { cw := 0.5 * e.CapWidth.Dots nv := len(e.YErrors) e.PXYs = make(plot.XYs, nv) - e.LineStyle.SetStroke(plt) + e.Line.SetStroke(plt) for i, err := range e.YErrors { x := plt.PX(e.XYs[i].X) ylow := plt.PY(e.XYs[i].Y - math32.Abs(err.Low)) @@ -165,12 +175,14 @@ type XErrorBars struct { // representing the high, center value of the error bar. PXYs plot.XYs - // LineStyle is the style used to draw the error bars. - LineStyle plot.LineStyle + // Line is the style used to draw the error bars. + Line plot.LineStyle // CapWidth is the width of the caps drawn at the top // of each error bar. CapWidth units.Value + + stylers plot.Stylers } // Returns a new XErrorBars plotter, or an error on failure. The error values @@ -204,10 +216,18 @@ func NewXErrorBars(xerrs interface { } func (eb *XErrorBars) Defaults() { - eb.LineStyle.Defaults() + eb.Line.Defaults() eb.CapWidth.Dp(10) } +// Styler adds a style function to set style parameters. +func (e *XErrorBars) Styler(f func(s *XErrorBars)) *XErrorBars { + e.stylers.Add(func(p plot.Plotter) { f(p.(*XErrorBars)) }) + return e +} + +func (e *XErrorBars) ApplyStyle() { e.stylers.Run(e) } + func (e *XErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { data = e.XYs pixels = e.PXYs @@ -224,7 +244,7 @@ func (e *XErrorBars) Plot(plt *plot.Plot) { nv := len(e.XErrors) e.PXYs = make(plot.XYs, nv) - e.LineStyle.SetStroke(plt) + e.Line.SetStroke(plt) for i, err := range e.XErrors { y := plt.PY(e.XYs[i].Y) xlow := plt.PX(e.XYs[i].X - math32.Abs(err.Low)) diff --git a/plot/plots/labels.go b/plot/plots/labels.go index be431c2665..39b5ade629 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -36,6 +36,7 @@ type Labels struct { // plot size and number of TextStyle when styles last generated -- don't regen styleSize image.Point styleN int + stylers plot.Stylers } // NewLabels returns a new Labels using defaults @@ -66,6 +67,16 @@ func NewLabels(d XYLabeler) (*Labels, error) { }, nil } +// Styler adds a style function to set style parameters. +func (l *Labels) Styler(f func(s *Labels)) *Labels { + l.stylers.Add(func(p plot.Plotter) { f(p.(*Labels)) }) + return l +} + +func (l *Labels) ApplyStyle() { + l.stylers.Run(l) +} + func (l *Labels) XYData() (data plot.XYer, pixels plot.XYer) { data = l.XYs pixels = l.PXYs diff --git a/plot/plots/line.go b/plot/plots/line.go index f1697e3691..277aa4d26e 100644 --- a/plot/plots/line.go +++ b/plot/plots/line.go @@ -14,6 +14,7 @@ package plots import ( "image" + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" ) @@ -47,9 +48,9 @@ type Line struct { // StepStyle is the kind of the step line. StepStyle StepKind - // LineStyle is the style of the line connecting the points. + // Line is the style of the line connecting the points. // Use zero width to disable lines. - LineStyle plot.LineStyle + Line plot.LineStyle // Fill is the color to fill the area below the plot. // Use nil to disable filling, which is the default. @@ -60,18 +61,20 @@ type Line struct { // default is false, so that repeated series of data across the X axis // are plotted separately. NegativeXDraw bool + + stylers plot.Stylers } // NewLine returns a Line that uses the default line style and // does not draw glyphs. -func NewLine(xys plot.XYer) (*Line, error) { +func NewLine(xys plot.XYer) *Line { data, err := plot.CopyXYs(xys) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } ln := &Line{XYs: data} ln.Defaults() - return ln, nil + return ln } // NewLinePoints returns both a Line and a @@ -86,32 +89,40 @@ func NewLinePoints(xys plot.XYer) (*Line, *Scatter, error) { return ln, sc, nil } -func (pts *Line) Defaults() { - pts.LineStyle.Defaults() +func (ln *Line) Defaults() { + ln.Line.Defaults() } -func (pts *Line) XYData() (data plot.XYer, pixels plot.XYer) { - data = pts.XYs - pixels = pts.PXYs +// Styler adds a style function to set style parameters. +func (ln *Line) Styler(f func(s *Line)) *Line { + ln.stylers.Add(func(p plot.Plotter) { f(p.(*Line)) }) + return ln +} + +func (ln *Line) ApplyStyle() { ln.stylers.Run(ln) } + +func (ln *Line) XYData() (data plot.XYer, pixels plot.XYer) { + data = ln.XYs + pixels = ln.PXYs return } // Plot draws the Line, implementing the plot.Plotter interface. -func (pts *Line) Plot(plt *plot.Plot) { +func (ln *Line) Plot(plt *plot.Plot) { pc := plt.Paint - ps := plot.PlotXYs(plt, pts.XYs) + ps := plot.PlotXYs(plt, ln.XYs) np := len(ps) - pts.PXYs = ps + ln.PXYs = ps - if pts.Fill != nil { - pc.FillStyle.Color = pts.Fill + if ln.Fill != nil { + pc.FillStyle.Color = ln.Fill minY := plt.PY(plt.Y.Min) prev := math32.Vec2(ps[0].X, minY) pc.MoveTo(prev.X, prev.Y) for i := range ps { pt := ps[i] - switch pts.StepStyle { + switch ln.StepStyle { case NoStep: if pt.X < prev.X { pc.LineTo(prev.X, minY) @@ -159,16 +170,16 @@ func (pts *Line) Plot(plt *plot.Plot) { } pc.FillStyle.Color = nil - if !pts.LineStyle.SetStroke(plt) { + if !ln.Line.SetStroke(plt) { return } prev := ps[0] pc.MoveTo(prev.X, prev.Y) for i := 1; i < np; i++ { pt := ps[i] - if pts.StepStyle != NoStep { + if ln.StepStyle != NoStep { if pt.X >= prev.X { - switch pts.StepStyle { + switch ln.StepStyle { case PreStep: pc.LineTo(prev.X, pt.Y) case MidStep: @@ -181,7 +192,7 @@ func (pts *Line) Plot(plt *plot.Plot) { pc.MoveTo(pt.X, pt.Y) } } - if !pts.NegativeXDraw && pt.X < prev.X { + if !ln.NegativeXDraw && pt.X < prev.X { pc.MoveTo(pt.X, pt.Y) } else { pc.LineTo(pt.X, pt.Y) @@ -193,25 +204,25 @@ func (pts *Line) Plot(plt *plot.Plot) { // DataRange returns the minimum and maximum // x and y values, implementing the plot.DataRanger interface. -func (pts *Line) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - return plot.XYRange(pts) +func (ln *Line) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { + return plot.XYRange(ln) } // Thumbnail returns the thumbnail for the LineTo, implementing the plot.Thumbnailer interface. -func (pts *Line) Thumbnail(plt *plot.Plot) { +func (ln *Line) Thumbnail(plt *plot.Plot) { pc := plt.Paint ptb := pc.Bounds midY := 0.5 * float32(ptb.Min.Y+ptb.Max.Y) - if pts.Fill != nil { + if ln.Fill != nil { tb := ptb - if pts.LineStyle.Width.Value > 0 { + if ln.Line.Width.Value > 0 { tb.Min.Y = int(midY) } - pc.FillBox(math32.FromPoint(tb.Min), math32.FromPoint(tb.Size()), pts.Fill) + pc.FillBox(math32.FromPoint(tb.Min), math32.FromPoint(tb.Size()), ln.Fill) } - if pts.LineStyle.SetStroke(plt) { + if ln.Line.SetStroke(plt) { pc.MoveTo(float32(ptb.Min.X), midY) pc.LineTo(float32(ptb.Max.X), midY) pc.Stroke() diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index c85f75020d..3341e099d3 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -17,20 +17,42 @@ import ( "cogentcore.org/core/plot" ) +func ExampleLine() { + data := make(plot.XYs, 42) + for i := range data { + x := float32(i % 21) + data[i].X = x * 5 + if i < 21 { + data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) + } else { + data[i].Y = float32(50) + 40*math32.Cos((x/8)*math32.Pi) + } + } + + plt := plot.New() + plt.Add(NewLine(data).Styler(func(ln *Line) { + ln.Line.Color = colors.Uniform(colors.Red) + ln.Line.Width.Pt(2) + })) + plt.Draw() + imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") + // Output: +} + func TestMain(m *testing.M) { paint.FontLibrary.InitFontPaths(paint.FontPaths...) os.Exit(m.Run()) } func TestLine(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Line" - pt.X.Min = 0 - pt.X.Max = 100 - pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Line" + plt.X.Min = 0 + plt.X.Max = 100 + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" // note: making two overlapping series data := make(plot.XYs, 42) @@ -44,51 +66,51 @@ func TestLine(t *testing.T) { } } - l1, err := NewLine(data) - if err != nil { - t.Error(err.Error()) + l1 := NewLine(data) + if l1 == nil { + t.Error("bad data") } - pt.Add(l1) - pt.Legend.Add("Sine", l1) - pt.Legend.Add("Cos", l1) + plt.Add(l1) + plt.Legend.Add("Sine", l1) + plt.Legend.Add("Cos", l1) - pt.Resize(image.Point{640, 480}) - pt.Draw() - imagex.Assert(t, pt.Pixels, "line.png") + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "line.png") l1.Fill = colors.Uniform(colors.Yellow) - pt.Draw() - imagex.Assert(t, pt.Pixels, "line-fill.png") + plt.Draw() + imagex.Assert(t, plt.Pixels, "line-fill.png") l1.StepStyle = PreStep - pt.Draw() - imagex.Assert(t, pt.Pixels, "line-prestep.png") + plt.Draw() + imagex.Assert(t, plt.Pixels, "line-prestep.png") l1.StepStyle = MidStep - pt.Draw() - imagex.Assert(t, pt.Pixels, "line-midstep.png") + plt.Draw() + imagex.Assert(t, plt.Pixels, "line-midstep.png") l1.StepStyle = PostStep - pt.Draw() - imagex.Assert(t, pt.Pixels, "line-poststep.png") + plt.Draw() + imagex.Assert(t, plt.Pixels, "line-poststep.png") l1.StepStyle = NoStep l1.Fill = nil l1.NegativeXDraw = true - pt.Draw() - imagex.Assert(t, pt.Pixels, "line-negx.png") + plt.Draw() + imagex.Assert(t, plt.Pixels, "line-negx.png") } func TestScatter(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Scatter" - pt.X.Min = 0 - pt.X.Max = 100 - pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Scatter" + plt.X.Min = 0 + plt.X.Max = 100 + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" data := make(plot.XYs, 21) for i := range data { @@ -100,23 +122,23 @@ func TestScatter(t *testing.T) { if err != nil { t.Error(err.Error()) } - pt.Add(l1) + plt.Add(l1) - pt.Resize(image.Point{640, 480}) + plt.Resize(image.Point{640, 480}) shs := ShapesValues() for _, sh := range shs { l1.PointShape = sh - pt.Draw() - imagex.Assert(t, pt.Pixels, "scatter-"+sh.String()+".png") + plt.Draw() + imagex.Assert(t, plt.Pixels, "scatter-"+sh.String()+".png") } } func TestLabels(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Labels" - pt.X.Label.Text = "X Axis" - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Labels" + plt.X.Label.Text = "X Axis" + plt.Y.Label.Text = "Y Axis" // note: making two overlapping series data := make(plot.XYs, 12) @@ -132,9 +154,9 @@ func TestLabels(t *testing.T) { if err != nil { t.Error(err.Error()) } - pt.Add(l1) - pt.Add(sc) - pt.Legend.Add("Sine", l1, sc) + plt.Add(l1) + plt.Add(sc) + plt.Legend.Add("Sine", l1, sc) l2, err := NewLabels(XYLabels{XYs: data, Labels: labels}) if err != nil { @@ -142,20 +164,20 @@ func TestLabels(t *testing.T) { } l2.Offset.X.Dp(6) l2.Offset.Y.Dp(-6) - pt.Add(l2) + plt.Add(l2) - pt.Resize(image.Point{640, 480}) - pt.Draw() - imagex.Assert(t, pt.Pixels, "labels.png") + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "labels.png") } func TestBarChart(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Bar Chart" - pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Bar Chart" + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" data := make(plot.Values, 21) for i := range data { @@ -174,36 +196,36 @@ func TestBarChart(t *testing.T) { t.Error(err.Error()) } l1.Color = colors.Uniform(colors.Red) - pt.Add(l1) - pt.Legend.Add("Sine", l1) + plt.Add(l1) + plt.Legend.Add("Sine", l1) - pt.Resize(image.Point{640, 480}) - pt.Draw() - imagex.Assert(t, pt.Pixels, "bar.png") + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "bar.png") l2, err := NewBarChart(cos, nil) if err != nil { t.Error(err.Error()) } l2.Color = colors.Uniform(colors.Blue) - pt.Legend.Add("Cosine", l2) + plt.Legend.Add("Cosine", l2) l1.Stride = 2 l2.Stride = 2 l2.Offset = 2 - pt.Add(l2) // note: range updated when added! - pt.Draw() - imagex.Assert(t, pt.Pixels, "bar-cos.png") + plt.Add(l2) // note: range updated when added! + plt.Draw() + imagex.Assert(t, plt.Pixels, "bar-cos.png") } func TestBarChartErr(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Bar Chart Errors" - pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Bar Chart Errors" + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" data := make(plot.Values, 21) for i := range data { @@ -222,28 +244,28 @@ func TestBarChartErr(t *testing.T) { t.Error(err.Error()) } l1.Color = colors.Uniform(colors.Red) - pt.Add(l1) - pt.Legend.Add("Sine", l1) + plt.Add(l1) + plt.Legend.Add("Sine", l1) - pt.Resize(image.Point{640, 480}) - pt.Draw() - imagex.Assert(t, pt.Pixels, "bar-err.png") + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "bar-err.png") l1.Horizontal = true - pt.UpdateRange() - pt.X.Min = 0 - pt.X.Max = 100 - pt.Draw() - imagex.Assert(t, pt.Pixels, "bar-err-horiz.png") + plt.UpdateRange() + plt.X.Min = 0 + plt.X.Max = 100 + plt.Draw() + imagex.Assert(t, plt.Pixels, "bar-err-horiz.png") } func TestBarChartStack(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Bar Chart Stacked" - pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Bar Chart Stacked" + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" data := make(plot.Values, 21) for i := range data { @@ -262,8 +284,8 @@ func TestBarChartStack(t *testing.T) { t.Error(err.Error()) } l1.Color = colors.Uniform(colors.Red) - pt.Add(l1) - pt.Legend.Add("Sine", l1) + plt.Add(l1) + plt.Legend.Add("Sine", l1) l2, err := NewBarChart(cos, nil) if err != nil { @@ -271,12 +293,12 @@ func TestBarChartStack(t *testing.T) { } l2.Color = colors.Uniform(colors.Blue) l2.StackedOn = l1 - pt.Add(l2) - pt.Legend.Add("Cos", l2) + plt.Add(l2) + plt.Legend.Add("Cos", l2) - pt.Resize(image.Point{640, 480}) - pt.Draw() - imagex.Assert(t, pt.Pixels, "bar-stacked.png") + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "bar-stacked.png") } type XYErr struct { @@ -285,12 +307,12 @@ type XYErr struct { } func TestErrBar(t *testing.T) { - pt := plot.New() - pt.Title.Text = "Test Line Errors" - pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 - pt.Y.Label.Text = "Y Axis" + plt := plot.New() + plt.Title.Text = "Test Line Errors" + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" data := make(plot.XYs, 21) for i := range data { @@ -308,20 +330,20 @@ func TestErrBar(t *testing.T) { xyerr := XYErr{XYs: data, YErrors: yerr} - l1, err := NewLine(data) - if err != nil { - t.Error(err.Error()) + l1 := NewLine(data) + if l1 == nil { + t.Error("bad data") } - pt.Add(l1) - pt.Legend.Add("Sine", l1) + plt.Add(l1) + plt.Legend.Add("Sine", l1) l2, err := NewYErrorBars(xyerr) if err != nil { t.Error(err.Error()) } - pt.Add(l2) + plt.Add(l2) - pt.Resize(image.Point{640, 480}) - pt.Draw() - imagex.Assert(t, pt.Pixels, "errbar.png") + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "errbar.png") } diff --git a/plot/plots/scatter.go b/plot/plots/scatter.go index bfd5cdf754..4425dac548 100644 --- a/plot/plots/scatter.go +++ b/plot/plots/scatter.go @@ -30,9 +30,12 @@ type Scatter struct { // shape to draw for each point PointShape Shapes - // LineStyle is the style of the line connecting the points. + // Line is the style of the line connecting the points. // Use zero width to disable lines. - LineStyle plot.LineStyle + Line plot.LineStyle + + // Stylers are the styler functions. + Stylers plot.Stylers } // NewScatter returns a Scatter that uses the @@ -43,51 +46,60 @@ func NewScatter(xys plot.XYer) (*Scatter, error) { return nil, err } sc := &Scatter{XYs: data} - sc.LineStyle.Defaults() + sc.Line.Defaults() sc.PointSize.Pt(4) return sc, nil } -func (pts *Scatter) XYData() (data plot.XYer, pixels plot.XYer) { - data = pts.XYs - pixels = pts.PXYs +func (sc *Scatter) Style(f func(s *Scatter)) *Scatter { + sc.Stylers.Add(func(p plot.Plotter) { f(p.(*Scatter)) }) + return sc +} + +func (sc *Scatter) ApplyStyle() { + sc.Stylers.Run(sc) +} + +func (sc *Scatter) XYData() (data plot.XYer, pixels plot.XYer) { + data = sc.XYs + pixels = sc.PXYs return } // Plot draws the Line, implementing the plot.Plotter interface. -func (pts *Scatter) Plot(plt *plot.Plot) { +func (sc *Scatter) Plot(plt *plot.Plot) { pc := plt.Paint - if !pts.LineStyle.SetStroke(plt) { + if !sc.Line.SetStroke(plt) { return } - pts.PointSize.ToDots(&pc.UnitContext) - pc.FillStyle.Color = pts.LineStyle.Color - ps := plot.PlotXYs(plt, pts.XYs) + sc.PointSize.ToDots(&pc.UnitContext) + pc.FillStyle.Color = sc.Line.Color + ps := plot.PlotXYs(plt, sc.XYs) for i := range ps { pt := ps[i] - DrawShape(pc, math32.Vec2(pt.X, pt.Y), pts.PointSize.Dots, pts.PointShape) + DrawShape(pc, math32.Vec2(pt.X, pt.Y), sc.PointSize.Dots, sc.PointShape) } pc.FillStyle.Color = nil } // DataRange returns the minimum and maximum // x and y values, implementing the plot.DataRanger interface. -func (pts *Scatter) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - return plot.XYRange(pts) +func (sc *Scatter) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { + return plot.XYRange(sc) } // Thumbnail the thumbnail for the Scatter, // implementing the plot.Thumbnailer interface. -func (pts *Scatter) Thumbnail(plt *plot.Plot) { - if !pts.LineStyle.SetStroke(plt) { +func (sc *Scatter) Thumbnail(plt *plot.Plot) { + if !sc.Line.SetStroke(plt) { return } pc := plt.Paint - pts.PointSize.ToDots(&pc.UnitContext) - pc.FillStyle.Color = pts.LineStyle.Color + sc.PointSize.ToDots(&pc.UnitContext) + pc.FillStyle.Color = sc.Line.Color ptb := pc.Bounds midX := 0.5 * float32(ptb.Min.X+ptb.Max.X) midY := 0.5 * float32(ptb.Min.Y+ptb.Max.Y) - DrawShape(pc, math32.Vec2(midX, midY), pts.PointSize.Dots, pts.PointShape) + DrawShape(pc, math32.Vec2(midX, midY), sc.PointSize.Dots, sc.PointShape) } diff --git a/plot/plots/table.go b/plot/plots/table.go index 484b66a534..0d3856d7c8 100644 --- a/plot/plots/table.go +++ b/plot/plots/table.go @@ -53,14 +53,14 @@ func (dt *TableXYer) XY(i int) (x, y float32) { } // AddTableLine adds Line with given x, y columns from given tabular data -func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) (*Line, error) { +func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *Line { txy := NewTableXYer(tab, xcolumn, ycolumn) - ln, err := NewLine(txy) - if err != nil { - return nil, err + ln := NewLine(txy) + if ln == nil { + return nil } plt.Add(ln) - return ln, nil + return ln } // AddTableLinePoints adds Line w/ Points with given x, y columns from given tabular data diff --git a/plot/plotter.go b/plot/plotter.go index a656a3f808..e98fdcb0d5 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -14,6 +14,9 @@ type Plotter interface { // including corresponding pixel data. // This allows gui interface to inspect data etc. XYData() (data XYer, pixels XYer) + + // ApplyStyle runs the style functions. + ApplyStyle() } // DataRanger wraps the DataRange method. @@ -21,3 +24,16 @@ type DataRanger interface { // DataRange returns the range of X and Y values. DataRange(pt *Plot) (xmin, xmax, ymin, ymax float32) } + +// Stylers implements the styling functions for plotters. +type Stylers []func(p Plotter) + +func (st *Stylers) Add(f func(p Plotter)) { + *st = append(*st, f) +} + +func (st *Stylers) Run(p Plotter) { + for _, f := range *st { + f(p) + } +} From fd42b320df87b90ef871d2750b5a1475e17a54fb Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 5 Nov 2024 22:10:05 -0800 Subject: [PATCH 260/311] plots: new functions don't return err so direct Add works --- plot/plots/barchart.go | 13 +++++----- plot/plots/errbars.go | 41 ++++++++++++++++--------------- plot/plots/labels.go | 13 +++++----- plot/plots/line.go | 9 +++---- plot/plots/plot_test.go | 54 ++++++++++++++++++++--------------------- plot/plots/scatter.go | 16 ++++++------ plot/plots/table.go | 10 ++++---- 7 files changed, 78 insertions(+), 78 deletions(-) diff --git a/plot/plots/barchart.go b/plot/plots/barchart.go index 7a55182afe..36141c0a5c 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/barchart.go @@ -14,6 +14,7 @@ package plots import ( "image" + "cogentcore.org/core/base/errors" "cogentcore.org/core/colors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" @@ -84,16 +85,16 @@ type BarChart struct { // The bars heights correspond to the values and their x locations correspond // to the index of their value in the Valuer. Optional error-bar values can be // provided. -func NewBarChart(vs, ers plot.Valuer) (*BarChart, error) { +func NewBarChart(vs, ers plot.Valuer) *BarChart { values, err := plot.CopyValues(vs) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } var errs plot.Values if ers != nil { errs, err = plot.CopyValues(ers) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } } b := &BarChart{ @@ -101,7 +102,7 @@ func NewBarChart(vs, ers plot.Valuer) (*BarChart, error) { Errors: errs, } b.Defaults() - return b, nil + return b } func (b *BarChart) Defaults() { diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 9eb8fe2985..481eb8fadd 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -5,6 +5,7 @@ package plots import ( + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" "cogentcore.org/core/styles/units" @@ -80,26 +81,26 @@ func (eb *YErrorBars) Defaults() { func NewYErrorBars(yerrs interface { plot.XYer YErrorer -}) (*YErrorBars, error) { +}) *YErrorBars { - errors := make(YErrors, yerrs.Len()) - for i := range errors { - errors[i].Low, errors[i].High = yerrs.YError(i) - if err := plot.CheckFloats(errors[i].Low, errors[i].High); err != nil { - return nil, err + errs := make(YErrors, yerrs.Len()) + for i := range errs { + errs[i].Low, errs[i].High = yerrs.YError(i) + if err := plot.CheckFloats(errs[i].Low, errs[i].High); errors.Log(err) != nil { + return nil } } xys, err := plot.CopyXYs(yerrs) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } eb := &YErrorBars{ XYs: xys, - YErrors: errors, + YErrors: errs, } eb.Defaults() - return eb, nil + return eb } // Styler adds a style function to set style parameters. @@ -193,26 +194,26 @@ type XErrorBars struct { func NewXErrorBars(xerrs interface { plot.XYer XErrorer -}) (*XErrorBars, error) { +}) *XErrorBars { - errors := make(XErrors, xerrs.Len()) - for i := range errors { - errors[i].Low, errors[i].High = xerrs.XError(i) - if err := plot.CheckFloats(errors[i].Low, errors[i].High); err != nil { - return nil, err + errs := make(XErrors, xerrs.Len()) + for i := range errs { + errs[i].Low, errs[i].High = xerrs.XError(i) + if err := plot.CheckFloats(errs[i].Low, errs[i].High); errors.Log(err) != nil { + return nil } } xys, err := plot.CopyXYs(xerrs) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } eb := &XErrorBars{ XYs: xys, - XErrors: errors, + XErrors: errs, } eb.Defaults() - return eb, nil + return eb } func (eb *XErrorBars) Defaults() { diff --git a/plot/plots/labels.go b/plot/plots/labels.go index 39b5ade629..20ce2badd7 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -5,9 +5,9 @@ package plots import ( - "errors" "image" + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" "cogentcore.org/core/styles/units" @@ -40,14 +40,15 @@ type Labels struct { } // NewLabels returns a new Labels using defaults -func NewLabels(d XYLabeler) (*Labels, error) { +func NewLabels(d XYLabeler) *Labels { xys, err := plot.CopyXYs(d) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } if d.Len() != len(xys) { - return nil, errors.New("plotter: number of points does not match the number of labels") + errors.Log(errors.New("plotter: number of points does not match the number of labels")) + return nil } strs := make([]string, d.Len()) @@ -64,7 +65,7 @@ func NewLabels(d XYLabeler) (*Labels, error) { XYs: xys, Labels: strs, TextStyle: styles, - }, nil + } } // Styler adds a style function to set style parameters. diff --git a/plot/plots/line.go b/plot/plots/line.go index 277aa4d26e..db653e5ebe 100644 --- a/plot/plots/line.go +++ b/plot/plots/line.go @@ -79,14 +79,11 @@ func NewLine(xys plot.XYer) *Line { // NewLinePoints returns both a Line and a // Scatter plot for the given point data. -func NewLinePoints(xys plot.XYer) (*Line, *Scatter, error) { - sc, err := NewScatter(xys) - if err != nil { - return nil, nil, err - } +func NewLinePoints(xys plot.XYer) (*Line, *Scatter) { + sc := NewScatter(xys) ln := &Line{XYs: sc.XYs} ln.Defaults() - return ln, sc, nil + return ln, sc } func (ln *Line) Defaults() { diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 3341e099d3..76ad1633b6 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -118,9 +118,9 @@ func TestScatter(t *testing.T) { data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) } - l1, err := NewScatter(data) - if err != nil { - t.Error(err.Error()) + l1 := NewScatter(data) + if l1 == nil { + t.Error("bad data") } plt.Add(l1) @@ -150,17 +150,17 @@ func TestLabels(t *testing.T) { labels[i] = fmt.Sprintf("%7.4g", data[i].Y) } - l1, sc, err := NewLinePoints(data) - if err != nil { - t.Error(err.Error()) + l1, sc := NewLinePoints(data) + if l1 == nil || sc == nil { + t.Error("bad data") } plt.Add(l1) plt.Add(sc) plt.Legend.Add("Sine", l1, sc) - l2, err := NewLabels(XYLabels{XYs: data, Labels: labels}) - if err != nil { - t.Error(err.Error()) + l2 := NewLabels(XYLabels{XYs: data, Labels: labels}) + if l2 == nil { + t.Error("bad data") } l2.Offset.X.Dp(6) l2.Offset.Y.Dp(-6) @@ -191,9 +191,9 @@ func TestBarChart(t *testing.T) { cos[i] = float32(50) + 40*math32.Cos((x/8)*math32.Pi) } - l1, err := NewBarChart(data, nil) - if err != nil { - t.Error(err.Error()) + l1 := NewBarChart(data, nil) + if l1 == nil { + t.Error("bad data") } l1.Color = colors.Uniform(colors.Red) plt.Add(l1) @@ -203,9 +203,9 @@ func TestBarChart(t *testing.T) { plt.Draw() imagex.Assert(t, plt.Pixels, "bar.png") - l2, err := NewBarChart(cos, nil) - if err != nil { - t.Error(err.Error()) + l2 := NewBarChart(cos, nil) + if l2 == nil { + t.Error("bad data") } l2.Color = colors.Uniform(colors.Blue) plt.Legend.Add("Cosine", l2) @@ -239,9 +239,9 @@ func TestBarChartErr(t *testing.T) { cos[i] = float32(5) + 4*math32.Cos((x/8)*math32.Pi) } - l1, err := NewBarChart(data, cos) - if err != nil { - t.Error(err.Error()) + l1 := NewBarChart(data, cos) + if l1 == nil { + t.Error("bad data") } l1.Color = colors.Uniform(colors.Red) plt.Add(l1) @@ -279,17 +279,17 @@ func TestBarChartStack(t *testing.T) { cos[i] = float32(5) + 4*math32.Cos((x/8)*math32.Pi) } - l1, err := NewBarChart(data, nil) - if err != nil { - t.Error(err.Error()) + l1 := NewBarChart(data, nil) + if l1 == nil { + t.Error("bad data") } l1.Color = colors.Uniform(colors.Red) plt.Add(l1) plt.Legend.Add("Sine", l1) - l2, err := NewBarChart(cos, nil) - if err != nil { - t.Error(err.Error()) + l2 := NewBarChart(cos, nil) + if l2 == nil { + t.Error("bad data") } l2.Color = colors.Uniform(colors.Blue) l2.StackedOn = l1 @@ -337,9 +337,9 @@ func TestErrBar(t *testing.T) { plt.Add(l1) plt.Legend.Add("Sine", l1) - l2, err := NewYErrorBars(xyerr) - if err != nil { - t.Error(err.Error()) + l2 := NewYErrorBars(xyerr) + if l2 == nil { + t.Error("bad data") } plt.Add(l2) diff --git a/plot/plots/scatter.go b/plot/plots/scatter.go index 4425dac548..400896e024 100644 --- a/plot/plots/scatter.go +++ b/plot/plots/scatter.go @@ -10,6 +10,7 @@ package plots import ( + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" "cogentcore.org/core/styles/units" @@ -34,30 +35,29 @@ type Scatter struct { // Use zero width to disable lines. Line plot.LineStyle - // Stylers are the styler functions. - Stylers plot.Stylers + stylers plot.Stylers } // NewScatter returns a Scatter that uses the // default glyph style. -func NewScatter(xys plot.XYer) (*Scatter, error) { +func NewScatter(xys plot.XYer) *Scatter { data, err := plot.CopyXYs(xys) - if err != nil { - return nil, err + if errors.Log(err) != nil { + return nil } sc := &Scatter{XYs: data} sc.Line.Defaults() sc.PointSize.Pt(4) - return sc, nil + return sc } func (sc *Scatter) Style(f func(s *Scatter)) *Scatter { - sc.Stylers.Add(func(p plot.Plotter) { f(p.(*Scatter)) }) + sc.stylers.Add(func(p plot.Plotter) { f(p.(*Scatter)) }) return sc } func (sc *Scatter) ApplyStyle() { - sc.Stylers.Run(sc) + sc.stylers.Run(sc) } func (sc *Scatter) XYData() (data plot.XYer, pixels plot.XYer) { diff --git a/plot/plots/table.go b/plot/plots/table.go index 0d3856d7c8..6a24918052 100644 --- a/plot/plots/table.go +++ b/plot/plots/table.go @@ -64,13 +64,13 @@ func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *Line { } // AddTableLinePoints adds Line w/ Points with given x, y columns from given tabular data -func AddTableLinePoints(plt *plot.Plot, tab Table, xcolumn, ycolumn int) (*Line, *Scatter, error) { +func AddTableLinePoints(plt *plot.Plot, tab Table, xcolumn, ycolumn int) (*Line, *Scatter) { txy := &TableXYer{Table: tab, XColumn: xcolumn, YColumn: ycolumn} - ln, sc, err := NewLinePoints(txy) - if err != nil { - return nil, nil, err + ln, sc := NewLinePoints(txy) + if ln == nil || sc == nil { + return nil, nil } plt.Add(ln) plt.Add(sc) - return ln, sc, nil + return ln, sc } From 844be5af4a2b9468193c53d65e9aabc6787ddee8 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 5 Nov 2024 23:02:00 -0800 Subject: [PATCH 261/311] plots: gearing up to test goal api for plotting using planets data --- base/fileinfo/mimetype.go | 2 -- goal/interpreter/config.go | 2 +- plot/data.go | 27 +++++++++++++++++++ plot/plotcore/barplot.go | 12 ++++----- plot/plotcore/xyplot.go | 10 +++---- plot/plots/barchart.go | 17 ++++++++++-- plot/plots/line.go | 7 +++++ tensor/examples/planets/planets.go | 3 +++ .../symbols/cogentcore_org-core-plot-plots.go | 2 ++ yaegicore/symbols/cogentcore_org-core-plot.go | 11 +++++--- 10 files changed, 73 insertions(+), 20 deletions(-) diff --git a/base/fileinfo/mimetype.go b/base/fileinfo/mimetype.go index 4d44d2bc7e..f0529b8916 100644 --- a/base/fileinfo/mimetype.go +++ b/base/fileinfo/mimetype.go @@ -12,7 +12,6 @@ import ( "regexp" "strings" - "cogentcore.org/core/base/errors" "github.com/h2non/filetype" ) @@ -107,7 +106,6 @@ var generatedRe = regexp.MustCompile(`^// Code generated .* DO NOT EDIT`) func IsGeneratedFile(fname string) bool { file, err := os.Open(fname) if err != nil { - errors.Log(err) return false } head := make([]byte, 2048) diff --git a/goal/interpreter/config.go b/goal/interpreter/config.go index 9312d34714..eb7dda9f7b 100644 --- a/goal/interpreter/config.go +++ b/goal/interpreter/config.go @@ -56,7 +56,6 @@ type Config struct { // it runs an interactive shell that allows the user to input goal. func Run(c *Config) error { //cli:cmd -root in := NewInterpreter(interp.Options{}) - in.Config() if len(c.Args) > 0 { in.Eval("args := goalib.StringsToAnys(" + fmt.Sprintf("%#v)", c.Args)) } @@ -91,6 +90,7 @@ func Run(c *Config) error { //cli:cmd -root // Interactive runs an interactive shell that allows the user to input goal. func Interactive(c *Config, in *Interpreter) error { + in.Config() if c.Expr != "" { in.Eval(c.Expr) } diff --git a/plot/data.go b/plot/data.go index adf8b662ff..ad51b9b10c 100644 --- a/plot/data.go +++ b/plot/data.go @@ -13,6 +13,7 @@ import ( "errors" "cogentcore.org/core/math32" + "cogentcore.org/core/tensor" ) // data defines the main data interfaces for plotting. @@ -91,6 +92,19 @@ func (vs Values) Value(i int) float32 { return vs[i] } +// TensorValues provides a Valuer interface wrapper for a tensor. +type TensorValues struct { + tensor.Tensor +} + +func (vs TensorValues) Len() int { + return vs.Tensor.Len() +} + +func (vs TensorValues) Value(i int) float32 { + return float32(vs.Tensor.Float1D(i)) +} + // CopyValues returns a Values that is a copy of the values // from a Valuer, or an error if there are no values, or if one of // the copied values is a Infinity. @@ -144,6 +158,19 @@ func (xys XYs) XY(i int) (float32, float32) { return xys[i].X, xys[i].Y } +// TensorXYs provides a XYer interface wrapper for a tensor. +type TensorXYs struct { + X, Y tensor.Tensor +} + +func (xys TensorXYs) Len() int { + return xys.X.Len() +} + +func (xys TensorXYs) XY(i int) (float32, float32) { + return float32(xys.X.Float1D(i)), float32(xys.Y.Float1D(i)) +} + // CopyXYs returns an XYs that is a copy of the x and y values from // an XYer, or an error if one of the data points contains a NaN or // Infinity. diff --git a/plot/plotcore/barplot.go b/plot/plotcore/barplot.go index f1ad505375..ba20f1a0df 100644 --- a/plot/plotcore/barplot.go +++ b/plot/plotcore/barplot.go @@ -130,15 +130,13 @@ func (pl *PlotEditor) genPlotBar() { var bar *plots.BarChart if ec >= 0 { exy, _ := newTableXY(lview, ec, 0, ec, 0, minmax.Range32{}) - bar, err = plots.NewBarChart(xy, exy) - if err != nil { - // log.Println(err) + bar = plots.NewBarChart(xy, exy) + if bar == nil { continue } } else { - bar, err = plots.NewBarChart(xy, nil) - if err != nil { - // log.Println(err) + bar = plots.NewBarChart(xy, nil) + if bar == nil { continue } } @@ -176,7 +174,7 @@ func (pl *PlotEditor) genPlotBar() { xyl.XYs[i] = math32.Vec2(x, y) xyl.Labels[i] = xy.Label(i) } - lbls, _ := plots.NewLabels(xyl) + lbls := plots.NewLabels(xyl) if lbls != nil { plt.Add(lbls) } diff --git a/plot/plotcore/xyplot.go b/plot/plotcore/xyplot.go index 14db52f215..9050f15a00 100644 --- a/plot/plotcore/xyplot.go +++ b/plot/plotcore/xyplot.go @@ -112,11 +112,11 @@ func (pl *PlotEditor) genPlotXY() { lbl = fmt.Sprintf("%s_%02d", lbl, idx) } if cp.Lines.Or(pl.Options.Lines) && cp.Points.Or(pl.Options.Points) { - lns, pts, _ = plots.NewLinePoints(xy) + lns, pts = plots.NewLinePoints(xy) } else if cp.Points.Or(pl.Options.Points) { - pts, _ = plots.NewScatter(xy) + pts = plots.NewScatter(xy) } else { - lns, _ = plots.NewLine(xy) + lns = plots.NewLine(xy) } if lns != nil { lns.Line.Width.Pt(float32(cp.LineWidth.Or(pl.Options.LineWidth))) @@ -143,7 +143,7 @@ func (pl *PlotEditor) genPlotXY() { ec := pl.table.ColumnIndex(cp.ErrColumn) if ec >= 0 { xy.errColumn = ec - eb, _ := plots.NewYErrorBars(xy) + eb := plots.NewYErrorBars(xy) eb.Line.Color = clr plt.Add(eb) } @@ -157,7 +157,7 @@ func (pl *PlotEditor) genPlotXY() { xy, _ := newTableXY(xview, xi, xp.TensorIndex, firstXY.yColumn, cp.TensorIndex, firstXY.yRange) xy.labelColumn = xview.ColumnIndex(cp.Column) xy.yIndex = firstXY.yIndex - lbls, _ := plots.NewLabels(xy) + lbls := plots.NewLabels(xy) if lbls != nil { plt.Add(lbls) } diff --git a/plot/plots/barchart.go b/plot/plots/barchart.go index 36141c0a5c..674dfabd3a 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/barchart.go @@ -18,6 +18,7 @@ import ( "cogentcore.org/core/colors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" + "cogentcore.org/core/tensor" ) // A BarChart presents ordinally-organized data with rectangular bars @@ -83,8 +84,8 @@ type BarChart struct { // NewBarChart returns a new bar chart with a single bar for each value. // The bars heights correspond to the values and their x locations correspond -// to the index of their value in the Valuer. Optional error-bar values can be -// provided. +// to the index of their value in the Valuer. +// Optional error-bar values can be provided. func NewBarChart(vs, ers plot.Valuer) *BarChart { values, err := plot.CopyValues(vs) if errors.Log(err) != nil { @@ -105,6 +106,18 @@ func NewBarChart(vs, ers plot.Valuer) *BarChart { return b } +// NewBarChartTensor returns a new bar chart with a single bar for each value. +// The bars heights correspond to the values and their x locations correspond +// to the index of their value in the Valuer. +// Optional error-bar values can be provided. +func NewBarChartTensor(vs, ers tensor.Tensor) *BarChart { + vt := plot.TensorValues{vs} + if ers == nil { + return NewBarChart(vt, nil) + } + return NewBarChart(vt, plot.TensorValues{ers}) +} + func (b *BarChart) Defaults() { b.Offset = 1 b.Stride = 1 diff --git a/plot/plots/line.go b/plot/plots/line.go index db653e5ebe..f5a40dbf86 100644 --- a/plot/plots/line.go +++ b/plot/plots/line.go @@ -17,6 +17,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" + "cogentcore.org/core/tensor" ) // StepKind specifies a form of a connection of two consecutive points. @@ -86,6 +87,12 @@ func NewLinePoints(xys plot.XYer) (*Line, *Scatter) { return ln, sc } +// NewLineTensor returns a Line that uses the default line style and +// does not draw glyphs, based on two tensors for X, Y values. +func NewLineTensor(x, y tensor.Tensor) *Line { + return NewLine(plot.TensorXYs{X: x, Y: y}) +} + func (ln *Line) Defaults() { ln.Line.Defaults() } diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go index 13c19a13e8..d72cea7f3d 100644 --- a/tensor/examples/planets/planets.go +++ b/tensor/examples/planets/planets.go @@ -19,6 +19,7 @@ import ( "cogentcore.org/core/tensor/stats/stats" "cogentcore.org/core/tensor/table" "cogentcore.org/core/tree" + "cogentcore.org/core/yaegicore/symbols" ) //go:embed *.csv @@ -92,6 +93,8 @@ func main() { } func Interactive(c *interpreter.Config, in *interpreter.Interpreter) error { + in.Interp.Use(symbols.Symbols) // gui imports + in.Config() br := databrowser.NewBrowserWindow(datafs.CurRoot, "Planets") b := br.Parent.(*core.Body) b.AddTopBar(func(bar *core.Frame) { diff --git a/yaegicore/symbols/cogentcore_org-core-plot-plots.go b/yaegicore/symbols/cogentcore_org-core-plot-plots.go index 956a47e0d0..570f75d20f 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot-plots.go +++ b/yaegicore/symbols/cogentcore_org-core-plot-plots.go @@ -26,9 +26,11 @@ func init() { "DrawTriangle": reflect.ValueOf(plots.DrawTriangle), "MidStep": reflect.ValueOf(plots.MidStep), "NewBarChart": reflect.ValueOf(plots.NewBarChart), + "NewBarChartTensor": reflect.ValueOf(plots.NewBarChartTensor), "NewLabels": reflect.ValueOf(plots.NewLabels), "NewLine": reflect.ValueOf(plots.NewLine), "NewLinePoints": reflect.ValueOf(plots.NewLinePoints), + "NewLineTensor": reflect.ValueOf(plots.NewLineTensor), "NewScatter": reflect.ValueOf(plots.NewScatter), "NewTableXYer": reflect.ValueOf(plots.NewTableXYer), "NewXErrorBars": reflect.ValueOf(plots.NewXErrorBars), diff --git a/yaegicore/symbols/cogentcore_org-core-plot.go b/yaegicore/symbols/cogentcore_org-core-plot.go index 6e2b8b230f..73c7dfb7e3 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot.go +++ b/yaegicore/symbols/cogentcore_org-core-plot.go @@ -42,6 +42,9 @@ func init() { "Normalizer": reflect.ValueOf((*plot.Normalizer)(nil)), "Plot": reflect.ValueOf((*plot.Plot)(nil)), "Plotter": reflect.ValueOf((*plot.Plotter)(nil)), + "Stylers": reflect.ValueOf((*plot.Stylers)(nil)), + "TensorValues": reflect.ValueOf((*plot.TensorValues)(nil)), + "TensorXYs": reflect.ValueOf((*plot.TensorXYs)(nil)), "Text": reflect.ValueOf((*plot.Text)(nil)), "TextStyle": reflect.ValueOf((*plot.TextStyle)(nil)), "Thumbnailer": reflect.ValueOf((*plot.Thumbnailer)(nil)), @@ -103,11 +106,13 @@ func (W _cogentcore_org_core_plot_Normalizer) Normalize(min float32, max float32 // _cogentcore_org_core_plot_Plotter is an interface wrapper for Plotter type type _cogentcore_org_core_plot_Plotter struct { - IValue interface{} - WPlot func(pt *plot.Plot) - WXYData func() (data plot.XYer, pixels plot.XYer) + IValue interface{} + WApplyStyle func() + WPlot func(pt *plot.Plot) + WXYData func() (data plot.XYer, pixels plot.XYer) } +func (W _cogentcore_org_core_plot_Plotter) ApplyStyle() { W.WApplyStyle() } func (W _cogentcore_org_core_plot_Plotter) Plot(pt *plot.Plot) { W.WPlot(pt) } func (W _cogentcore_org_core_plot_Plotter) XYData() (data plot.XYer, pixels plot.XYer) { return W.WXYData() From a2f6978fb8a301b7a65ac8cad7cd7cc3dd95770c Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 6 Nov 2024 00:19:36 -0800 Subject: [PATCH 262/311] ploteditor styling functions --- plot/plotcore/options.go | 60 ++++++++++++++++++++++++++++++++++--- plot/plotcore/ploteditor.go | 15 ++++++++-- tensor/values.go | 4 +-- 3 files changed, 71 insertions(+), 8 deletions(-) diff --git a/plot/plotcore/options.go b/plot/plotcore/options.go index 66f798fed9..e0fb1fb4d3 100644 --- a/plot/plotcore/options.go +++ b/plot/plotcore/options.go @@ -7,6 +7,7 @@ package plotcore import ( "image" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/option" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" @@ -75,8 +76,8 @@ type PlotOptions struct { //types:add YAxisLabel string } -// defaults sets defaults if unset values are present. -func (po *PlotOptions) defaults() { +// Defaults sets defaults if unset values are present. +func (po *PlotOptions) Defaults() { if po.LineWidth == 0 { po.LineWidth = 1 po.Lines = true @@ -138,8 +139,8 @@ type ColumnOptions struct { //types:add IsString bool `edit:"-"` } -// defaults sets defaults if unset values are present. -func (co *ColumnOptions) defaults() { +// Defaults sets defaults if unset values are present. +func (co *ColumnOptions) Defaults() { if co.NTicks == 0 { co.NTicks = 10 } @@ -163,3 +164,54 @@ const ( // Bar plots vertical bars. Bar ) + +//////// Stylers + +// PlotStylers are plot styling functions. +type PlotStylers struct { + Plot []func(po *PlotOptions) + Column map[string][]func(co *ColumnOptions) +} + +// PlotStyler adds a plot styling function. +func (ps *PlotStylers) PlotStyler(f func(po *PlotOptions)) { + ps.Plot = append(ps.Plot, f) +} + +// ColumnStyler adds a column styling function for given column name. +func (ps *PlotStylers) ColumnStyler(col string, f func(co *ColumnOptions)) { + if ps.Column == nil { + ps.Column = make(map[string][]func(co *ColumnOptions)) + } + cs := ps.Column[col] + ps.Column[col] = append(cs, f) +} + +// ApplyToPlot applies stylers to plot options. +func (ps *PlotStylers) ApplyToPlot(po *PlotOptions) { + for _, f := range ps.Plot { + f(po) + } +} + +// ApplyToColumn applies stylers to column of given name +func (ps *PlotStylers) ApplyToColumn(co *ColumnOptions) { + if ps.Column == nil { + return + } + fs := ps.Column[co.Column] + for _, f := range fs { + f(co) + } +} + +// SetPlotStylers sets the PlotStylers into given metadata. +func SetShapeNames(md *metadata.Data, ps *PlotStylers) { + md.Set("PlotStylers", ps) +} + +// GetPlotStylers gets the PlotStylers from given metadata (nil if none). +func GetPlotStylers(md *metadata.Data) *PlotStylers { + ps, _ := metadata.Get[*PlotStylers](*md, "PlotStylers") + return ps +} diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index e3f44462c6..f2602a25cc 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -50,6 +50,9 @@ type PlotEditor struct { //types:add // Columns are the options for each column of the table. Columns []*ColumnOptions `set:"-"` + // Stylers are plot styling functions. + Stylers PlotStylers + // plot is the plot object. plot *plot.Plot @@ -94,7 +97,8 @@ func NewSubPlot(parent ...tree.Node) *PlotEditor { func (pl *PlotEditor) Init() { pl.Frame.Init() - pl.Options.defaults() + pl.Options.Defaults() + pl.Stylers.ApplyToPlot(&pl.Options) pl.Styler(func(s *styles.Style) { s.Grow.Set(1, 1) if pl.SizeClass() == core.SizeCompact { @@ -140,6 +144,9 @@ func (pl *PlotEditor) Init() { // This is safe to call from a different goroutine. func (pl *PlotEditor) setTable(tab *table.Table) *PlotEditor { pl.table = tab + if ps := GetPlotStylers(&tab.Meta); ps != nil { + pl.Stylers = *ps + } pl.Update() return pl } @@ -150,6 +157,9 @@ func (pl *PlotEditor) setTable(tab *table.Table) *PlotEditor { // This is safe to call from a different goroutine. func (pl *PlotEditor) SetTable(tab *table.Table) *PlotEditor { pl.table = table.NewView(tab) + if ps := GetPlotStylers(&tab.Meta); ps != nil { + pl.Stylers = *ps + } pl.Update() return pl } @@ -428,7 +438,8 @@ func (pl *PlotEditor) columnsListUpdate() { pl.Options.XAxis = cn // x-axis defaults to the first column } cp := &ColumnOptions{Column: cn} - cp.defaults() + cp.Defaults() + pl.Stylers.ApplyToColumn(cp) tcol := dt.ColumnByIndex(ci) tc := tcol.Tensor if tc.IsString() { diff --git a/tensor/values.go b/tensor/values.go index 2d614e9b28..6fd9a41d3a 100644 --- a/tensor/values.go +++ b/tensor/values.go @@ -8,7 +8,6 @@ import ( "fmt" "reflect" - "cogentcore.org/core/base/errors" "cogentcore.org/core/base/metadata" ) @@ -123,5 +122,6 @@ func SetShapeNames(md *metadata.Data, names ...string) { // ShapeNames gets the tensor shape dimension names from given metadata. func ShapeNames(md *metadata.Data) []string { - return errors.Log1(metadata.Get[[]string](*md, "ShapeNames")) + names, _ := metadata.Get[[]string](*md, "ShapeNames") + return names } From 5167ef5b2abee1e14c2720a843b2fb32b3cd6c77 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 6 Nov 2024 13:31:46 -0800 Subject: [PATCH 263/311] updated datafs-sim example with stat funcs that do everything in one place; major overhaul of datafs to use Recycle logic with fs.ErrExists to indicate existing, per Go os package standard behavior. --- filetree/node.go | 16 +- plot/plotcore/options.go | 6 +- tensor/datafs/README.md | 10 + tensor/datafs/data.go | 77 +++--- tensor/datafs/dir.go | 57 +++-- tensor/examples/datafs-sim/enumgen.go | 50 ++++ tensor/examples/datafs-sim/sim.go | 352 ++++++++++++++++---------- tensor/stats/stats/describe.go | 8 +- tensor/stats/stats/group.go | 34 +-- 9 files changed, 391 insertions(+), 219 deletions(-) create mode 100644 tensor/examples/datafs-sim/enumgen.go diff --git a/filetree/node.go b/filetree/node.go index 2f26abff3d..f164b474be 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -300,9 +300,11 @@ func (fn *Node) dirFileList() []fs.FileInfo { var files []fs.FileInfo var dirs []fs.FileInfo // for DirsOnTop mode var di []fs.DirEntry + isFS := false if fn.FileRoot.FS == nil { di = errors.Log1(os.ReadDir(path)) } else { + isFS = true di = errors.Log1(fs.ReadDir(fn.FileRoot.FS, path)) } for _, d := range di { @@ -326,22 +328,26 @@ func (fn *Node) dirFileList() []fs.FileInfo { if fn.FileRoot.DirsOnTop { if doModSort { - sortByModTime(dirs) - sortByModTime(files) + sortByModTime(dirs, isFS) // note: FS = ascending, otherwise descending + sortByModTime(files, isFS) } files = append(dirs, files...) } else { if doModSort { - sortByModTime(files) + sortByModTime(files, isFS) } } return files } // sortByModTime sorts by _reverse_ mod time (newest first) -func sortByModTime(files []fs.FileInfo) { +func sortByModTime(files []fs.FileInfo, ascending bool) { slices.SortFunc(files, func(a, b fs.FileInfo) int { - return b.ModTime().Compare(a.ModTime()) // reverse order + if ascending { + return a.ModTime().Compare(b.ModTime()) + } else { + return b.ModTime().Compare(a.ModTime()) + } }) } diff --git a/plot/plotcore/options.go b/plot/plotcore/options.go index e0fb1fb4d3..e1349131d2 100644 --- a/plot/plotcore/options.go +++ b/plot/plotcore/options.go @@ -173,6 +173,10 @@ type PlotStylers struct { Column map[string][]func(co *ColumnOptions) } +func NewPlotStylers() *PlotStylers { + return &PlotStylers{} +} + // PlotStyler adds a plot styling function. func (ps *PlotStylers) PlotStyler(f func(po *PlotOptions)) { ps.Plot = append(ps.Plot, f) @@ -206,7 +210,7 @@ func (ps *PlotStylers) ApplyToColumn(co *ColumnOptions) { } // SetPlotStylers sets the PlotStylers into given metadata. -func SetShapeNames(md *metadata.Data, ps *PlotStylers) { +func SetPlotStylers(md *metadata.Data, ps *PlotStylers) { md.Set("PlotStylers", ps) } diff --git a/tensor/datafs/README.md b/tensor/datafs/README.md index 78d9fc848c..712be074a5 100644 --- a/tensor/datafs/README.md +++ b/tensor/datafs/README.md @@ -12,4 +12,14 @@ Each Data node has a name which must be unique within the directory. The nodes i The hierarchical structure of a filesystem naturally supports various kinds of functions, such as various time scales of logging, with lower-level data aggregated into upper levels. Or hierarchical splits for a pivot-table effect. +# Usage + +## Existing items and unique names + +As in a real filesystem, names must be unique within each directory, which creates issues for how to manage conflicts between existing and new items. We adopt the same behavior as the Go `os` package in general: + +* If an existing item with the same name is present, return that existing item and an `fs.ErrExist` error, so that the caller can decide how to proceed, using `errors.Is(fs.ErrExist)`. + +* There are also `Recycle` versions of functions that do not return an error and are preferred when specifically expecting an existing item. + diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 0a213b8de7..83a53d7417 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -5,6 +5,7 @@ package datafs import ( + "io/fs" "reflect" "time" @@ -45,43 +46,33 @@ type Data struct { } // newData returns a new Data item in given directory Data item, -// which can be nil. If not a directory, or the name is not unique, -// an error will be generated. +// which can be nil. If dir is not a directory, returns nil and an error. +// If an item already exists in dir with that name, that item is returned +// with an [fs.ErrExist] error, and the caller can decide how to proceed. // The modTime is set to now. The name must be unique within parent. func newData(dir *Data, name string) (*Data, error) { - d := &Data{Parent: dir, name: name, modTime: time.Now()} - var err error - if dir != nil { - err = dir.Add(d) + if dir == nil { + return &Data{name: name, modTime: time.Now()}, nil } - return d, err -} - -// NewScalar returns new scalar Data value(s) (as a [tensor.Tensor]) -// of given data type, in given directory. -// The names must be unique in the directory. -// Returns the first item created, for immediate use of one value. -func NewScalar[T tensor.DataTypes](dir *Data, names ...string) tensor.Tensor { - var first tensor.Tensor - for _, nm := range names { - tsr := tensor.New[T](1) - tsr.Metadata().SetName(nm) - d, err := newData(dir, nm) - if errors.Log(err) != nil { - return nil - } - d.Data = tsr - if first == nil { - first = d.Data - } + if err := dir.mustDir("newData", name); err != nil { + return nil, err + } + if ex, ok := dir.Dir.AtTry(name); ok { + return ex, fs.ErrExist } - return first + d := &Data{Parent: dir, name: name, modTime: time.Now()} + dir.Dir.Add(name, d) + return d, nil } -// NewValue returns a new Data value as a [tensor.Tensor] +// Value returns a Data value as a [tensor.Tensor] // of given data type and shape sizes, in given directory Data item. -// The name must be unique in the directory. -func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Values { +// If it already exists, it is returned, else a new one is made. +func Value[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.Values { + it := dir.Item(name) + if it != nil { + return it.Data.(tensor.Values) + } tsr := tensor.New[T](sizes...) tsr.Metadata().SetName(name) d, err := newData(dir, name) @@ -92,11 +83,31 @@ func NewValue[T tensor.DataTypes](dir *Data, name string, sizes ...int) tensor.V return tsr } +// Scalar returns a scalar Data value (as a [tensor.Tensor]) +// of given data type, in given directory and name. +// If it already exists, it is returned, else a new one is made. +func Scalar[T tensor.DataTypes](dir *Data, name string) tensor.Values { + return Value[T](dir, name, 1) +} + +// NewScalars makes new scalar Data value(s) (as a [tensor.Tensor]) +// of given data type, in given directory. +// The names must be unique in the directory (existing items are recycled). +func NewScalars[T tensor.DataTypes](dir *Data, names ...string) { + for _, nm := range names { + Scalar[T](dir, nm) + } +} + // NewOfType returns a new Data value as a [tensor.Tensor] // of given reflect.Kind type and shape sizes per dimension, in given directory Data item. // Supported types are string, bool (for [Bool]), float32, float64, int, int32, and byte. -// The name must be unique in the directory. +// If an item with that name already exists, then it is returned. func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Values { + it := d.Item(name) + if it != nil { + return it.Data.(tensor.Values) + } tsr := tensor.NewOfType(typ, sizes...) tsr.Metadata().SetName(name) nd, err := newData(d, name) @@ -108,11 +119,11 @@ func (d *Data) NewOfType(name string, typ reflect.Kind, sizes ...int) tensor.Val } // NewData creates a new Data node for given tensor with given name. -// returns an error if the data name already exists +// If the name already exists, that item is returned with [fs.ErrExists] error. func (d *Data) NewData(tsr tensor.Tensor, name string) (*Data, error) { nd, err := newData(d, name) if err != nil { - return nil, err + return nd, err } nd.Data = tsr return nd, nil diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index 9e0b70dc7e..f2726ca19f 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -5,13 +5,13 @@ package datafs import ( - "errors" "fmt" "io/fs" "path" "slices" "sort" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/keylist" "cogentcore.org/core/tensor" @@ -23,11 +23,12 @@ import ( // the natural order items are processed in. type Dir = keylist.List[string, *Data] -// NewDir returns a new datafs directory with given name. -// if parent != nil and a directory, this dir is added to it. -// if name is empty, then it is set to "root", the root directory. +// NewDir returns a new datafs directory with the given name. +// If parent != nil and a directory, this dir is added to it. +// If the parent already has an item of that name, it is returned, +// with an [fs.ErrExist] error. +// If the name is empty, then it is set to "root", the root directory. // Note that "/" is not allowed for the root directory in Go [fs]. -// Names must be unique within a directory. func NewDir(name string, parent ...*Data) (*Data, error) { if name == "" { name = "root" @@ -37,7 +38,9 @@ func NewDir(name string, parent ...*Data) (*Data, error) { par = parent[0] } d, err := newData(par, name) - d.Dir = &Dir{} + if d != nil && d.Dir == nil { + d.Dir = &Dir{} + } return d, err } @@ -317,7 +320,7 @@ func (d *Data) mustDir(op, path string) error { // Add adds an item to this directory data item. // The only errors are if this item is not a directory, -// or the name already exists. +// or the name already exists, in which case an [fs.ErrExist] is returned. // Names must be unique within a directory. func (d *Data) Add(it *Data) error { if err := d.mustDir("Add", it.name); err != nil { @@ -325,16 +328,17 @@ func (d *Data) Add(it *Data) error { } err := d.Dir.Add(it.name, it) if err != nil { - return &fs.PathError{Op: "Add", Path: it.name, Err: errors.New("data item already exists; names must be unique")} + return fs.ErrExist } return nil } // Mkdir creates a new directory with the specified name. -// Returns an error if this item is not a directory, -// or if the name is already used within this directory. -// See [Data.RecycleDir] for a version ensures a directory -// exists whether it needs to be made or already does. +// Returns an error if this parent item is not a directory. +// Returns existing directory and [fs.ErrExist] if an item +// with the same name already exists. +// See [Data.RecycleDir] for a version with no error return +// that is preferable when expecting an existing directory. func (d *Data) Mkdir(name string) (*Data, error) { if err := d.mustDir("Mkdir", name); err != nil { return nil, err @@ -344,15 +348,32 @@ func (d *Data) Mkdir(name string) (*Data, error) { // RecycleDir creates a new directory with the specified name // if it doesn't already exist, otherwise returns the existing one. -// It only returns an error is if this item is not a directory. -func (d *Data) RecycleDir(name string) (*Data, error) { - if err := d.mustDir("RecycleDir", name); err != nil { - return nil, err +// It logs an error and returns nil if this parent item is not a directory. +func (d *Data) RecycleDir(name string) *Data { + if err := d.mustDir("RecycleDir", name); errors.Log(err) != nil { + return nil } if cd := d.Dir.At(name); cd != nil { - return cd, nil + return cd } - return NewDir(name, d) + nd, _ := NewDir(name, d) + return nd +} + +// Recycle ensures that an item with the given Data item's name +// exists within this directory, returning the actual item. +// If there is no such item already, the given Data item is added, +// otherwise the existing one is returned. +// It will log an error and return nil if the parent Data is not a directory. +func (d *Data) Recycle(it *Data) *Data { + if err := d.mustDir("Recycle", it.name); errors.Log(err) != nil { + return nil + } + if ex, ok := d.Dir.AtTry(it.name); ok { + return ex + } + d.Dir.Add(it.name, it) + return it } // GetDirTable gets the DirTable as a [table.Table] for this directory item, diff --git a/tensor/examples/datafs-sim/enumgen.go b/tensor/examples/datafs-sim/enumgen.go new file mode 100644 index 0000000000..d159954e83 --- /dev/null +++ b/tensor/examples/datafs-sim/enumgen.go @@ -0,0 +1,50 @@ +// Code generated by "core generate"; DO NOT EDIT. + +package main + +import ( + "cogentcore.org/core/enums" +) + +var _TimesValues = []Times{0, 1, 2} + +// TimesN is the highest valid value for type Times, plus one. +// +//gosl:start +const TimesN Times = 3 + +//gosl:end + +var _TimesValueMap = map[string]Times{`Trial`: 0, `Epoch`: 1, `Run`: 2} + +var _TimesDescMap = map[Times]string{0: ``, 1: ``, 2: ``} + +var _TimesMap = map[Times]string{0: `Trial`, 1: `Epoch`, 2: `Run`} + +// String returns the string representation of this Times value. +func (i Times) String() string { return enums.String(i, _TimesMap) } + +// SetString sets the Times value from its string representation, +// and returns an error if the string is invalid. +func (i *Times) SetString(s string) error { return enums.SetString(i, s, _TimesValueMap, "Times") } + +// Int64 returns the Times value as an int64. +func (i Times) Int64() int64 { return int64(i) } + +// SetInt64 sets the Times value from an int64. +func (i *Times) SetInt64(in int64) { *i = Times(in) } + +// Desc returns the description of the Times value. +func (i Times) Desc() string { return enums.Desc(i, _TimesDescMap) } + +// TimesValues returns all possible values for the type Times. +func TimesValues() []Times { return _TimesValues } + +// Values returns all possible values for the type Times. +func (i Times) Values() []enums.Enum { return enums.Values(_TimesValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i Times) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *Times) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Times") } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 97162c97ba..15604d73e3 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -1,13 +1,15 @@ +// Code generated by "goal build"; DO NOT EDIT. +//line sim.goal:1 // Copyright (c) 2024, Cogent Core. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package main +//go:generate core generate + import ( "math/rand/v2" - "reflect" - "strconv" "cogentcore.org/core/core" "cogentcore.org/core/plot/plotcore" @@ -17,168 +19,252 @@ import ( "cogentcore.org/core/tensor/stats/stats" ) +type Times int32 //enums:enum + +const ( + Trial Times = iota + Epoch + Run +) + type Sim struct { - Root *datafs.Data - Config *datafs.Data - Stats *datafs.Data - Logs *datafs.Data + Root *datafs.Data + Config *datafs.Data + Stats *datafs.Data + StatFuncs []func(tm Times) + Counters [TimesN]int } // ConfigAll configures the sim func (ss *Sim) ConfigAll() { ss.Root, _ = datafs.NewDir("Root") ss.Config, _ = ss.Root.Mkdir("Config") - datafs.NewScalar[int](ss.Config, "NRun", "NEpoch", "NTrial") - ss.Config.Item("NRun").SetInt(5) - ss.Config.Item("NEpoch").SetInt(20) - ss.Config.Item("NTrial").SetInt(25) - - ss.Stats = ss.ConfigStats(ss.Root) - ss.Logs = ss.ConfigLogs(ss.Root) + mx := datafs.Value[int](ss.Config, "Max", int(TimesN)).(*tensor.Int) + mx.Set1D(5, int(Trial)) + mx.Set1D(4, int(Epoch)) + mx.Set1D(3, int(Run)) + // todo: failing - assigns 3 to all + // # mx[Trial] = 5 + // # mx[Epoch] = 4 + // # mx[Run] = 3 + ss.ConfigStats() } -// ConfigStats adds basic stats that we record for our simulation. -func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { - stats, _ := dir.Mkdir("Stats") - datafs.NewScalar[int](stats, "Run", "Epoch", "Trial") // counters - datafs.NewScalar[string](stats, "TrialName") - datafs.NewScalar[float32](stats, "SSE", "AvgSSE", "TrlErr") - z1, key := plotcore.PlotColumnZeroOne() - stats.SetMetaItems(key, z1, "AvgErr", "TrlErr") - zmax, _ := plotcore.PlotColumnZeroOne() - zmax.Range.FixMax = false - stats.SetMetaItems(key, z1, "SSE") - return stats +func (ss *Sim) AddStat(f func(tm Times)) { + ss.StatFuncs = append(ss.StatFuncs, f) } -// ConfigLogs adds first-level logging of stats into tensors -func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { - logd, _ := dir.Mkdir("Log") - trial := ss.ConfigTrialLog(logd) - ss.ConfigAggLog(logd, "Epoch", trial, stats.StatMean, stats.StatSem, stats.StatMin) - return logd +func (ss *Sim) RunStats(tm Times) { + for _, sf := range ss.StatFuncs { + sf(tm) + } } -// ConfigTrialLog adds first-level logging of stats into tensors -func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { - logd, _ := dir.Mkdir("Trial") - ntrial := ss.Config.Item("NTrial").AsInt() - sitems := ss.Stats.ValuesFunc(nil) - for _, st := range sitems { - nm := st.Metadata().Name() - lt := logd.NewOfType(nm, st.DataType(), ntrial) - lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source - tensor.SetCalcFunc(lt, func() error { - trl := ss.Stats.Item("Trial").AsInt() - if st.IsString() { - lt.SetStringRow(st.String1D(0), trl) - } else { - lt.SetFloatRow(st.Float1D(0), trl) +func (ss *Sim) ConfigStats() { + ss.Stats, _ = ss.Root.Mkdir("Stats") + mx := ss.Config.Value("Max").(*tensor.Int) + ctrs := []Times{Run, Epoch, Trial} + for _, ctr := range ctrs { + ss.AddStat(func(tm Times) { + name := ctr.String() + if tm > ctr { + return } - return nil - }) - } - alllogd, _ := dir.Mkdir("AllTrials") - for _, st := range sitems { - nm := st.Metadata().Name() - // allocate full size - lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) - lt.SetShapeSizes(0) // then truncate to 0 - lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source - tensor.SetCalcFunc(lt, func() error { - row := lt.DimSize(0) - lt.SetShapeSizes(row + 1) - if st.IsString() { - lt.SetStringRow(st.String1D(0), row) - } else { - lt.SetFloatRow(st.Float1D(0), row) + ctv := ss.Counters[ctr] + mxi := mx.Value1D(int(tm)) + td := ss.Stats.RecycleDir(tm.String()) + cd := ss.Stats.RecycleDir("Current") + datafs.Scalar[int](cd, name).SetInt1D(ctv, 0) + tv := datafs.Value[int](td, name, mxi) + tv.SetInt1D(ctv, ss.Counters[tm]) + if ps := plotcore.GetPlotStylers(tv.Metadata()); ps == nil { + ps = plotcore.NewPlotStylers() + ps.ColumnStyler(name, func(co *plotcore.ColumnOptions) { + co.Range.FixMin = true + co.On = false + }) + if tm == ctr { + ps.PlotStyler(func(po *plotcore.PlotOptions) { + po.XAxis = name + }) + } + plotcore.SetPlotStylers(tv.Metadata(), ps) } - return nil }) } - return logd -} - -// ConfigAggLog adds a higher-level logging of lower-level into higher-level tensors -func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, aggs ...stats.Stats) *datafs.Data { - logd, _ := dir.Mkdir(level) - sitems := ss.Stats.ValuesFunc(nil) - nctr := ss.Config.Item("N" + level).AsInt() - for _, st := range sitems { - if st.IsString() { - continue - } - nm := st.Metadata().Name() - src := from.Value(nm) - if st.DataType() >= reflect.Float32 { - // todo: pct correct etc - dd, _ := logd.Mkdir(nm) - for _, ag := range aggs { // key advantage of dir structure: multiple stats per item - lt := dd.NewOfType(ag.String(), st.DataType(), nctr) - lt.Metadata().Copy(*st.Metadata()) - tensor.SetCalcFunc(lt, func() error { - stout := ag.Call(src) - ctr := ss.Stats.Item(level).AsInt() - lt.SetFloatRow(stout.FloatRow(0), ctr) - return nil - }) + ss.AddStat(func(tm Times) { + sseName := "SSE" + errName := "Err" + td := ss.Stats.RecycleDir(tm.String()) + switch tm { + case Trial: + ctv := ss.Counters[tm] + mxi := mx.Value1D(int(tm)) + cd := ss.Stats.RecycleDir("Current") + sse := rand.Float32() + terr := float32(1) + if sse < 0.5 { + terr = 0 } - } else { - lt := logd.NewOfType(nm, st.DataType(), nctr) - lt.Metadata().Copy(*st.Metadata()) - tensor.SetCalcFunc(lt, func() error { - v := st.Float1D(0) - ctr := ss.Stats.Item(level).AsInt() - lt.SetFloatRow(v, ctr) - return nil - }) + datafs.Scalar[float32](cd, sseName).SetFloat(float64(sse), 0) + datafs.Scalar[float32](cd, errName).SetFloat(float64(terr), 0) + datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) + datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + case Epoch: + ctv := ss.Counters[tm] + mxi := mx.Value1D(int(tm)) + trld, _ := ss.Stats.Mkdir(Trial.String()) + sse := stats.StatMean.Call(trld.Value(sseName)).Float1D(0) + terr := stats.StatMean.Call(trld.Value(errName)).Float1D(0) + datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) + datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) } - } - return logd + }) } +// // ConfigStats adds basic stats that we record for our simulation. +// func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { +// stats, _ := dir.Mkdir("Stats") +// datafs.NewScalar[int](stats, "Run", "Epoch", "Trial") // counters +// datafs.NewScalar[string](stats, "TrialName") +// datafs.NewScalar[float32](stats, "SSE", "AvgSSE", "TrlErr") +// z1, key := plotcore.PlotColumnZeroOne() +// stats.SetMetaItems(key, z1, "AvgErr", "TrlErr") +// zmax, _ := plotcore.PlotColumnZeroOne() +// zmax.Range.FixMax = false +// stats.SetMetaItems(key, z1, "SSE") +// return stats +// } +// +// // ConfigLogs adds first-level logging of stats into tensors +// func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { +// logd, _ := dir.Mkdir("Log") +// trial := ss.ConfigTrialLog(logd) +// ss.ConfigAggLog(logd, "Epoch", trial, stats.StatMean, stats.StatSem, stats.StatMin) +// return logd +// } +// +// // ConfigTrialLog adds first-level logging of stats into tensors +// func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { +// logd, _ := dir.Mkdir("Trial") +// ntrial := ss.Config.Item("NTrial").AsInt() +// sitems := ss.Stats.ValuesFunc(nil) +// for _, st := range sitems { +// nm := st.Metadata().Name() +// lt := logd.NewOfType(nm, st.DataType(), ntrial) +// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source +// tensor.SetCalcFunc(lt, func() error { +// trl := ss.Stats.Item("Trial").AsInt() +// if st.IsString() { +// lt.SetStringRow(st.String1D(0), trl) +// } else { +// lt.SetFloatRow(st.Float1D(0), trl) +// } +// return nil +// }) +// } +// alllogd, _ := dir.Mkdir("AllTrials") +// for _, st := range sitems { +// nm := st.Metadata().Name() +// // allocate full size +// lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) +// lt.SetShapeSizes(0) // then truncate to 0 +// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source +// tensor.SetCalcFunc(lt, func() error { +// row := lt.DimSize(0) +// lt.SetShapeSizes(row + 1) +// if st.IsString() { +// lt.SetStringRow(st.String1D(0), row) +// } else { +// lt.SetFloatRow(st.Float1D(0), row) +// } +// return nil +// }) +// } +// return logd +// } +// +// // ConfigAggLog adds a higher-level logging of lower-level into higher-level tensors +// func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, aggs ...stats.Stats) *datafs.Data { +// logd, _ := dir.Mkdir(level) +// sitems := ss.Stats.ValuesFunc(nil) +// nctr := ss.Config.Item("N" + level).AsInt() +// for _, st := range sitems { +// if st.IsString() { +// continue +// } +// nm := st.Metadata().Name() +// src := from.Value(nm) +// if st.DataType() >= reflect.Float32 { +// // todo: pct correct etc +// dd, _ := logd.Mkdir(nm) +// for _, ag := range aggs { // key advantage of dir structure: multiple stats per item +// lt := dd.NewOfType(ag.String(), st.DataType(), nctr) +// lt.Metadata().Copy(*st.Metadata()) +// tensor.SetCalcFunc(lt, func() error { +// stout := ag.Call(src) +// ctr := ss.Stats.Item(level).AsInt() +// lt.SetFloatRow(stout.FloatRow(0), ctr) +// return nil +// }) +// } +// } else { +// lt := logd.NewOfType(nm, st.DataType(), nctr) +// lt.Metadata().Copy(*st.Metadata()) +// tensor.SetCalcFunc(lt, func() error { +// v := st.Float1D(0) +// ctr := ss.Stats.Item(level).AsInt() +// lt.SetFloatRow(v, ctr) +// return nil +// }) +// } +// } +// return logd +// } + func (ss *Sim) Run() { - nrun := ss.Config.Item("NRun").AsInt() - nepc := ss.Config.Item("NEpoch").AsInt() - ntrl := ss.Config.Item("NTrial").AsInt() + mx := ss.Config.Value("Max").(*tensor.Int) + nrun := mx.Value1D(int(Run)) + nepc := mx.Value1D(int(Epoch)) + ntrl := mx.Value1D(int(Trial)) for run := range nrun { - ss.Stats.Item("Run").SetInt(run) + ss.Counters[Run] = run for epc := range nepc { - ss.Stats.Item("Epoch").SetInt(epc) + ss.Counters[Epoch] = epc for trl := range ntrl { - ss.Stats.Item("Trial").SetInt(trl) - ss.RunTrial(trl) + ss.Counters[Trial] = trl + ss.RunStats(Trial) } - ss.EpochDone() + ss.RunStats(Epoch) } + ss.RunStats(Run) } - alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) - dir, _ := ss.Logs.Mkdir("Stats") - stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") - sts := []string{"SSE", "AvgSSE", "TrlErr"} - stats.TableGroupStats(dir, stats.StatMean, alldt, sts...) - stats.TableGroupStats(dir, stats.StatSem, alldt, sts...) - + // alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) + // dir, _ := ss.Logs.Mkdir("Stats") + // stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") + // sts := []string{"SSE", "AvgSSE", "TrlErr"} + // stats.TableGroupStats(dir, stats.StatMean, alldt, sts...) + // stats.TableGroupStats(dir, stats.StatSem, alldt, sts...) } -func (ss *Sim) RunTrial(trl int) { - ss.Stats.Item("TrialName").SetString("Trial_" + strconv.Itoa(trl)) - sse := rand.Float32() - avgSSE := rand.Float32() - ss.Stats.Item("SSE").SetFloat32(sse) - ss.Stats.Item("AvgSSE").SetFloat32(avgSSE) - trlErr := float32(1) - if sse < 0.5 { - trlErr = 0 - } - ss.Stats.Item("TrlErr").SetFloat32(trlErr) - ss.Logs.Item("Trial").CalcAll() - ss.Logs.Item("AllTrials").CalcAll() -} +// func (ss *Sim) RunTrial(trl int) { +// ss.Stats.Item("TrialName").SetString("Trial_" + strconv.Itoa(trl)) +// sse := rand.Float32() +// avgSSE := rand.Float32() +// ss.Stats.Item("SSE").SetFloat32(sse) +// ss.Stats.Item("AvgSSE").SetFloat32(avgSSE) +// trlErr := float32(1) +// if sse < 0.5 { +// trlErr = 0 +// } +// ss.Stats.Item("TrlErr").SetFloat32(trlErr) +// ss.Logs.Item("Trial").CalcAll() +// ss.Logs.Item("AllTrials").CalcAll() +// } -func (ss *Sim) EpochDone() { - ss.Logs.Item("Epoch").CalcAll() -} +// func (ss *Sim) EpochDone() { +// ss.Logs.Item("Epoch").CalcAll() +// } func main() { ss := &Sim{} diff --git a/tensor/stats/stats/describe.go b/tensor/stats/stats/describe.go index fd69fa9424..69ba5da0be 100644 --- a/tensor/stats/stats/describe.go +++ b/tensor/stats/stats/describe.go @@ -7,7 +7,6 @@ package stats import ( "strconv" - "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" @@ -24,10 +23,7 @@ var DescriptiveStats = []Stats{StatCount, StatMean, StatStd, StatSem, StatMin, S // The [DescriptiveStats] list is: [Count], [Mean], [Std], [Sem], // [Min], [Max], [Q1], [Median], [Q3] func Describe(dir *datafs.Data, tsrs ...tensor.Tensor) { - dd, err := dir.RecycleDir("Describe") - if errors.Log(err) != nil { - return - } + dd := dir.RecycleDir("Describe") for i, tsr := range tsrs { nr := tsr.DimSize(0) if nr == 0 { @@ -40,7 +36,7 @@ func Describe(dir *datafs.Data, tsrs ...tensor.Tensor) { td, _ := dd.Mkdir(nm) for _, st := range DescriptiveStats { stnm := st.String() - sv := datafs.NewValue[float64](td, stnm, 1) + sv := datafs.Scalar[float64](td, stnm) stout := st.Call(tsr) sv.CopyFrom(stout) } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 1ad3c08349..20defc58e3 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -30,13 +30,10 @@ import ( // the results can be used directly as Indexes into the corresponding tensor data. // Uses a stable sort on columns, so ordering of other dimensions is preserved. func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) error { - gd, err := dir.RecycleDir("Groups") - if err != nil { - return err - } + gd := dir.RecycleDir("Groups") makeIdxs := func(dir *datafs.Data, srt *tensor.Rows, val string, start, r int) { n := r - start - it := datafs.NewValue[int](dir, val, n) + it := datafs.Value[int](dir, val, n) for j := range n { it.SetIntRow(srt.Indexes[start+j], j) // key to indirect through sort indexes } @@ -99,17 +96,14 @@ func TableGroups(dir *datafs.Data, dt *table.Table, columns ...string) error { // be used with [GroupStats] to generate summary statistics across // all the data. See [Groups] for more general documentation. func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { - gd, err := dir.RecycleDir("Groups") - if err != nil { - return err - } + gd := dir.RecycleDir("Groups") tsr := tensor.AsRows(tsrs[0]) nr := tsr.NumRows() if nr == 0 { return nil } td, _ := gd.Mkdir("All") - it := datafs.NewValue[int](td, "All", nr) + it := datafs.Value[int](td, "All", nr) for j := range nr { it.SetIntRow(tsr.RowIndex(j), j) // key to indirect through any existing indexes } @@ -129,35 +123,29 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { // and a aligned Float64 tensor with the statistics results for each such // unique group value. See the README.md file for a diagram of the results. func GroupStats(dir *datafs.Data, stat Stats, tsrs ...tensor.Tensor) error { - gd, err := dir.RecycleDir("Groups") - if err != nil { - return err - } - sd, err := dir.RecycleDir("Stats") - if err != nil { - return err - } + gd := dir.RecycleDir("Groups") + sd := dir.RecycleDir("Stats") stnm := StripPackage(stat.String()) groups := gd.ItemsFunc(nil) for _, gp := range groups { gpnm := gp.Name() - ggd, _ := gd.RecycleDir(gpnm) + ggd := gd.RecycleDir(gpnm) vals := ggd.ValuesFunc(nil) nv := len(vals) if nv == 0 { continue } - sgd, _ := sd.RecycleDir(gpnm) + sgd := sd.RecycleDir(gpnm) gv := sgd.Item(gpnm) if gv == nil { - gtsr := datafs.NewValue[string](sgd, gpnm, nv) + gtsr := datafs.Value[string](sgd, gpnm, nv) for i, v := range vals { gtsr.SetStringRow(v.Metadata().Name(), i) } } for _, tsr := range tsrs { - vd, _ := sgd.RecycleDir(tsr.Metadata().Name()) - sv := datafs.NewValue[float64](vd, stnm, nv) + vd := sgd.RecycleDir(tsr.Metadata().Name()) + sv := datafs.Value[float64](vd, stnm, nv) for i, v := range vals { idx := tensor.AsIntSlice(v) sg := tensor.NewRows(tsr.AsValues(), idx...) From 462bfdf7351f37e34de1d94bf8d7ea9a2e2cbd6a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 6 Nov 2024 16:40:32 -0800 Subject: [PATCH 264/311] first pass on new unified style, only for plots for now; got rid of separate Scatter plot -- just use the same Line logic with different styling. --- plot/line.go | 46 ++++- plot/plots/barchart.go | 72 ++------ plot/plots/enumgen.go | 87 --------- plot/plots/errbars.go | 114 +++++------- plot/plots/labels.go | 99 ++++------ plot/plots/line.go | 147 ++++++--------- plot/plots/plot_test.go | 109 ++++++----- plot/plots/scatter.go | 105 ----------- plot/plots/table.go | 12 -- plot/plotter.go | 15 +- plot/{plots/shapes.go => point.go} | 106 ++++++++--- plot/style.go | 128 +++++++++++++ plot/text.go | 3 + plot/typegen.go | 152 ++++++++++++++- tensor/examples/datafs-sim/sim.goal | 274 ++++++++++++++++++++++++++++ 15 files changed, 892 insertions(+), 577 deletions(-) delete mode 100644 plot/plots/enumgen.go delete mode 100644 plot/plots/scatter.go rename plot/{plots/shapes.go => point.go} (62%) create mode 100644 plot/style.go create mode 100644 tensor/examples/datafs-sim/sim.goal diff --git a/plot/line.go b/plot/line.go index 2e05073391..b40de6c4fe 100644 --- a/plot/line.go +++ b/plot/line.go @@ -12,18 +12,36 @@ import ( "cogentcore.org/core/styles/units" ) -// LineStyle has style properties for line drawing -type LineStyle struct { +// LineStyle has style properties for drawing lines. +type LineStyle struct { //types:add -setters + // On indicates whether to plot lines. + On DefaultOffOn - // stroke color image specification; stroking is off if nil + // Color is the stroke color image specification. + // Setting to nil turns line off. Color image.Image - // line width + // Width is the line width, with a default of 1 Pt (point). + // Setting to 0 turns line off. Width units.Value // Dashes are the dashes of the stroke. Each pair of values specifies // the amount to paint and then the amount to skip. Dashes []float32 + + // Fill is the color to fill solid regions, in a plot-specific + // way (e.g., the area below a Line plot, the bar color). + // Use nil to disable filling. + Fill image.Image + + // NegativeX specifies whether to draw lines that connect points with a negative + // X-axis direction; otherwise there is a break in the line. + // default is false, so that repeated series of data across the X axis + // are plotted separately. + NegativeX bool + + // Step specifies how to step the line between points. + Step StepKind } func (ls *LineStyle) Defaults() { @@ -34,7 +52,7 @@ func (ls *LineStyle) Defaults() { // SetStroke sets the stroke style in plot paint to current line style. // returns false if either the Width = 0 or Color is nil func (ls *LineStyle) SetStroke(pt *Plot) bool { - if ls.Color == nil { + if ls.On == Off || ls.Color == nil { return false } pc := pt.Paint @@ -61,3 +79,21 @@ func (ls *LineStyle) Draw(pt *Plot, start, end math32.Vector2) bool { pc.Stroke() return true } + +// StepKind specifies a form of a connection of two consecutive points. +type StepKind int32 //enums:enum + +const ( + // NoStep connects two points by simple line. + NoStep StepKind = iota + + // PreStep connects two points by following lines: vertical, horizontal. + PreStep + + // MidStep connects two points by following lines: horizontal, vertical, horizontal. + // Vertical line is placed in the middle of the interval. + MidStep + + // PostStep connects two points by following lines: horizontal, vertical. + PostStep +) diff --git a/plot/plots/barchart.go b/plot/plots/barchart.go index 674dfabd3a..ad4d822656 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/barchart.go @@ -12,10 +12,7 @@ package plots import ( - "image" - "cogentcore.org/core/base/errors" - "cogentcore.org/core/colors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" "cogentcore.org/core/tensor" @@ -43,32 +40,8 @@ type BarChart struct { // PXYs is the actual pixel plotting coordinates for each value. PXYs plot.XYs - // Offset is offset added to each X axis value relative to the - // Stride computed value (X = offset + index * Stride) - // Defaults to 1. - Offset float32 - - // Stride is distance between bars. Defaults to 1. - Stride float32 - - // Width is the width of the bars, which should be less than - // the Stride to prevent bar overlap. - // Defaults to .8 - Width float32 - - // Pad is additional space at start / end of data range, to keep bars from - // overflowing ends. This amount is subtracted from Offset - // and added to (len(Values)-1)*Stride -- no other accommodation for bar - // width is provided, so that should be built into this value as well. - // Defaults to 1. - Pad float32 - - // Color is the fill color of the bars. - Color image.Image - - // Line is the style of the line connecting the points. - // Use zero width to disable lines. - Line plot.LineStyle + // Style has the properties used to render the bars. + Style plot.Style // Horizontal dictates whether the bars should be in the vertical // (default) or horizontal direction. If Horizontal is true, all @@ -119,22 +92,15 @@ func NewBarChartTensor(vs, ers tensor.Tensor) *BarChart { } func (b *BarChart) Defaults() { - b.Offset = 1 - b.Stride = 1 - b.Width = .8 - b.Pad = 1 - b.Color = colors.Scheme.OnSurface - b.Line.Defaults() + b.Style.Defaults() } -func (b *BarChart) Styler(f func(s *BarChart)) *BarChart { - b.stylers.Add(func(p plot.Plotter) { f(p.(*BarChart)) }) +func (b *BarChart) Styler(f func(s *plot.Style)) *BarChart { + b.stylers.Add(f) return b } -func (b *BarChart) ApplyStyle() { - b.stylers.Run(b) -} +func (b *BarChart) ApplyStyle() { b.stylers.Run(&b.Style) } func (b *BarChart) XYData() (data plot.XYer, pixels plot.XYer) { data = b.XYs @@ -163,26 +129,25 @@ func (b *BarChart) BarHeight(i int) float32 { // and sets the bar positioning options to that of the // chart upon which it is being stacked. func (b *BarChart) StackOn(on *BarChart) { - b.Offset = on.Offset - b.Stride = on.Stride - b.Pad = on.Pad + b.Style.Width = on.Style.Width b.StackedOn = on } // Plot implements the plot.Plotter interface. func (b *BarChart) Plot(plt *plot.Plot) { pc := plt.Paint - pc.FillStyle.Color = b.Color - b.Line.SetStroke(plt) + b.Style.Line.SetStroke(plt) + pc.FillStyle.Color = b.Style.Line.Fill + bw := b.Style.Width nv := len(b.Values) b.XYs = make(plot.XYs, nv) b.PXYs = make(plot.XYs, nv) - hw := 0.5 * b.Width - ew := b.Width / 3 + hw := 0.5 * bw.Width + ew := bw.Width / 3 for i, ht := range b.Values { - cat := b.Offset + float32(i)*b.Stride + cat := bw.Offset + float32(i)*bw.Stride var bottom, catVal, catMin, catMax, valMin, valMax float32 var box math32.Box2 if b.Horizontal { @@ -230,12 +195,14 @@ func (b *BarChart) Plot(plt *plot.Plot) { pc.Stroke() } } + pc.FillStyle.Color = nil } // DataRange implements the plot.DataRanger interface. func (b *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - catMin := b.Offset - b.Pad - catMax := b.Offset + float32(len(b.Values)-1)*b.Stride + b.Pad + bw := b.Style.Width + catMin := bw.Offset - bw.Pad + catMax := bw.Offset + float32(len(b.Values)-1)*bw.Stride + bw.Pad valMin := math32.Inf(1) valMax := math32.Inf(-1) @@ -257,9 +224,10 @@ func (b *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { // Thumbnail fulfills the plot.Thumbnailer interface. func (b *BarChart) Thumbnail(plt *plot.Plot) { pc := plt.Paint - pc.FillStyle.Color = b.Color - b.Line.SetStroke(plt) + b.Style.Line.SetStroke(plt) + pc.FillStyle.Color = b.Style.Line.Fill ptb := pc.Bounds pc.DrawRectangle(float32(ptb.Min.X), float32(ptb.Min.Y), float32(ptb.Size().X), float32(ptb.Size().Y)) pc.FillStrokeClear() + pc.FillStyle.Color = nil } diff --git a/plot/plots/enumgen.go b/plot/plots/enumgen.go deleted file mode 100644 index b795361bc0..0000000000 --- a/plot/plots/enumgen.go +++ /dev/null @@ -1,87 +0,0 @@ -// Code generated by "core generate"; DO NOT EDIT. - -package plots - -import ( - "cogentcore.org/core/enums" -) - -var _StepKindValues = []StepKind{0, 1, 2, 3} - -// StepKindN is the highest valid value for type StepKind, plus one. -const StepKindN StepKind = 4 - -var _StepKindValueMap = map[string]StepKind{`NoStep`: 0, `PreStep`: 1, `MidStep`: 2, `PostStep`: 3} - -var _StepKindDescMap = map[StepKind]string{0: `NoStep connects two points by simple line`, 1: `PreStep connects two points by following lines: vertical, horizontal.`, 2: `MidStep connects two points by following lines: horizontal, vertical, horizontal. Vertical line is placed in the middle of the interval.`, 3: `PostStep connects two points by following lines: horizontal, vertical.`} - -var _StepKindMap = map[StepKind]string{0: `NoStep`, 1: `PreStep`, 2: `MidStep`, 3: `PostStep`} - -// String returns the string representation of this StepKind value. -func (i StepKind) String() string { return enums.String(i, _StepKindMap) } - -// SetString sets the StepKind value from its string representation, -// and returns an error if the string is invalid. -func (i *StepKind) SetString(s string) error { - return enums.SetString(i, s, _StepKindValueMap, "StepKind") -} - -// Int64 returns the StepKind value as an int64. -func (i StepKind) Int64() int64 { return int64(i) } - -// SetInt64 sets the StepKind value from an int64. -func (i *StepKind) SetInt64(in int64) { *i = StepKind(in) } - -// Desc returns the description of the StepKind value. -func (i StepKind) Desc() string { return enums.Desc(i, _StepKindDescMap) } - -// StepKindValues returns all possible values for the type StepKind. -func StepKindValues() []StepKind { return _StepKindValues } - -// Values returns all possible values for the type StepKind. -func (i StepKind) Values() []enums.Enum { return enums.Values(_StepKindValues) } - -// MarshalText implements the [encoding.TextMarshaler] interface. -func (i StepKind) MarshalText() ([]byte, error) { return []byte(i.String()), nil } - -// UnmarshalText implements the [encoding.TextUnmarshaler] interface. -func (i *StepKind) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "StepKind") } - -var _ShapesValues = []Shapes{0, 1, 2, 3, 4, 5, 6, 7} - -// ShapesN is the highest valid value for type Shapes, plus one. -const ShapesN Shapes = 8 - -var _ShapesValueMap = map[string]Shapes{`Ring`: 0, `Circle`: 1, `Square`: 2, `Box`: 3, `Triangle`: 4, `Pyramid`: 5, `Plus`: 6, `Cross`: 7} - -var _ShapesDescMap = map[Shapes]string{0: `Ring is the outline of a circle`, 1: `Circle is a solid circle`, 2: `Square is the outline of a square`, 3: `Box is a filled square`, 4: `Triangle is the outline of a triangle`, 5: `Pyramid is a filled triangle`, 6: `Plus is a plus sign`, 7: `Cross is a big X`} - -var _ShapesMap = map[Shapes]string{0: `Ring`, 1: `Circle`, 2: `Square`, 3: `Box`, 4: `Triangle`, 5: `Pyramid`, 6: `Plus`, 7: `Cross`} - -// String returns the string representation of this Shapes value. -func (i Shapes) String() string { return enums.String(i, _ShapesMap) } - -// SetString sets the Shapes value from its string representation, -// and returns an error if the string is invalid. -func (i *Shapes) SetString(s string) error { return enums.SetString(i, s, _ShapesValueMap, "Shapes") } - -// Int64 returns the Shapes value as an int64. -func (i Shapes) Int64() int64 { return int64(i) } - -// SetInt64 sets the Shapes value from an int64. -func (i *Shapes) SetInt64(in int64) { *i = Shapes(in) } - -// Desc returns the description of the Shapes value. -func (i Shapes) Desc() string { return enums.Desc(i, _ShapesDescMap) } - -// ShapesValues returns all possible values for the type Shapes. -func ShapesValues() []Shapes { return _ShapesValues } - -// Values returns all possible values for the type Shapes. -func (i Shapes) Values() []enums.Enum { return enums.Values(_ShapesValues) } - -// MarshalText implements the [encoding.TextMarshaler] interface. -func (i Shapes) MarshalText() ([]byte, error) { return []byte(i.String()), nil } - -// UnmarshalText implements the [encoding.TextUnmarshaler] interface. -func (i *Shapes) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Shapes") } diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 481eb8fadd..a43bd95078 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -8,7 +8,6 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" - "cogentcore.org/core/styles/units" ) ////////////////////////////////////////////////// @@ -59,18 +58,14 @@ type YErrorBars struct { // representing the high, center value of the error bar. PXYs plot.XYs - // Line is the style used to draw the error bars. - Line plot.LineStyle - - // CapWidth is the width of the caps drawn at the top of each error bar. - CapWidth units.Value + // Style is the style for plotting. + Style plot.Style stylers plot.Stylers } func (eb *YErrorBars) Defaults() { - eb.Line.Defaults() - eb.CapWidth.Dp(10) + eb.Style.Defaults() } // NewYErrorBars returns a new YErrorBars plotter, or an error on failure. @@ -104,36 +99,36 @@ func NewYErrorBars(yerrs interface { } // Styler adds a style function to set style parameters. -func (e *YErrorBars) Styler(f func(s *YErrorBars)) *YErrorBars { - e.stylers.Add(func(p plot.Plotter) { f(p.(*YErrorBars)) }) - return e +func (eb *YErrorBars) Styler(f func(s *plot.Style)) *YErrorBars { + eb.stylers.Add(f) + return eb } -func (e *YErrorBars) ApplyStyle() { e.stylers.Run(e) } +func (eb *YErrorBars) ApplyStyle() { eb.stylers.Run(&eb.Style) } -func (e *YErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { - data = e.XYs - pixels = e.PXYs +func (eb *YErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { + data = eb.XYs + pixels = eb.PXYs return } // Plot implements the Plotter interface, drawing labels. -func (e *YErrorBars) Plot(plt *plot.Plot) { +func (eb *YErrorBars) Plot(plt *plot.Plot) { pc := plt.Paint uc := &pc.UnitContext - e.CapWidth.ToDots(uc) - cw := 0.5 * e.CapWidth.Dots - nv := len(e.YErrors) - e.PXYs = make(plot.XYs, nv) - e.Line.SetStroke(plt) - for i, err := range e.YErrors { - x := plt.PX(e.XYs[i].X) - ylow := plt.PY(e.XYs[i].Y - math32.Abs(err.Low)) - yhigh := plt.PY(e.XYs[i].Y + math32.Abs(err.High)) + eb.Style.Width.Cap.ToDots(uc) + cw := 0.5 * eb.Style.Width.Cap.Dots + nv := len(eb.YErrors) + eb.PXYs = make(plot.XYs, nv) + eb.Style.Line.SetStroke(plt) + for i, err := range eb.YErrors { + x := plt.PX(eb.XYs[i].X) + ylow := plt.PY(eb.XYs[i].Y - math32.Abs(err.Low)) + yhigh := plt.PY(eb.XYs[i].Y + math32.Abs(err.High)) - e.PXYs[i].X = x - e.PXYs[i].Y = yhigh + eb.PXYs[i].X = x + eb.PXYs[i].Y = yhigh pc.MoveTo(x, ylow) pc.LineTo(x, yhigh) @@ -148,12 +143,12 @@ func (e *YErrorBars) Plot(plt *plot.Plot) { } // DataRange implements the plot.DataRanger interface. -func (e *YErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - xmin, xmax = plot.Range(plot.XValues{e}) +func (eb *YErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { + xmin, xmax = plot.Range(plot.XValues{eb}) ymin = math32.Inf(1) ymax = math32.Inf(-1) - for i, err := range e.YErrors { - y := e.XYs[i].Y + for i, err := range eb.YErrors { + y := eb.XYs[i].Y ylow := y - math32.Abs(err.Low) yhigh := y + math32.Abs(err.High) ymin = math32.Min(math32.Min(math32.Min(ymin, y), ylow), yhigh) @@ -176,12 +171,8 @@ type XErrorBars struct { // representing the high, center value of the error bar. PXYs plot.XYs - // Line is the style used to draw the error bars. - Line plot.LineStyle - - // CapWidth is the width of the caps drawn at the top - // of each error bar. - CapWidth units.Value + // Style is the style for plotting. + Style plot.Style stylers plot.Stylers } @@ -217,42 +208,35 @@ func NewXErrorBars(xerrs interface { } func (eb *XErrorBars) Defaults() { - eb.Line.Defaults() - eb.CapWidth.Dp(10) + eb.Style.Defaults() } // Styler adds a style function to set style parameters. -func (e *XErrorBars) Styler(f func(s *XErrorBars)) *XErrorBars { - e.stylers.Add(func(p plot.Plotter) { f(p.(*XErrorBars)) }) - return e +func (eb *XErrorBars) Styler(f func(s *plot.Style)) *XErrorBars { + eb.stylers.Add(f) + return eb } -func (e *XErrorBars) ApplyStyle() { e.stylers.Run(e) } - -func (e *XErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { - data = e.XYs - pixels = e.PXYs - return -} +func (eb *XErrorBars) ApplyStyle() { eb.stylers.Run(&eb.Style) } // Plot implements the Plotter interface, drawing labels. -func (e *XErrorBars) Plot(plt *plot.Plot) { +func (eb *XErrorBars) Plot(plt *plot.Plot) { pc := plt.Paint uc := &pc.UnitContext - e.CapWidth.ToDots(uc) - cw := 0.5 * e.CapWidth.Dots + eb.Style.Width.Cap.ToDots(uc) + cw := 0.5 * eb.Style.Width.Cap.Dots - nv := len(e.XErrors) - e.PXYs = make(plot.XYs, nv) - e.Line.SetStroke(plt) - for i, err := range e.XErrors { - y := plt.PY(e.XYs[i].Y) - xlow := plt.PX(e.XYs[i].X - math32.Abs(err.Low)) - xhigh := plt.PX(e.XYs[i].X + math32.Abs(err.High)) + nv := len(eb.XErrors) + eb.PXYs = make(plot.XYs, nv) + eb.Style.Line.SetStroke(plt) + for i, err := range eb.XErrors { + y := plt.PY(eb.XYs[i].Y) + xlow := plt.PX(eb.XYs[i].X - math32.Abs(err.Low)) + xhigh := plt.PX(eb.XYs[i].X + math32.Abs(err.High)) - e.PXYs[i].X = xhigh - e.PXYs[i].Y = y + eb.PXYs[i].X = xhigh + eb.PXYs[i].Y = y pc.MoveTo(xlow, y) pc.LineTo(xhigh, y) @@ -267,12 +251,12 @@ func (e *XErrorBars) Plot(plt *plot.Plot) { } // DataRange implements the plot.DataRanger interface. -func (e *XErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - ymin, ymax = plot.Range(plot.YValues{e}) +func (eb *XErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { + ymin, ymax = plot.Range(plot.YValues{eb}) xmin = math32.Inf(1) xmax = math32.Inf(-1) - for i, err := range e.XErrors { - x := e.XYs[i].X + for i, err := range eb.XErrors { + x := eb.XYs[i].X xlow := x - math32.Abs(err.Low) xhigh := x + math32.Abs(err.High) xmin = math32.Min(math32.Min(math32.Min(xmin, x), xlow), xhigh) diff --git a/plot/plots/labels.go b/plot/plots/labels.go index 20ce2badd7..2d5467f16e 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -10,7 +10,6 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" - "cogentcore.org/core/styles/units" ) // Labels implements the Plotter interface, @@ -25,17 +24,11 @@ type Labels struct { // Labels is the set of labels corresponding to each point. Labels []string - // TextStyle is the style of the label text. - // Each label can have a different text style, but - // by default they share a common one (len = 1) - TextStyle []plot.TextStyle - - // Offset is added directly to the final label location. - Offset units.XY + // Style is the style of the label text. + Style plot.TextStyle // plot size and number of TextStyle when styles last generated -- don't regen styleSize image.Point - styleN int stylers plot.Stylers } @@ -56,99 +49,69 @@ func NewLabels(d XYLabeler) *Labels { strs[i] = d.Label(i) } - styles := make([]plot.TextStyle, 1) - for i := range styles { - styles[i].Defaults() - } - - return &Labels{ - XYs: xys, - Labels: strs, - TextStyle: styles, - } + lb := &Labels{XYs: xys, Labels: strs} + lb.Style.Defaults() + return lb } // Styler adds a style function to set style parameters. -func (l *Labels) Styler(f func(s *Labels)) *Labels { - l.stylers.Add(func(p plot.Plotter) { f(p.(*Labels)) }) - return l +func (lb *Labels) Styler(f func(s *plot.Style)) *Labels { + lb.stylers.Add(f) + return lb } -func (l *Labels) ApplyStyle() { - l.stylers.Run(l) +func (lb *Labels) ApplyStyle() { + st := lb.stylers.NewStyle() + lb.Style = st.Text } -func (l *Labels) XYData() (data plot.XYer, pixels plot.XYer) { - data = l.XYs - pixels = l.PXYs +func (lb *Labels) XYData() (data plot.XYer, pixels plot.XYer) { + data = lb.XYs + pixels = lb.PXYs return } -// updateStyles updates the text styles and dots. -// returns true if custom styles are used per point -func (l *Labels) updateStyles(plt *plot.Plot) bool { - customStyles := len(l.TextStyle) == len(l.XYs) - if plt.Size == l.styleSize && len(l.TextStyle) == l.styleN { - return customStyles - } - l.styleSize = plt.Size - l.styleN = len(l.TextStyle) +// Plot implements the Plotter interface, drawing labels. +func (lb *Labels) Plot(plt *plot.Plot) { + ps := plot.PlotXYs(plt, lb.XYs) pc := plt.Paint uc := &pc.UnitContext - l.Offset.ToDots(uc) - for i := range l.TextStyle { - l.TextStyle[i].ToDots(uc) - } - return customStyles -} - -// Plot implements the Plotter interface, drawing labels. -func (l *Labels) Plot(plt *plot.Plot) { - ps := plot.PlotXYs(plt, l.XYs) - customStyles := l.updateStyles(plt) + lb.Style.Offset.ToDots(uc) + lb.Style.ToDots(uc) var ltxt plot.Text - for i, label := range l.Labels { + ltxt.Style = lb.Style + for i, label := range lb.Labels { if label == "" { continue } - if customStyles { - ltxt.Style = l.TextStyle[i] - } else { - ltxt.Style = l.TextStyle[0] - } ltxt.Text = label ltxt.Config(plt) tht := ltxt.PaintText.BBox.Size().Y - ltxt.Draw(plt, math32.Vec2(ps[i].X+l.Offset.X.Dots, ps[i].Y+l.Offset.Y.Dots-tht)) + ltxt.Draw(plt, math32.Vec2(ps[i].X+lb.Style.Offset.X.Dots, ps[i].Y+lb.Style.Offset.Y.Dots-tht)) } } // DataRange returns the minimum and maximum X and Y values -func (l *Labels) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - xmin, xmax, ymin, ymax = plot.XYRange(l) // first get basic numerical range +func (lb *Labels) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { + xmin, xmax, ymin, ymax = plot.XYRange(lb) // first get basic numerical range pxToData := math32.FromPoint(plt.Size) pxToData.X = (xmax - xmin) / pxToData.X pxToData.Y = (ymax - ymin) / pxToData.Y - customStyles := l.updateStyles(plt) var ltxt plot.Text - for i, label := range l.Labels { + ltxt.Style = lb.Style + for i, label := range lb.Labels { if label == "" { continue } - if customStyles { - ltxt.Style = l.TextStyle[i] - } else { - ltxt.Style = l.TextStyle[0] - } ltxt.Text = label ltxt.Config(plt) tht := pxToData.Y * ltxt.PaintText.BBox.Size().Y twd := 1.1 * pxToData.X * ltxt.PaintText.BBox.Size().X - x, y := l.XY(i) + x, y := lb.XY(i) minx := x - maxx := x + pxToData.X*l.Offset.X.Dots + twd + maxx := x + pxToData.X*lb.Style.Offset.X.Dots + twd miny := y - maxy := y + pxToData.Y*l.Offset.Y.Dots + tht // y is up here + maxy := y + pxToData.Y*lb.Style.Offset.Y.Dots + tht // y is up here xmin = min(xmin, minx) xmax = max(xmax, maxx) ymin = min(ymin, miny) @@ -171,8 +134,8 @@ type XYLabels struct { } // Label returns the label for point index i. -func (l XYLabels) Label(i int) string { - return l.Labels[i] +func (lb XYLabels) Label(i int) string { + return lb.Labels[i] } var _ XYLabeler = (*XYLabels)(nil) diff --git a/plot/plots/line.go b/plot/plots/line.go index f5a40dbf86..e070534874 100644 --- a/plot/plots/line.go +++ b/plot/plots/line.go @@ -12,33 +12,14 @@ package plots //go:generate core generate import ( - "image" - "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" "cogentcore.org/core/plot" "cogentcore.org/core/tensor" ) -// StepKind specifies a form of a connection of two consecutive points. -type StepKind int32 //enums:enum - -const ( - // NoStep connects two points by simple line - NoStep StepKind = iota - - // PreStep connects two points by following lines: vertical, horizontal. - PreStep - - // MidStep connects two points by following lines: horizontal, vertical, horizontal. - // Vertical line is placed in the middle of the interval. - MidStep - - // PostStep connects two points by following lines: horizontal, vertical. - PostStep -) - -// Line implements the Plotter interface, drawing a line using XYer data. +// Line draws lines between and / or points for XY data values, +// based on Style properties. type Line struct { // XYs is a copy of the points for this line. plot.XYs @@ -46,28 +27,13 @@ type Line struct { // PXYs is the actual pixel plotting coordinates for each XY value. PXYs plot.XYs - // StepStyle is the kind of the step line. - StepStyle StepKind - - // Line is the style of the line connecting the points. - // Use zero width to disable lines. - Line plot.LineStyle - - // Fill is the color to fill the area below the plot. - // Use nil to disable filling, which is the default. - Fill image.Image - - // if true, draw lines that connect points with a negative X-axis direction; - // otherwise there is a break in the line. - // default is false, so that repeated series of data across the X axis - // are plotted separately. - NegativeXDraw bool + // Style is the style for plotting. + Style plot.Style stylers plot.Stylers } -// NewLine returns a Line that uses the default line style and -// does not draw glyphs. +// NewLine returns a Line plot element. func NewLine(xys plot.XYer) *Line { data, err := plot.CopyXYs(xys) if errors.Log(err) != nil { @@ -78,32 +44,23 @@ func NewLine(xys plot.XYer) *Line { return ln } -// NewLinePoints returns both a Line and a -// Scatter plot for the given point data. -func NewLinePoints(xys plot.XYer) (*Line, *Scatter) { - sc := NewScatter(xys) - ln := &Line{XYs: sc.XYs} - ln.Defaults() - return ln, sc -} - -// NewLineTensor returns a Line that uses the default line style and -// does not draw glyphs, based on two tensors for X, Y values. +// NewLineTensor returns a Line plot element +// using two tensors for X, Y values. func NewLineTensor(x, y tensor.Tensor) *Line { return NewLine(plot.TensorXYs{X: x, Y: y}) } func (ln *Line) Defaults() { - ln.Line.Defaults() + ln.Style.Defaults() } // Styler adds a style function to set style parameters. -func (ln *Line) Styler(f func(s *Line)) *Line { - ln.stylers.Add(func(p plot.Plotter) { f(p.(*Line)) }) +func (ln *Line) Styler(f func(s *plot.Style)) *Line { + ln.stylers.Add(f) return ln } -func (ln *Line) ApplyStyle() { ln.stylers.Run(ln) } +func (ln *Line) ApplyStyle() { ln.stylers.Run(&ln.Style) } func (ln *Line) XYData() (data plot.XYer, pixels plot.XYer) { data = ln.XYs @@ -119,22 +76,22 @@ func (ln *Line) Plot(plt *plot.Plot) { np := len(ps) ln.PXYs = ps - if ln.Fill != nil { - pc.FillStyle.Color = ln.Fill + if ln.Style.Line.Fill != nil { + pc.FillStyle.Color = ln.Style.Line.Fill minY := plt.PY(plt.Y.Min) prev := math32.Vec2(ps[0].X, minY) pc.MoveTo(prev.X, prev.Y) for i := range ps { pt := ps[i] - switch ln.StepStyle { - case NoStep: + switch ln.Style.Line.Step { + case plot.NoStep: if pt.X < prev.X { pc.LineTo(prev.X, minY) pc.ClosePath() pc.MoveTo(pt.X, minY) } pc.LineTo(pt.X, pt.Y) - case PreStep: + case plot.PreStep: if i == 0 { continue } @@ -146,7 +103,7 @@ func (ln *Line) Plot(plt *plot.Plot) { pc.LineTo(prev.X, pt.Y) } pc.LineTo(pt.X, pt.Y) - case MidStep: + case plot.MidStep: if pt.X < prev.X { pc.LineTo(prev.X, minY) pc.ClosePath() @@ -156,7 +113,7 @@ func (ln *Line) Plot(plt *plot.Plot) { pc.LineTo(0.5*(prev.X+pt.X), pt.Y) } pc.LineTo(pt.X, pt.Y) - case PostStep: + case plot.PostStep: if pt.X < prev.X { pc.LineTo(prev.X, minY) pc.ClosePath() @@ -174,36 +131,42 @@ func (ln *Line) Plot(plt *plot.Plot) { } pc.FillStyle.Color = nil - if !ln.Line.SetStroke(plt) { - return - } - prev := ps[0] - pc.MoveTo(prev.X, prev.Y) - for i := 1; i < np; i++ { - pt := ps[i] - if ln.StepStyle != NoStep { - if pt.X >= prev.X { - switch ln.StepStyle { - case PreStep: - pc.LineTo(prev.X, pt.Y) - case MidStep: - pc.LineTo(0.5*(prev.X+pt.X), prev.Y) - pc.LineTo(0.5*(prev.X+pt.X), pt.Y) - case PostStep: - pc.LineTo(pt.X, prev.Y) + if ln.Style.Line.SetStroke(plt) { + prev := ps[0] + pc.MoveTo(prev.X, prev.Y) + for i := 1; i < np; i++ { + pt := ps[i] + if ln.Style.Line.Step != plot.NoStep { + if pt.X >= prev.X { + switch ln.Style.Line.Step { + case plot.PreStep: + pc.LineTo(prev.X, pt.Y) + case plot.MidStep: + pc.LineTo(0.5*(prev.X+pt.X), prev.Y) + pc.LineTo(0.5*(prev.X+pt.X), pt.Y) + case plot.PostStep: + pc.LineTo(pt.X, prev.Y) + } + } else { + pc.MoveTo(pt.X, pt.Y) } - } else { + } + if !ln.Style.Line.NegativeX && pt.X < prev.X { pc.MoveTo(pt.X, pt.Y) + } else { + pc.LineTo(pt.X, pt.Y) } + prev = pt } - if !ln.NegativeXDraw && pt.X < prev.X { - pc.MoveTo(pt.X, pt.Y) - } else { - pc.LineTo(pt.X, pt.Y) + pc.Stroke() + } + if ln.Style.Point.SetStroke(plt) { + for i := range ps { + pt := ps[i] + ln.Style.Point.DrawShape(pc, math32.Vec2(pt.X, pt.Y)) } - prev = pt } - pc.Stroke() + pc.FillStyle.Color = nil } // DataRange returns the minimum and maximum @@ -218,17 +181,23 @@ func (ln *Line) Thumbnail(plt *plot.Plot) { ptb := pc.Bounds midY := 0.5 * float32(ptb.Min.Y+ptb.Max.Y) - if ln.Fill != nil { + if ln.Style.Line.Fill != nil { tb := ptb - if ln.Line.Width.Value > 0 { + if ln.Style.Line.Width.Value > 0 { tb.Min.Y = int(midY) } - pc.FillBox(math32.FromPoint(tb.Min), math32.FromPoint(tb.Size()), ln.Fill) + pc.FillBox(math32.FromPoint(tb.Min), math32.FromPoint(tb.Size()), ln.Style.Line.Fill) } - if ln.Line.SetStroke(plt) { + if ln.Style.Line.SetStroke(plt) { pc.MoveTo(float32(ptb.Min.X), midY) pc.LineTo(float32(ptb.Max.X), midY) pc.Stroke() } + + if ln.Style.Point.SetStroke(plt) { + midX := 0.5 * float32(ptb.Min.X+ptb.Max.X) + ln.Style.Point.DrawShape(pc, math32.Vec2(midX, midY)) + } + pc.FillStyle.Color = nil } diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 76ad1633b6..ebe1c80adc 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -30,9 +30,9 @@ func ExampleLine() { } plt := plot.New() - plt.Add(NewLine(data).Styler(func(ln *Line) { - ln.Line.Color = colors.Uniform(colors.Red) - ln.Line.Width.Pt(2) + plt.Add(NewLine(data).Styler(func(s *plot.Style) { + s.Line.Color = colors.Uniform(colors.Red) + s.Line.Width.Pt(2) })) plt.Draw() imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") @@ -78,61 +78,61 @@ func TestLine(t *testing.T) { plt.Draw() imagex.Assert(t, plt.Pixels, "line.png") - l1.Fill = colors.Uniform(colors.Yellow) + l1.Style.Line.Fill = colors.Uniform(colors.Yellow) plt.Draw() imagex.Assert(t, plt.Pixels, "line-fill.png") - l1.StepStyle = PreStep + l1.Style.Line.Step = plot.PreStep plt.Draw() imagex.Assert(t, plt.Pixels, "line-prestep.png") - l1.StepStyle = MidStep + l1.Style.Line.Step = plot.MidStep plt.Draw() imagex.Assert(t, plt.Pixels, "line-midstep.png") - l1.StepStyle = PostStep + l1.Style.Line.Step = plot.PostStep plt.Draw() imagex.Assert(t, plt.Pixels, "line-poststep.png") - l1.StepStyle = NoStep - l1.Fill = nil - l1.NegativeXDraw = true + l1.Style.Line.Step = plot.NoStep + l1.Style.Line.Fill = nil + l1.Style.Line.NegativeX = true plt.Draw() imagex.Assert(t, plt.Pixels, "line-negx.png") } -func TestScatter(t *testing.T) { - plt := plot.New() - plt.Title.Text = "Test Scatter" - plt.X.Min = 0 - plt.X.Max = 100 - plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 - plt.Y.Label.Text = "Y Axis" - - data := make(plot.XYs, 21) - for i := range data { - data[i].X = float32(i * 5) - data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) - } - - l1 := NewScatter(data) - if l1 == nil { - t.Error("bad data") - } - plt.Add(l1) - - plt.Resize(image.Point{640, 480}) - - shs := ShapesValues() - for _, sh := range shs { - l1.PointShape = sh - plt.Draw() - imagex.Assert(t, plt.Pixels, "scatter-"+sh.String()+".png") - } -} +// func TestScatter(t *testing.T) { +// plt := plot.New() +// plt.Title.Text = "Test Scatter" +// plt.X.Min = 0 +// plt.X.Max = 100 +// plt.X.Label.Text = "X Axis" +// plt.Y.Min = 0 +// plt.Y.Max = 100 +// plt.Y.Label.Text = "Y Axis" +// +// data := make(plot.XYs, 21) +// for i := range data { +// data[i].X = float32(i * 5) +// data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) +// } +// +// l1 := NewScatter(data) +// if l1 == nil { +// t.Error("bad data") +// } +// plt.Add(l1) +// +// plt.Resize(image.Point{640, 480}) +// +// shs := ShapesValues() +// for _, sh := range shs { +// l1.PointShape = sh +// plt.Draw() +// imagex.Assert(t, plt.Pixels, "scatter-"+sh.String()+".png") +// } +// } func TestLabels(t *testing.T) { plt := plot.New() @@ -150,20 +150,19 @@ func TestLabels(t *testing.T) { labels[i] = fmt.Sprintf("%7.4g", data[i].Y) } - l1, sc := NewLinePoints(data) - if l1 == nil || sc == nil { + l1 := NewLine(data) + if l1 == nil { t.Error("bad data") } plt.Add(l1) - plt.Add(sc) - plt.Legend.Add("Sine", l1, sc) + plt.Legend.Add("Sine", l1) l2 := NewLabels(XYLabels{XYs: data, Labels: labels}) if l2 == nil { t.Error("bad data") } - l2.Offset.X.Dp(6) - l2.Offset.Y.Dp(-6) + l2.Style.Offset.X.Dp(6) + l2.Style.Offset.Y.Dp(-6) plt.Add(l2) plt.Resize(image.Point{640, 480}) @@ -195,7 +194,7 @@ func TestBarChart(t *testing.T) { if l1 == nil { t.Error("bad data") } - l1.Color = colors.Uniform(colors.Red) + l1.Style.Line.Fill = colors.Uniform(colors.Red) plt.Add(l1) plt.Legend.Add("Sine", l1) @@ -207,12 +206,12 @@ func TestBarChart(t *testing.T) { if l2 == nil { t.Error("bad data") } - l2.Color = colors.Uniform(colors.Blue) + l2.Style.Line.Fill = colors.Uniform(colors.Blue) plt.Legend.Add("Cosine", l2) - l1.Stride = 2 - l2.Stride = 2 - l2.Offset = 2 + l1.Style.Width.Stride = 2 + l2.Style.Width.Stride = 2 + l2.Style.Width.Offset = 2 plt.Add(l2) // note: range updated when added! plt.Draw() @@ -243,7 +242,7 @@ func TestBarChartErr(t *testing.T) { if l1 == nil { t.Error("bad data") } - l1.Color = colors.Uniform(colors.Red) + l1.Style.Line.Fill = colors.Uniform(colors.Red) plt.Add(l1) plt.Legend.Add("Sine", l1) @@ -283,7 +282,7 @@ func TestBarChartStack(t *testing.T) { if l1 == nil { t.Error("bad data") } - l1.Color = colors.Uniform(colors.Red) + l1.Style.Line.Fill = colors.Uniform(colors.Red) plt.Add(l1) plt.Legend.Add("Sine", l1) @@ -291,7 +290,7 @@ func TestBarChartStack(t *testing.T) { if l2 == nil { t.Error("bad data") } - l2.Color = colors.Uniform(colors.Blue) + l2.Style.Line.Fill = colors.Uniform(colors.Blue) l2.StackedOn = l1 plt.Add(l2) plt.Legend.Add("Cos", l2) diff --git a/plot/plots/scatter.go b/plot/plots/scatter.go deleted file mode 100644 index 400896e024..0000000000 --- a/plot/plots/scatter.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Adapted from github.com/gonum/plot: -// Copyright ©2015 The Gonum Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package plots - -import ( - "cogentcore.org/core/base/errors" - "cogentcore.org/core/math32" - "cogentcore.org/core/plot" - "cogentcore.org/core/styles/units" -) - -// Scatter implements the Plotter interface, drawing -// a shape for each point. -type Scatter struct { - // XYs is a copy of the points for this scatter. - plot.XYs - - // PXYs is the actual plotting coordinates for each XY value. - PXYs plot.XYs - - // size of shape to draw for each point - PointSize units.Value - - // shape to draw for each point - PointShape Shapes - - // Line is the style of the line connecting the points. - // Use zero width to disable lines. - Line plot.LineStyle - - stylers plot.Stylers -} - -// NewScatter returns a Scatter that uses the -// default glyph style. -func NewScatter(xys plot.XYer) *Scatter { - data, err := plot.CopyXYs(xys) - if errors.Log(err) != nil { - return nil - } - sc := &Scatter{XYs: data} - sc.Line.Defaults() - sc.PointSize.Pt(4) - return sc -} - -func (sc *Scatter) Style(f func(s *Scatter)) *Scatter { - sc.stylers.Add(func(p plot.Plotter) { f(p.(*Scatter)) }) - return sc -} - -func (sc *Scatter) ApplyStyle() { - sc.stylers.Run(sc) -} - -func (sc *Scatter) XYData() (data plot.XYer, pixels plot.XYer) { - data = sc.XYs - pixels = sc.PXYs - return -} - -// Plot draws the Line, implementing the plot.Plotter interface. -func (sc *Scatter) Plot(plt *plot.Plot) { - pc := plt.Paint - if !sc.Line.SetStroke(plt) { - return - } - sc.PointSize.ToDots(&pc.UnitContext) - pc.FillStyle.Color = sc.Line.Color - ps := plot.PlotXYs(plt, sc.XYs) - for i := range ps { - pt := ps[i] - DrawShape(pc, math32.Vec2(pt.X, pt.Y), sc.PointSize.Dots, sc.PointShape) - } - pc.FillStyle.Color = nil -} - -// DataRange returns the minimum and maximum -// x and y values, implementing the plot.DataRanger interface. -func (sc *Scatter) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - return plot.XYRange(sc) -} - -// Thumbnail the thumbnail for the Scatter, -// implementing the plot.Thumbnailer interface. -func (sc *Scatter) Thumbnail(plt *plot.Plot) { - if !sc.Line.SetStroke(plt) { - return - } - pc := plt.Paint - sc.PointSize.ToDots(&pc.UnitContext) - pc.FillStyle.Color = sc.Line.Color - ptb := pc.Bounds - midX := 0.5 * float32(ptb.Min.X+ptb.Max.X) - midY := 0.5 * float32(ptb.Min.Y+ptb.Max.Y) - - DrawShape(pc, math32.Vec2(midX, midY), sc.PointSize.Dots, sc.PointShape) -} diff --git a/plot/plots/table.go b/plot/plots/table.go index 6a24918052..69acf026fa 100644 --- a/plot/plots/table.go +++ b/plot/plots/table.go @@ -62,15 +62,3 @@ func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *Line { plt.Add(ln) return ln } - -// AddTableLinePoints adds Line w/ Points with given x, y columns from given tabular data -func AddTableLinePoints(plt *plot.Plot, tab Table, xcolumn, ycolumn int) (*Line, *Scatter) { - txy := &TableXYer{Table: tab, XColumn: xcolumn, YColumn: ycolumn} - ln, sc := NewLinePoints(txy) - if ln == nil || sc == nil { - return nil, nil - } - plt.Add(ln) - plt.Add(sc) - return ln, sc -} diff --git a/plot/plotter.go b/plot/plotter.go index e98fdcb0d5..b888b8b1c7 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -15,7 +15,7 @@ type Plotter interface { // This allows gui interface to inspect data etc. XYData() (data XYer, pixels XYer) - // ApplyStyle runs the style functions. + // ApplyStyle applies any stylers to this element. ApplyStyle() } @@ -24,16 +24,3 @@ type DataRanger interface { // DataRange returns the range of X and Y values. DataRange(pt *Plot) (xmin, xmax, ymin, ymax float32) } - -// Stylers implements the styling functions for plotters. -type Stylers []func(p Plotter) - -func (st *Stylers) Add(f func(p Plotter)) { - *st = append(*st, f) -} - -func (st *Stylers) Run(p Plotter) { - for _, f := range *st { - f(p) - } -} diff --git a/plot/plots/shapes.go b/plot/point.go similarity index 62% rename from plot/plots/shapes.go rename to plot/point.go index 3d6252343d..3a94ee99ab 100644 --- a/plot/plots/shapes.go +++ b/plot/point.go @@ -2,44 +2,76 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package plots +package plot import ( + "image" + + "cogentcore.org/core/colors" "cogentcore.org/core/math32" "cogentcore.org/core/paint" + "cogentcore.org/core/styles/units" ) -type Shapes int32 //enums:enum +// PointStyle has style properties for drawing points as different shapes. +type PointStyle struct { //types:add -setters + // On indicates whether to plot points. + On DefaultOffOn -const ( - // Ring is the outline of a circle - Ring Shapes = iota - - // Circle is a solid circle - Circle + // Shape to draw. + Shape Shapes - // Square is the outline of a square - Square + // Color is the stroke color image specification. + // Setting to nil turns line off. + Color image.Image - // Box is a filled square - Box + // Fill is the color to fill solid regions, in a plot-specific + // way (e.g., the area below a Line plot, the bar color). + // Use nil to disable filling. + Fill image.Image - // Triangle is the outline of a triangle - Triangle + // Width is the line width, with a default of 1 Pt (point). + // Setting to 0 turns line off. + Width units.Value - // Pyramid is a filled triangle - Pyramid + // Size of shape to draw for each point. + // Defaults to 4 Pt (point). + Size units.Value +} - // Plus is a plus sign - Plus +func (ps *PointStyle) Defaults() { + ps.Color = colors.Scheme.OnSurface + ps.Width.Pt(1) + ps.Size.Pt(4) +} - // Cross is a big X - Cross -) +// SetStroke sets the stroke style in plot paint to current line style. +// returns false if either the Width = 0 or Color is nil +func (ps *PointStyle) SetStroke(pt *Plot) bool { + if ps.On == Off || ps.Color == nil { + return false + } + pc := pt.Paint + uc := &pc.UnitContext + ps.Width.ToDots(uc) + ps.Size.ToDots(uc) + if ps.Width.Dots == 0 || ps.Size.Dots == 0 { + return false + } + pc.StrokeStyle.Width = ps.Width + pc.StrokeStyle.Color = ps.Color + pc.StrokeStyle.ToDots(uc) + pc.FillStyle.Color = ps.Fill + return true +} // DrawShape draws the given shape -func DrawShape(pc *paint.Context, pos math32.Vector2, size float32, shape Shapes) { - switch shape { +func (ps *PointStyle) DrawShape(pc *paint.Context, pos math32.Vector2) { + size := ps.Size.Dots + if size == 0 { + return + } + switch ps.Shape { case Ring: DrawRing(pc, pos, size) case Circle: @@ -126,3 +158,31 @@ func DrawCross(pc *paint.Context, pos math32.Vector2, size float32) { pc.ClosePath() pc.Stroke() } + +type Shapes int32 //enums:enum + +const ( + // Ring is the outline of a circle + Ring Shapes = iota + + // Circle is a solid circle + Circle + + // Square is the outline of a square + Square + + // Box is a filled square + Box + + // Triangle is the outline of a triangle + Triangle + + // Pyramid is a filled triangle + Pyramid + + // Plus is a plus sign + Plus + + // Cross is a big X + Cross +) diff --git a/plot/style.go b/plot/style.go new file mode 100644 index 0000000000..cac64f3c64 --- /dev/null +++ b/plot/style.go @@ -0,0 +1,128 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package plot + +import ( + "cogentcore.org/core/math32/minmax" + "cogentcore.org/core/styles/units" +) + +// Style contains the plot styling properties relevant across +// most plot types. These properties apply both to individual plot elements +// and to the plot as a whole. +type Style struct { //types:add -setters + // On specifies whether to plot this item, for cases where it can be turned off. + On DefaultOffOn + + // Range is the effective range of data to plot, where either end can be fixed. + Range minmax.Range32 `display:"inline"` + + // Label provides an alternative label to use for axis, if set. + Label string + + // NTicks sets the desired number of ticks for the axis, if > 0. + NTicks int + + // Line has style properties for drawing lines. + Line LineStyle + + // Point has style properties for drawing points. + Point PointStyle + + // Text has style properties for rendering text. + Text TextStyle + + // Width has various plot width properties. + Width WidthStyle +} + +// NewStyle returns a new Style object with defaults applied. +func NewStyle() *Style { + st := &Style{} + st.Defaults() + return st +} + +func (st *Style) Defaults() { + st.Line.Defaults() + st.Point.Defaults() + st.Text.Defaults() + st.Width.Defaults() +} + +// WidthStyle contains various plot width properties relevant across +// different plot types. +type WidthStyle struct { //types:add -setters + // Cap is the width of the caps drawn at the top of error bars. + // The default is 10dp + Cap units.Value + + // Offset for Bar plot is the offset added to each X axis value + // relative to the Stride computed value (X = offset + index * Stride) + // Defaults to 1. + Offset float32 + + // Stride for Bar plot is distance between bars. Defaults to 1. + Stride float32 + + // Width for Bar plot is the width of the bars, which should be less than + // the Stride to prevent bar overlap. + // Defaults to .8 + Width float32 + + // Pad for Bar plot is additional space at start / end of data range, + // to keep bars from overflowing ends. This amount is subtracted from Offset + // and added to (len(Values)-1)*Stride -- no other accommodation for bar + // width is provided, so that should be built into this value as well. + // Defaults to 1. + Pad float32 +} + +func (ws *WidthStyle) Defaults() { + ws.Cap.Dp(10) + ws.Offset = 1 + ws.Stride = 1 + ws.Width = .8 + ws.Pad = 1 +} + +// Stylers is a list of styling functions that set Style properties. +// These are called in the order added. +type Stylers []func(s *Style) + +// Add Adds a styling function to the list. +func (st *Stylers) Add(f func(s *Style)) { + *st = append(*st, f) +} + +// Run runs the list of styling functions on given [Style] object. +func (st *Stylers) Run(s *Style) { + for _, f := range *st { + f(s) + } +} + +// NewStyle returns a new Style object with styling functions applied +// on top of Style defaults. +func (st *Stylers) NewStyle() *Style { + s := NewStyle() + st.Run(s) + return s +} + +// DefaultOffOn specifies whether to use the default value for a bool option, +// or to override the default and set Off or On. +type DefaultOffOn int32 //enums:enum + +const ( + // Default means use the default value. + Default DefaultOffOn = iota + + // Off means to override the default and turn Off. + Off + + // On means to override the default and turn On. + On +) diff --git a/plot/text.go b/plot/text.go index cb86017a65..5c14245168 100644 --- a/plot/text.go +++ b/plot/text.go @@ -28,6 +28,9 @@ type TextStyle struct { // rotation of the text, in Degrees Rotation float32 + + // Offset is added directly to the final label location. + Offset units.XY } func (ts *TextStyle) Defaults() { diff --git a/plot/typegen.go b/plot/typegen.go index c076e347ab..4f5869f1a6 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -3,6 +3,10 @@ package plot import ( + "image" + + "cogentcore.org/core/math32/minmax" + "cogentcore.org/core/styles/units" "cogentcore.org/core/types" ) @@ -20,10 +24,14 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Valuer", IDNam var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Values", IDName: "values", Doc: "Values implements the Valuer interface."}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TensorValues", IDName: "tensor-values", Doc: "TensorValues provides a Valuer interface wrapper for a tensor.", Embeds: []types.Field{{Name: "Tensor"}}}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYer", IDName: "x-yer", Doc: "XYer provides an interface for a list of X,Y data pairs", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of x, y pairs.", Returns: []string{"int"}}, {Name: "XY", Doc: "XY returns an x, y pair.", Args: []string{"i"}, Returns: []string{"x", "y"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYs", IDName: "x-ys", Doc: "XYs implements the XYer interface."}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TensorXYs", IDName: "tensor-x-ys", Doc: "TensorXYs provides a XYer interface wrapper for a tensor.", Fields: []types.Field{{Name: "X"}, {Name: "Y"}}}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XValues", IDName: "x-values", Doc: "XValues implements the Valuer interface,\nreturning the x value from an XYer.", Embeds: []types.Field{{Name: "XYer"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.YValues", IDName: "y-values", Doc: "YValues implements the Valuer interface,\nreturning the y value from an XYer.", Embeds: []types.Field{{Name: "XYer"}}}) @@ -50,14 +58,154 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Thumbnailer", var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LegendEntry", IDName: "legend-entry", Doc: "A LegendEntry represents a single line of a legend, it\nhas a name and an icon.", Fields: []types.Field{{Name: "Text", Doc: "text is the text associated with this entry."}, {Name: "Thumbs", Doc: "thumbs is a slice of all of the thumbnails styles"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LineStyle", IDName: "line-style", Doc: "LineStyle has style properties for line drawing", Fields: []types.Field{{Name: "Color", Doc: "stroke color image specification; stroking is off if nil"}, {Name: "Width", Doc: "line width"}, {Name: "Dashes", Doc: "Dashes are the dashes of the stroke. Each pair of values specifies\nthe amount to paint and then the amount to skip."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LineStyle", IDName: "line-style", Doc: "LineStyle has style properties for drawing lines.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On indicates whether to plot lines."}, {Name: "Color", Doc: "Color is the stroke color image specification.\nSetting to nil turns line off."}, {Name: "Width", Doc: "Width is the line width, with a default of 1 Pt (point).\nSetting to 0 turns line off."}, {Name: "Dashes", Doc: "Dashes are the dashes of the stroke. Each pair of values specifies\nthe amount to paint and then the amount to skip."}, {Name: "Fill", Doc: "Fill is the color to fill solid regions, in a plot-specific\nway (e.g., the area below a Line plot, the bar color).\nUse nil to disable filling."}, {Name: "NegativeX", Doc: "NegativeX specifies whether to draw lines that connect points with a negative\nX-axis direction; otherwise there is a break in the line.\ndefault is false, so that repeated series of data across the X axis\nare plotted separately."}, {Name: "Step", Doc: "Step specifies how to step the line between points."}}}) + +// SetOn sets the [LineStyle.On]: +// On indicates whether to plot lines. +func (t *LineStyle) SetOn(v DefaultOffOn) *LineStyle { t.On = v; return t } + +// SetColor sets the [LineStyle.Color]: +// Color is the stroke color image specification. +// Setting to nil turns line off. +func (t *LineStyle) SetColor(v image.Image) *LineStyle { t.Color = v; return t } + +// SetWidth sets the [LineStyle.Width]: +// Width is the line width, with a default of 1 Pt (point). +// Setting to 0 turns line off. +func (t *LineStyle) SetWidth(v units.Value) *LineStyle { t.Width = v; return t } + +// SetDashes sets the [LineStyle.Dashes]: +// Dashes are the dashes of the stroke. Each pair of values specifies +// the amount to paint and then the amount to skip. +func (t *LineStyle) SetDashes(v ...float32) *LineStyle { t.Dashes = v; return t } + +// SetFill sets the [LineStyle.Fill]: +// Fill is the color to fill solid regions, in a plot-specific +// way (e.g., the area below a Line plot, the bar color). +// Use nil to disable filling. +func (t *LineStyle) SetFill(v image.Image) *LineStyle { t.Fill = v; return t } + +// SetNegativeX sets the [LineStyle.NegativeX]: +// NegativeX specifies whether to draw lines that connect points with a negative +// X-axis direction; otherwise there is a break in the line. +// default is false, so that repeated series of data across the X axis +// are plotted separately. +func (t *LineStyle) SetNegativeX(v bool) *LineStyle { t.NegativeX = v; return t } + +// SetStep sets the [LineStyle.Step]: +// Step specifies how to step the line between points. +func (t *LineStyle) SetStep(v StepKind) *LineStyle { t.Step = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.StepKind", IDName: "step-kind", Doc: "StepKind specifies a form of a connection of two consecutive points."}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.\nThe Axis ranges are updated automatically when plots\nare added, so setting a fixed range should happen\nafter that point. See [UpdateRange] method as well.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Y", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nSome standard implementations of Plotter can be found in plotters.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nSome standard implementations of Plotter can be found in plotters.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DataRanger", IDName: "data-ranger", Doc: "DataRanger wraps the DataRange method.", Methods: []types.Method{{Name: "DataRange", Doc: "DataRange returns the range of X and Y values.", Args: []string{"pt"}, Returns: []string{"xmin", "xmax", "ymin", "ymax"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PointStyle", IDName: "point-style", Doc: "PointStyle has style properties for drawing points as different shapes.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On indicates whether to plot points."}, {Name: "Shape", Doc: "Shape to draw."}, {Name: "Color", Doc: "Color is the stroke color image specification.\nSetting to nil turns line off."}, {Name: "Fill", Doc: "Fill is the color to fill solid regions, in a plot-specific\nway (e.g., the area below a Line plot, the bar color).\nUse nil to disable filling."}, {Name: "Width", Doc: "Width is the line width, with a default of 1 Pt (point).\nSetting to 0 turns line off."}, {Name: "Size", Doc: "Size of shape to draw for each point.\nDefaults to 4 Pt (point)."}}}) + +// SetOn sets the [PointStyle.On]: +// On indicates whether to plot points. +func (t *PointStyle) SetOn(v DefaultOffOn) *PointStyle { t.On = v; return t } + +// SetShape sets the [PointStyle.Shape]: +// Shape to draw. +func (t *PointStyle) SetShape(v Shapes) *PointStyle { t.Shape = v; return t } + +// SetColor sets the [PointStyle.Color]: +// Color is the stroke color image specification. +// Setting to nil turns line off. +func (t *PointStyle) SetColor(v image.Image) *PointStyle { t.Color = v; return t } + +// SetFill sets the [PointStyle.Fill]: +// Fill is the color to fill solid regions, in a plot-specific +// way (e.g., the area below a Line plot, the bar color). +// Use nil to disable filling. +func (t *PointStyle) SetFill(v image.Image) *PointStyle { t.Fill = v; return t } + +// SetWidth sets the [PointStyle.Width]: +// Width is the line width, with a default of 1 Pt (point). +// Setting to 0 turns line off. +func (t *PointStyle) SetWidth(v units.Value) *PointStyle { t.Width = v; return t } + +// SetSize sets the [PointStyle.Size]: +// Size of shape to draw for each point. +// Defaults to 4 Pt (point). +func (t *PointStyle) SetSize(v units.Value) *PointStyle { t.Size = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Shapes", IDName: "shapes"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply both to individual plot elements\nand to the plot as a whole.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On specifies whether to plot this item, for cases where it can be turned off."}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) + +// SetOn sets the [Style.On]: +// On specifies whether to plot this item, for cases where it can be turned off. +func (t *Style) SetOn(v DefaultOffOn) *Style { t.On = v; return t } + +// SetRange sets the [Style.Range]: +// Range is the effective range of data to plot, where either end can be fixed. +func (t *Style) SetRange(v minmax.Range32) *Style { t.Range = v; return t } + +// SetLabel sets the [Style.Label]: +// Label provides an alternative label to use for axis, if set. +func (t *Style) SetLabel(v string) *Style { t.Label = v; return t } + +// SetNTicks sets the [Style.NTicks]: +// NTicks sets the desired number of ticks for the axis, if > 0. +func (t *Style) SetNTicks(v int) *Style { t.NTicks = v; return t } + +// SetLine sets the [Style.Line]: +// Line has style properties for drawing lines. +func (t *Style) SetLine(v LineStyle) *Style { t.Line = v; return t } + +// SetPoint sets the [Style.Point]: +// Point has style properties for drawing points. +func (t *Style) SetPoint(v PointStyle) *Style { t.Point = v; return t } + +// SetText sets the [Style.Text]: +// Text has style properties for rendering text. +func (t *Style) SetText(v TextStyle) *Style { t.Text = v; return t } + +// SetWidth sets the [Style.Width]: +// Width has various plot width properties. +func (t *Style) SetWidth(v WidthStyle) *Style { t.Width = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.WidthStyle", IDName: "width-style", Doc: "WidthStyle contains various plot width properties relevant across\ndifferent plot types.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Cap", Doc: "Cap is the width of the caps drawn at the top of error bars.\nThe default is 10dp"}, {Name: "Offset", Doc: "Offset for Bar plot is the offset added to each X axis value\nrelative to the Stride computed value (X = offset + index * Stride)\nDefaults to 1."}, {Name: "Stride", Doc: "Stride for Bar plot is distance between bars. Defaults to 1."}, {Name: "Width", Doc: "Width for Bar plot is the width of the bars, which should be less than\nthe Stride to prevent bar overlap.\nDefaults to .8"}, {Name: "Pad", Doc: "Pad for Bar plot is additional space at start / end of data range,\nto keep bars from overflowing ends. This amount is subtracted from Offset\nand added to (len(Values)-1)*Stride -- no other accommodation for bar\nwidth is provided, so that should be built into this value as well.\nDefaults to 1."}}}) + +// SetCap sets the [WidthStyle.Cap]: +// Cap is the width of the caps drawn at the top of error bars. +// The default is 10dp +func (t *WidthStyle) SetCap(v units.Value) *WidthStyle { t.Cap = v; return t } + +// SetOffset sets the [WidthStyle.Offset]: +// Offset for Bar plot is the offset added to each X axis value +// relative to the Stride computed value (X = offset + index * Stride) +// Defaults to 1. +func (t *WidthStyle) SetOffset(v float32) *WidthStyle { t.Offset = v; return t } + +// SetStride sets the [WidthStyle.Stride]: +// Stride for Bar plot is distance between bars. Defaults to 1. +func (t *WidthStyle) SetStride(v float32) *WidthStyle { t.Stride = v; return t } + +// SetWidth sets the [WidthStyle.Width]: +// Width for Bar plot is the width of the bars, which should be less than +// the Stride to prevent bar overlap. +// Defaults to .8 +func (t *WidthStyle) SetWidth(v float32) *WidthStyle { t.Width = v; return t } + +// SetPad sets the [WidthStyle.Pad]: +// Pad for Bar plot is additional space at start / end of data range, +// to keep bars from overflowing ends. This amount is subtracted from Offset +// and added to (len(Values)-1)*Stride -- no other accommodation for bar +// width is provided, so that should be built into this value as well. +// Defaults to 1. +func (t *WidthStyle) SetPad(v float32) *WidthStyle { t.Pad = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Stylers", IDName: "stylers", Doc: "Stylers is a list of styling functions that set Style properties.\nThese are called in the order added."}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DefaultOffOn", IDName: "default-off-on", Doc: "DefaultOffOn specifies whether to use the default value for a bool option,\nor to override the default and set Off or On."}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TextStyle", IDName: "text-style", Doc: "TextStyle specifies styling parameters for Text elements", Embeds: []types.Field{{Name: "FontRender"}}, Fields: []types.Field{{Name: "Align", Doc: "how to align text along the relevant dimension for the text element"}, {Name: "Padding", Doc: "Padding is used in a case-dependent manner to add space around text elements"}, {Name: "Rotation", Doc: "rotation of the text, in Degrees"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Text", IDName: "text", Doc: "Text specifies a single text element in a plot", Fields: []types.Field{{Name: "Text", Doc: "text string, which can use HTML formatting"}, {Name: "Style", Doc: "styling for this text element"}, {Name: "PaintText", Doc: "PaintText is the [paint.Text] for the text."}}}) diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal new file mode 100644 index 0000000000..ce75d8bcfa --- /dev/null +++ b/tensor/examples/datafs-sim/sim.goal @@ -0,0 +1,274 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +//go:generate core generate + +import ( + "math/rand/v2" + + "cogentcore.org/core/core" + "cogentcore.org/core/plot/plotcore" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/databrowser" + "cogentcore.org/core/tensor/datafs" + "cogentcore.org/core/tensor/stats/stats" +) + +type Times int32 //enums:enum + +const ( + Trial Times = iota + Epoch + Run +) + +type Sim struct { + Root *datafs.Data + Config *datafs.Data + Stats *datafs.Data + StatFuncs []func(tm Times) + Counters [TimesN]int +} + +// ConfigAll configures the sim +func (ss *Sim) ConfigAll() { + ss.Root, _ = datafs.NewDir("Root") + ss.Config, _ = ss.Root.Mkdir("Config") + mx := datafs.Value[int](ss.Config, "Max", int(TimesN)).(*tensor.Int) + mx.Set1D(5, int(Trial)) + mx.Set1D(4, int(Epoch)) + mx.Set1D(3, int(Run)) + // todo: failing - assigns 3 to all + // # mx[Trial] = 5 + // # mx[Epoch] = 4 + // # mx[Run] = 3 + ss.ConfigStats() +} + +func (ss *Sim) AddStat(f func(tm Times)) { + ss.StatFuncs = append(ss.StatFuncs, f) +} + +func (ss *Sim) RunStats(tm Times) { + for _, sf := range ss.StatFuncs { + sf(tm) + } +} + +func (ss *Sim) ConfigStats() { + ss.Stats, _ = ss.Root.Mkdir("Stats") + mx := ss.Config.Value("Max").(*tensor.Int) + ctrs := []Times{Run, Epoch, Trial} + for _, ctr := range ctrs { + ss.AddStat(func(tm Times) { + if tm > ctr { + return + } + name := ctr.String() + ctv := ss.Counters[ctr] + mxi := mx.Value1D(int(tm)) + td := ss.Stats.RecycleDir(tm.String()) + cd := ss.Stats.RecycleDir("Current") + datafs.Scalar[int](cd, name).SetInt1D(ctv, 0) + tv := datafs.Value[int](td, name, mxi) + tv.SetInt1D(ctv, ss.Counters[tm]) + if ps := plotcore.GetPlotStylers(tv.Metadata()); ps == nil { + ps = plotcore.NewPlotStylers() + ps.ColumnStyler(name, func(co *plotcore.ColumnOptions) { + co.Range.FixMin = true + co.On = false + }) + if tm == ctr { + ps.PlotStyler(func(po *plotcore.PlotOptions) { + po.XAxis = name + }) + } + plotcore.SetPlotStylers(tv.Metadata(), ps) + } + }) + } + ss.AddStat(func(tm Times) { + sseName := "SSE" + errName := "Err" + td := ss.Stats.RecycleDir(tm.String()) + switch tm { + case Trial: + ctv := ss.Counters[tm] + mxi := mx.Value1D(int(tm)) + cd := ss.Stats.RecycleDir("Current") + sse := rand.Float32() + terr := float32(1) + if sse < 0.5 { + terr = 0 + } + datafs.Scalar[float32](cd, sseName).SetFloat(float64(sse), 0) + datafs.Scalar[float32](cd, errName).SetFloat(float64(terr), 0) + datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) + datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + case Epoch: + ctv := ss.Counters[tm] + mxi := mx.Value1D(int(tm)) + trld, _ := ss.Stats.Mkdir(Trial.String()) + sse := stats.StatMean.Call(trld.Value(sseName)).Float1D(0) + terr := stats.StatMean.Call(trld.Value(errName)).Float1D(0) + datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) + datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + } + }) +} + +// // ConfigStats adds basic stats that we record for our simulation. +// func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { +// stats, _ := dir.Mkdir("Stats") +// datafs.NewScalar[int](stats, "Run", "Epoch", "Trial") // counters +// datafs.NewScalar[string](stats, "TrialName") +// datafs.NewScalar[float32](stats, "SSE", "AvgSSE", "TrlErr") +// z1, key := plotcore.PlotColumnZeroOne() +// stats.SetMetaItems(key, z1, "AvgErr", "TrlErr") +// zmax, _ := plotcore.PlotColumnZeroOne() +// zmax.Range.FixMax = false +// stats.SetMetaItems(key, z1, "SSE") +// return stats +// } +// +// // ConfigLogs adds first-level logging of stats into tensors +// func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { +// logd, _ := dir.Mkdir("Log") +// trial := ss.ConfigTrialLog(logd) +// ss.ConfigAggLog(logd, "Epoch", trial, stats.StatMean, stats.StatSem, stats.StatMin) +// return logd +// } +// +// // ConfigTrialLog adds first-level logging of stats into tensors +// func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { +// logd, _ := dir.Mkdir("Trial") +// ntrial := ss.Config.Item("NTrial").AsInt() +// sitems := ss.Stats.ValuesFunc(nil) +// for _, st := range sitems { +// nm := st.Metadata().Name() +// lt := logd.NewOfType(nm, st.DataType(), ntrial) +// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source +// tensor.SetCalcFunc(lt, func() error { +// trl := ss.Stats.Item("Trial").AsInt() +// if st.IsString() { +// lt.SetStringRow(st.String1D(0), trl) +// } else { +// lt.SetFloatRow(st.Float1D(0), trl) +// } +// return nil +// }) +// } +// alllogd, _ := dir.Mkdir("AllTrials") +// for _, st := range sitems { +// nm := st.Metadata().Name() +// // allocate full size +// lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) +// lt.SetShapeSizes(0) // then truncate to 0 +// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source +// tensor.SetCalcFunc(lt, func() error { +// row := lt.DimSize(0) +// lt.SetShapeSizes(row + 1) +// if st.IsString() { +// lt.SetStringRow(st.String1D(0), row) +// } else { +// lt.SetFloatRow(st.Float1D(0), row) +// } +// return nil +// }) +// } +// return logd +// } +// +// // ConfigAggLog adds a higher-level logging of lower-level into higher-level tensors +// func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, aggs ...stats.Stats) *datafs.Data { +// logd, _ := dir.Mkdir(level) +// sitems := ss.Stats.ValuesFunc(nil) +// nctr := ss.Config.Item("N" + level).AsInt() +// for _, st := range sitems { +// if st.IsString() { +// continue +// } +// nm := st.Metadata().Name() +// src := from.Value(nm) +// if st.DataType() >= reflect.Float32 { +// // todo: pct correct etc +// dd, _ := logd.Mkdir(nm) +// for _, ag := range aggs { // key advantage of dir structure: multiple stats per item +// lt := dd.NewOfType(ag.String(), st.DataType(), nctr) +// lt.Metadata().Copy(*st.Metadata()) +// tensor.SetCalcFunc(lt, func() error { +// stout := ag.Call(src) +// ctr := ss.Stats.Item(level).AsInt() +// lt.SetFloatRow(stout.FloatRow(0), ctr) +// return nil +// }) +// } +// } else { +// lt := logd.NewOfType(nm, st.DataType(), nctr) +// lt.Metadata().Copy(*st.Metadata()) +// tensor.SetCalcFunc(lt, func() error { +// v := st.Float1D(0) +// ctr := ss.Stats.Item(level).AsInt() +// lt.SetFloatRow(v, ctr) +// return nil +// }) +// } +// } +// return logd +// } + +func (ss *Sim) Run() { + mx := ss.Config.Value("Max").(*tensor.Int) + nrun := mx.Value1D(int(Run)) + nepc := mx.Value1D(int(Epoch)) + ntrl := mx.Value1D(int(Trial)) + for run := range nrun { + ss.Counters[Run] = run + for epc := range nepc { + ss.Counters[Epoch] = epc + for trl := range ntrl { + ss.Counters[Trial] = trl + ss.RunStats(Trial) + } + ss.RunStats(Epoch) + } + ss.RunStats(Run) + } + // alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) + // dir, _ := ss.Logs.Mkdir("Stats") + // stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") + // sts := []string{"SSE", "AvgSSE", "TrlErr"} + // stats.TableGroupStats(dir, stats.StatMean, alldt, sts...) + // stats.TableGroupStats(dir, stats.StatSem, alldt, sts...) +} + +// func (ss *Sim) RunTrial(trl int) { +// ss.Stats.Item("TrialName").SetString("Trial_" + strconv.Itoa(trl)) +// sse := rand.Float32() +// avgSSE := rand.Float32() +// ss.Stats.Item("SSE").SetFloat32(sse) +// ss.Stats.Item("AvgSSE").SetFloat32(avgSSE) +// trlErr := float32(1) +// if sse < 0.5 { +// trlErr = 0 +// } +// ss.Stats.Item("TrlErr").SetFloat32(trlErr) +// ss.Logs.Item("Trial").CalcAll() +// ss.Logs.Item("AllTrials").CalcAll() +// } + +// func (ss *Sim) EpochDone() { +// ss.Logs.Item("Epoch").CalcAll() +// } + +func main() { + ss := &Sim{} + ss.ConfigAll() + ss.Run() + + databrowser.NewBrowserWindow(ss.Root, "Root") + core.Wait() +} From 9974fe10c864429cc571bffaa89316c930e68d07 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 7 Nov 2024 00:02:48 -0800 Subject: [PATCH 265/311] XY is the common Plotter type for Line and Scatter, just sets Line vs. Point On --- plot/plots/plot_test.go | 63 ++++++++++++++++++----------------- plot/plots/table.go | 4 +-- plot/plots/{line.go => xy.go} | 53 ++++++++++++++++++++--------- plot/point.go | 1 + 4 files changed, 72 insertions(+), 49 deletions(-) rename plot/plots/{line.go => xy.go} (74%) diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index ebe1c80adc..841c514c60 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -102,37 +102,37 @@ func TestLine(t *testing.T) { } -// func TestScatter(t *testing.T) { -// plt := plot.New() -// plt.Title.Text = "Test Scatter" -// plt.X.Min = 0 -// plt.X.Max = 100 -// plt.X.Label.Text = "X Axis" -// plt.Y.Min = 0 -// plt.Y.Max = 100 -// plt.Y.Label.Text = "Y Axis" -// -// data := make(plot.XYs, 21) -// for i := range data { -// data[i].X = float32(i * 5) -// data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) -// } -// -// l1 := NewScatter(data) -// if l1 == nil { -// t.Error("bad data") -// } -// plt.Add(l1) -// -// plt.Resize(image.Point{640, 480}) -// -// shs := ShapesValues() -// for _, sh := range shs { -// l1.PointShape = sh -// plt.Draw() -// imagex.Assert(t, plt.Pixels, "scatter-"+sh.String()+".png") -// } -// } +func TestScatter(t *testing.T) { + plt := plot.New() + plt.Title.Text = "Test Scatter" + plt.X.Min = 0 + plt.X.Max = 100 + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" + + data := make(plot.XYs, 21) + for i := range data { + data[i].X = float32(i * 5) + data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) + } + + l1 := NewScatter(data) + if l1 == nil { + t.Error("bad data") + } + plt.Add(l1) + + plt.Resize(image.Point{640, 480}) + + shs := plot.ShapesValues() + for _, sh := range shs { + l1.Style.Point.Shape = sh + plt.Draw() + imagex.Assert(t, plt.Pixels, "scatter-"+sh.String()+".png") + } +} func TestLabels(t *testing.T) { plt := plot.New() @@ -154,6 +154,7 @@ func TestLabels(t *testing.T) { if l1 == nil { t.Error("bad data") } + l1.Style.Point.On = plot.On plt.Add(l1) plt.Legend.Add("Sine", l1) diff --git a/plot/plots/table.go b/plot/plots/table.go index 69acf026fa..70ab4fec38 100644 --- a/plot/plots/table.go +++ b/plot/plots/table.go @@ -52,8 +52,8 @@ func (dt *TableXYer) XY(i int) (x, y float32) { return dt.Table.PlotData(dt.XColumn, i), dt.Table.PlotData(dt.YColumn, i) } -// AddTableLine adds Line with given x, y columns from given tabular data -func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *Line { +// AddTableLine adds XY Line with given x, y columns from given tabular data. +func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *XY { txy := NewTableXYer(tab, xcolumn, ycolumn) ln := NewLine(txy) if ln == nil { diff --git a/plot/plots/line.go b/plot/plots/xy.go similarity index 74% rename from plot/plots/line.go rename to plot/plots/xy.go index e070534874..4fc79585ba 100644 --- a/plot/plots/line.go +++ b/plot/plots/xy.go @@ -18,9 +18,9 @@ import ( "cogentcore.org/core/tensor" ) -// Line draws lines between and / or points for XY data values, +// XY draws lines between and / or points for XY data values, // based on Style properties. -type Line struct { +type XY struct { // XYs is a copy of the points for this line. plot.XYs @@ -33,43 +33,64 @@ type Line struct { stylers plot.Stylers } -// NewLine returns a Line plot element. -func NewLine(xys plot.XYer) *Line { +// NewLine returns an XY plot drawing Lines by default. +func NewLine(xys plot.XYer) *XY { data, err := plot.CopyXYs(xys) if errors.Log(err) != nil { return nil } - ln := &Line{XYs: data} + ln := &XY{XYs: data} ln.Defaults() + ln.Style.Line.On = plot.On + ln.Style.Point.On = plot.Off return ln } -// NewLineTensor returns a Line plot element +// NewLineTensor returns an XY plot drawing Lines, // using two tensors for X, Y values. -func NewLineTensor(x, y tensor.Tensor) *Line { +func NewLineTensor(x, y tensor.Tensor) *XY { return NewLine(plot.TensorXYs{X: x, Y: y}) } -func (ln *Line) Defaults() { +// NewScatter returns an XY scatter plot drawing Points by default. +func NewScatter(xys plot.XYer) *XY { + data, err := plot.CopyXYs(xys) + if errors.Log(err) != nil { + return nil + } + ln := &XY{XYs: data} + ln.Defaults() + ln.Style.Line.On = plot.Off + ln.Style.Point.On = plot.On + return ln +} + +// NewScatterTensor returns an XY scatter plot drawing Points by default, +// using two tensors for X, Y values. +func NewScatterTensor(x, y tensor.Tensor) *XY { + return NewScatter(plot.TensorXYs{X: x, Y: y}) +} + +func (ln *XY) Defaults() { ln.Style.Defaults() } // Styler adds a style function to set style parameters. -func (ln *Line) Styler(f func(s *plot.Style)) *Line { +func (ln *XY) Styler(f func(s *plot.Style)) *XY { ln.stylers.Add(f) return ln } -func (ln *Line) ApplyStyle() { ln.stylers.Run(&ln.Style) } +func (ln *XY) ApplyStyle() { ln.stylers.Run(&ln.Style) } -func (ln *Line) XYData() (data plot.XYer, pixels plot.XYer) { +func (ln *XY) XYData() (data plot.XYer, pixels plot.XYer) { data = ln.XYs pixels = ln.PXYs return } -// Plot draws the Line, implementing the plot.Plotter interface. -func (ln *Line) Plot(plt *plot.Plot) { +// Plot does the drawing, implementing the plot.Plotter interface. +func (ln *XY) Plot(plt *plot.Plot) { pc := plt.Paint ps := plot.PlotXYs(plt, ln.XYs) @@ -171,12 +192,12 @@ func (ln *Line) Plot(plt *plot.Plot) { // DataRange returns the minimum and maximum // x and y values, implementing the plot.DataRanger interface. -func (ln *Line) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { +func (ln *XY) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { return plot.XYRange(ln) } -// Thumbnail returns the thumbnail for the LineTo, implementing the plot.Thumbnailer interface. -func (ln *Line) Thumbnail(plt *plot.Plot) { +// Thumbnail returns the thumbnail, implementing the plot.Thumbnailer interface. +func (ln *XY) Thumbnail(plt *plot.Plot) { pc := plt.Paint ptb := pc.Bounds midY := 0.5 * float32(ptb.Min.Y+ptb.Max.Y) diff --git a/plot/point.go b/plot/point.go index 3a94ee99ab..3b9269553d 100644 --- a/plot/point.go +++ b/plot/point.go @@ -41,6 +41,7 @@ type PointStyle struct { //types:add -setters func (ps *PointStyle) Defaults() { ps.Color = colors.Scheme.OnSurface + ps.Fill = colors.Scheme.OnSurface ps.Width.Pt(1) ps.Size.Pt(4) } From 1ecc504c5c79965d44013fcf07443496905f3595 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 7 Nov 2024 02:12:47 -0800 Subject: [PATCH 266/311] plot styles mostly in place and working -- need styler logic etc. --- plot/axis.go | 78 +++++++++++++++++++++++++++------------------ plot/draw.go | 85 ++++++++++++++++++++++++++++---------------------- plot/legend.go | 49 +++++++++++++++++++---------- plot/plot.go | 79 +++++++++++++++++++++++++++++++++++++--------- 4 files changed, 191 insertions(+), 100 deletions(-) diff --git a/plot/axis.go b/plot/axis.go index 91c1d81daa..7547681243 100644 --- a/plot/axis.go +++ b/plot/axis.go @@ -15,12 +15,41 @@ import ( "cogentcore.org/core/styles/units" ) -// Normalizer rescales values from the data coordinate system to the -// normalized coordinate system. -type Normalizer interface { - // Normalize transforms a value x in the data coordinate system to - // the normalized coordinate system. - Normalize(min, max, x float32) float32 +// AxisStyle has style properties for the axis. +type AxisStyle struct { + + // Text has the text style parameters for the text label. + Text TextStyle + + // Line has styling properties for the axis line. + Line LineStyle + + // Padding between the axis line and the data. Having + // non-zero padding ensures that the data is never drawn + // on the axis, thus making it easier to see. + Padding units.Value + + // TickText has the text style for rendering tick labels, + // and is shared for actual rendering. + TickText TextStyle + + // TickLine has line style for drawing tick lines. + TickLine LineStyle + + // TickLength is the length of tick lines. + TickLength units.Value +} + +func (ax *AxisStyle) Defaults() { + ax.Line.Defaults() + ax.Text.Defaults() + ax.Text.Size.Dp(20) + ax.Padding.Pt(5) + ax.TickText.Defaults() + ax.TickText.Size.Dp(16) + ax.TickText.Padding.Dp(2) + ax.TickLine.Defaults() + ax.TickLength.Pt(8) } // Axis represents either a horizontal or vertical @@ -36,23 +65,12 @@ type Axis struct { // Label for the axis Label Text - // Line styling properties for the axis line. - Line LineStyle - - // Padding between the axis line and the data. Having - // non-zero padding ensures that the data is never drawn - // on the axis, thus making it easier to see. - Padding units.Value + // Style has the style parameters for the Axis. + Style AxisStyle - // has the text style for rendering tick labels, and is shared for actual rendering + // TickText is used for rendering the tick text labels. TickText Text - // line style for drawing tick lines - TickLine LineStyle - - // length of tick lines - TickLength units.Value - // Ticker generates the tick marks. Any tick marks // returned by the Marker function that are not in // range of the axis are not drawn. @@ -74,21 +92,13 @@ type Axis struct { // Sets Defaults, range is (∞, ­∞), and thus any finite // value is less than Min and greater than Max. func (ax *Axis) Defaults(dim math32.Dims) { + ax.Style.Defaults() ax.Min = math32.Inf(+1) ax.Max = math32.Inf(-1) ax.Axis = dim - ax.Line.Defaults() - ax.Label.Defaults() - ax.Label.Style.Size.Dp(20) - ax.Padding.Pt(5) - ax.TickText.Defaults() - ax.TickText.Style.Size.Dp(16) - ax.TickText.Style.Padding.Dp(2) - ax.TickLine.Defaults() - ax.TickLength.Pt(8) if dim == math32.Y { ax.Label.Style.Rotation = -90 - ax.TickText.Style.Align = styles.End + ax.Style.TickText.Align = styles.End } ax.Scale = LinearScale{} ax.Ticker = DefaultTicks{} @@ -119,6 +129,14 @@ func (ax *Axis) SanitizeRange() { } } +// Normalizer rescales values from the data coordinate system to the +// normalized coordinate system. +type Normalizer interface { + // Normalize transforms a value x in the data coordinate system to + // the normalized coordinate system. + Normalize(min, max, x float32) float32 +} + // LinearScale an be used as the value of an Axis.Scale function to // set the axis to a standard linear scale. type LinearScale struct{} diff --git a/plot/draw.go b/plot/draw.go index 608fa4e8f9..985c43ce5b 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -57,7 +57,16 @@ func (pt *Plot) drawConfig() { for _, plt := range pt.Plotters { plt.ApplyStyle() } - pt.Legend.TextStyle.openFont(pt) + pt.Legend.Style = pt.Style.Legend + pt.Legend.Style.Text.openFont(pt) + pt.X.Style = pt.Style.Axis + pt.Y.Style = pt.Style.Axis + pt.X.Label.Style = pt.Style.Axis.Text + pt.Y.Label.Style = pt.Style.Axis.Text + pt.X.TickText.Style = pt.Style.Axis.TickText + pt.Y.TickText.Style = pt.Style.Axis.TickText + pt.Y.Label.Style.Rotation = -90 + pt.Y.Style.TickText.Align = styles.End pt.Paint.ToDots() } @@ -73,8 +82,8 @@ func (pt *Plot) Draw() { ptb := image.Rectangle{Max: pt.Size} pc.PushBounds(ptb) - if pt.Background != nil { - pc.BlitBox(math32.Vector2{}, math32.FromPoint(pt.Size), pt.Background) + if pt.Style.Background != nil { + pc.BlitBox(math32.Vector2{}, math32.FromPoint(pt.Size), pt.Style.Background) } if pt.Title.Text != "" { @@ -134,14 +143,14 @@ func (pt *Plot) Draw() { // drawTicks returns true if the tick marks should be drawn. func (ax *Axis) drawTicks() bool { - return ax.TickLine.Width.Value > 0 && ax.TickLength.Value > 0 + return ax.Style.TickLine.Width.Value > 0 && ax.Style.TickLength.Value > 0 } // sizeX returns the total height of the axis, left and right padding func (ax *Axis) sizeX(pt *Plot, axw float32) (ht, lpad, rpad int) { pc := pt.Paint uc := &pc.UnitContext - ax.TickLength.ToDots(uc) + ax.Style.TickLength.ToDots(uc) ax.ticks = ax.Ticker.Ticks(ax.Min, ax.Max) h := float32(0) if ax.Label.Text != "" { // We assume that the label isn't rotated. @@ -149,13 +158,13 @@ func (ax *Axis) sizeX(pt *Plot, axw float32) (ht, lpad, rpad int) { h += ax.Label.PaintText.BBox.Size().Y h += ax.Label.Style.Padding.Dots } - lw := ax.Line.Width.Dots + lw := ax.Style.Line.Width.Dots lpad = int(math32.Ceil(lw)) + 2 rpad = int(math32.Ceil(lw)) + 10 tht := float32(0) if len(ax.ticks) > 0 { if ax.drawTicks() { - h += ax.TickLength.Dots + h += ax.Style.TickLength.Dots } ftk := ax.firstTickLabel() if ftk.Label != "" { @@ -180,7 +189,7 @@ func (ax *Axis) sizeX(pt *Plot, axw float32) (ht, lpad, rpad int) { } h += ax.TickText.Style.Padding.Dots } - h += tht + lw + ax.Padding.Dots + h += tht + lw + ax.Style.Padding.Dots ht = int(math32.Ceil(h)) return @@ -235,7 +244,7 @@ func (ax *Axis) sizeY(pt *Plot) (ywidth, tickWidth, tpad, bpad int) { pc := pt.Paint uc := &pc.UnitContext ax.ticks = ax.Ticker.Ticks(ax.Min, ax.Max) - ax.TickLength.ToDots(uc) + ax.Style.TickLength.ToDots(uc) w := float32(0) if ax.Label.Text != "" { @@ -244,13 +253,13 @@ func (ax *Axis) sizeY(pt *Plot) (ywidth, tickWidth, tpad, bpad int) { w += ax.Label.Style.Padding.Dots } - lw := ax.Line.Width.Dots + lw := ax.Style.Line.Width.Dots tpad = int(math32.Ceil(lw)) + 2 bpad = int(math32.Ceil(lw)) + 2 if len(ax.ticks) > 0 { if ax.drawTicks() { - w += ax.TickLength.Dots + w += ax.Style.TickLength.Dots } ax.TickText.Text = ax.longestTickLabel() if ax.TickText.Text != "" { @@ -264,7 +273,7 @@ func (ax *Axis) sizeY(pt *Plot) (ywidth, tickWidth, tpad, bpad int) { bpad += tht } } - w += lw + ax.Padding.Dots + w += lw + ax.Style.Padding.Dots ywidth = int(math32.Ceil(w)) return } @@ -308,7 +317,7 @@ func (ax *Axis) drawX(pt *Plot, lpad, rpad int) { } if len(ax.ticks) > 0 && ax.drawTicks() { - ln := ax.TickLength.Dots + ln := ax.Style.TickLength.Dots for _, t := range ax.ticks { yoff := float32(0) if t.IsMinor() { @@ -319,12 +328,12 @@ func (ax *Axis) drawX(pt *Plot, lpad, rpad int) { continue } x += float32(ab.Min.X) - ax.TickLine.Draw(pt, math32.Vec2(x, float32(ab.Max.Y)-yoff), math32.Vec2(x, float32(ab.Max.Y)-ln)) + ax.Style.TickLine.Draw(pt, math32.Vec2(x, float32(ab.Max.Y)-yoff), math32.Vec2(x, float32(ab.Max.Y)-ln)) } - ab.Max.Y -= int(ln - 0.5*ax.Line.Width.Dots) + ab.Max.Y -= int(ln - 0.5*ax.Style.Line.Width.Dots) } - ax.Line.Draw(pt, math32.Vec2(float32(ab.Min.X), float32(ab.Max.Y)), math32.Vec2(float32(ab.Min.X)+axw, float32(ab.Max.Y))) + ax.Style.Line.Draw(pt, math32.Vec2(float32(ab.Min.X), float32(ab.Max.Y)), math32.Vec2(float32(ab.Min.X)+axw, float32(ab.Max.Y))) } // drawY draws the Y axis along the left side @@ -365,7 +374,7 @@ func (ax *Axis) drawY(pt *Plot, tickWidth, tpad, bpad int) { } if len(ax.ticks) > 0 && ax.drawTicks() { - ln := ax.TickLength.Dots + ln := ax.Style.TickLength.Dots for _, t := range ax.ticks { xoff := float32(0) if t.IsMinor() { @@ -376,12 +385,12 @@ func (ax *Axis) drawY(pt *Plot, tickWidth, tpad, bpad int) { continue } y += float32(ab.Min.Y) - ax.TickLine.Draw(pt, math32.Vec2(float32(ab.Min.X)+xoff, y), math32.Vec2(float32(ab.Min.X)+ln, y)) + ax.Style.TickLine.Draw(pt, math32.Vec2(float32(ab.Min.X)+xoff, y), math32.Vec2(float32(ab.Min.X)+ln, y)) } - ab.Min.X += int(ln + 0.5*ax.Line.Width.Dots) + ab.Min.X += int(ln + 0.5*ax.Style.Line.Width.Dots) } - ax.Line.Draw(pt, math32.Vec2(float32(ab.Min.X), float32(ab.Min.Y)), math32.Vec2(float32(ab.Min.X), float32(ab.Max.Y))) + ax.Style.Line.Draw(pt, math32.Vec2(float32(ab.Min.X), float32(ab.Min.Y)), math32.Vec2(float32(ab.Min.X), float32(ab.Max.Y))) } //////////////////////////////////////////////// @@ -393,17 +402,17 @@ func (lg *Legend) draw(pt *Plot) { uc := &pc.UnitContext ptb := pc.Bounds - lg.ThumbnailWidth.ToDots(uc) - lg.TextStyle.ToDots(uc) - lg.Position.XOffs.ToDots(uc) - lg.Position.YOffs.ToDots(uc) - lg.TextStyle.openFont(pt) + lg.Style.ThumbnailWidth.ToDots(uc) + lg.Style.Text.ToDots(uc) + lg.Style.Position.XOffs.ToDots(uc) + lg.Style.Position.YOffs.ToDots(uc) + lg.Style.Text.openFont(pt) - em := lg.TextStyle.Font.Face.Metrics.Em - pad := math32.Ceil(lg.TextStyle.Padding.Dots) + em := lg.Style.Text.Font.Face.Metrics.Em + pad := math32.Ceil(lg.Style.Text.Padding.Dots) var ltxt Text - ltxt.Style = lg.TextStyle + ltxt.Style = lg.Style.Text var sz image.Point maxTht := 0 for _, e := range lg.Entries { @@ -416,25 +425,25 @@ func (lg *Legend) draw(pt *Plot) { sz.X += int(em) sz.Y = len(lg.Entries) * maxTht txsz := sz - sz.X += int(lg.ThumbnailWidth.Dots) + sz.X += int(lg.Style.ThumbnailWidth.Dots) pos := ptb.Min - if lg.Position.Left { - pos.X += int(lg.Position.XOffs.Dots) + if lg.Style.Position.Left { + pos.X += int(lg.Style.Position.XOffs.Dots) } else { - pos.X = ptb.Max.X - sz.X - int(lg.Position.XOffs.Dots) + pos.X = ptb.Max.X - sz.X - int(lg.Style.Position.XOffs.Dots) } - if lg.Position.Top { - pos.Y += int(lg.Position.YOffs.Dots) + if lg.Style.Position.Top { + pos.Y += int(lg.Style.Position.YOffs.Dots) } else { - pos.Y = ptb.Max.Y - sz.Y - int(lg.Position.YOffs.Dots) + pos.Y = ptb.Max.Y - sz.Y - int(lg.Style.Position.YOffs.Dots) } - if lg.Fill != nil { - pc.FillBox(math32.FromPoint(pos), math32.FromPoint(sz), lg.Fill) + if lg.Style.Fill != nil { + pc.FillBox(math32.FromPoint(pos), math32.FromPoint(sz), lg.Style.Fill) } cp := pos - thsz := image.Point{X: int(lg.ThumbnailWidth.Dots), Y: maxTht - 2*int(pad)} + thsz := image.Point{X: int(lg.Style.ThumbnailWidth.Dots), Y: maxTht - 2*int(pad)} for _, e := range lg.Entries { tp := cp tp.X += int(txsz.X) diff --git a/plot/legend.go b/plot/legend.go index b7b4a2596a..ee86b38451 100644 --- a/plot/legend.go +++ b/plot/legend.go @@ -12,6 +12,35 @@ import ( "cogentcore.org/core/styles/units" ) +// LegendStyle has the styling properties for the Legend. +type LegendStyle struct { + + // Column is for table-based plotting, specifying the column with legend values. + Column string + + // Text is the style given to the legend entry texts. + Text TextStyle + + // position of the legend + Position LegendPosition `display:"inline"` + + // ThumbnailWidth is the width of legend thumbnails. + ThumbnailWidth units.Value + + // Fill specifies the background fill color for the legend box, + // if non-nil. + Fill image.Image +} + +func (ls *LegendStyle) Defaults() { + ls.Text.Defaults() + ls.Text.Padding.Dp(2) + ls.Text.Font.Size.Dp(20) + ls.Position.Defaults() + ls.ThumbnailWidth.Pt(20) + ls.Fill = gradient.ApplyOpacity(colors.Scheme.Surface, 0.75) +} + // LegendPosition specifies where to put the legend type LegendPosition struct { // Top and Left specify the location of the legend. @@ -31,30 +60,16 @@ func (lg *LegendPosition) Defaults() { // and a thumbnail, where the thumbnail shows a small // sample of the display style of the corresponding data. type Legend struct { - // TextStyle is the style given to the legend entry texts. - TextStyle TextStyle - - // position of the legend - Position LegendPosition `display:"inline"` - // ThumbnailWidth is the width of legend thumbnails. - ThumbnailWidth units.Value - - // Fill specifies the background fill color for the legend box, - // if non-nil. - Fill image.Image + // Style has the legend styling parameters. + Style LegendStyle // Entries are all of the LegendEntries described by this legend. Entries []LegendEntry } func (lg *Legend) Defaults() { - lg.TextStyle.Defaults() - lg.TextStyle.Padding.Dp(2) - lg.TextStyle.Font.Size.Dp(20) - lg.Position.Defaults() - lg.ThumbnailWidth.Pt(20) - lg.Fill = gradient.ApplyOpacity(colors.Scheme.Surface, 0.75) + lg.Style.Defaults() } // Add adds an entry to the legend with the given name. diff --git a/plot/plot.go b/plot/plot.go index cdd40511b1..fb756c434b 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -31,9 +31,8 @@ type Plot struct { // Title of the plot Title Text - // Background is the background of the plot. - // The default is [colors.Scheme.Surface]. - Background image.Image + // Style has the styling properties for the plot. + Style PlotStyle // standard text style with default options StandardTextStyle styles.Text @@ -69,9 +68,9 @@ type Plot struct { // Defaults sets defaults func (pt *Plot) Defaults() { + pt.Style.Defaults() pt.Title.Defaults() pt.Title.Style.Size.Dp(24) - pt.Background = colors.Scheme.Surface pt.X.Defaults(math32.X) pt.Y.Defaults(math32.Y) pt.Legend.Defaults() @@ -136,10 +135,10 @@ func (pt *Plot) SaveImage(filename string) error { // that do not end up in range of the X axis will not have // tick marks. func (pt *Plot) NominalX(names ...string) { - pt.X.TickLine.Width.Pt(0) - pt.X.TickLength.Pt(0) - pt.X.Line.Width.Pt(0) - // pt.Y.Padding.Pt(pt.X.Tick.Label.Width(names[0]) / 2) + pt.X.Style.TickLine.Width.Pt(0) + pt.X.Style.TickLength.Pt(0) + pt.X.Style.Line.Width.Pt(0) + // pt.Y.Padding.Pt(pt.X.Style.Tick.Label.Width(names[0]) / 2) ticks := make([]Tick, len(names)) for i, name := range names { ticks[i] = Tick{float32(i), name} @@ -149,15 +148,15 @@ func (pt *Plot) NominalX(names ...string) { // HideX configures the X axis so that it will not be drawn. func (pt *Plot) HideX() { - pt.X.TickLength.Pt(0) - pt.X.Line.Width.Pt(0) + pt.X.Style.TickLength.Pt(0) + pt.X.Style.Line.Width.Pt(0) pt.X.Ticker = ConstantTicks([]Tick{}) } // HideY configures the Y axis so that it will not be drawn. func (pt *Plot) HideY() { - pt.Y.TickLength.Pt(0) - pt.Y.Line.Width.Pt(0) + pt.Y.Style.TickLength.Pt(0) + pt.Y.Style.Line.Width.Pt(0) pt.Y.Ticker = ConstantTicks([]Tick{}) } @@ -169,9 +168,9 @@ func (pt *Plot) HideAxes() { // NominalY is like NominalX, but for the Y axis. func (pt *Plot) NominalY(names ...string) { - pt.Y.TickLine.Width.Pt(0) - pt.Y.TickLength.Pt(0) - pt.Y.Line.Width.Pt(0) + pt.Y.Style.TickLine.Width.Pt(0) + pt.Y.Style.TickLength.Pt(0) + pt.Y.Style.Line.Width.Pt(0) // pt.X.Padding = pt.Y.Tick.Label.Height(names[0]) / 2 ticks := make([]Tick, len(names)) for i, name := range names { @@ -238,3 +237,53 @@ func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float } return } + +//////// PlotStyle + +// PlotStyle has overall plot level styling parameters. +type PlotStyle struct { + + // Title is the overall title of the plot. + Title string + + // TitleStyle is the text styling parameters for the title. + TitleStyle TextStyle + + // Background is the background of the plot. + // The default is [colors.Scheme.Surface]. + Background image.Image + + // Scale multiplies the plot DPI value, to change the overall scale + // of the rendered plot. Larger numbers produce larger scaling. + // Typically use larger numbers when generating plots for inclusion in + // documents or other cases where the overall plot size will be small. + Scale float32 `default:"1,2"` + + // Legend has the styling properties for the Legend. + Legend LegendStyle + + // Axis has the styling properties for the Axes. + Axis AxisStyle + + // what column to use for the common X axis. if empty or not found, + // the row number is used. This optional for Bar plots, if present and + // Legend is also present, then an extra space will be put between X values. + XAxis string + + // XAxisRotation is the rotation of the X Axis labels, in degrees. + XAxisRotation float32 + + // XAxisLabel is the optional label to use for the XAxis instead of the default. + XAxisLabel string + + // YAxisLabel is the optional label to use for the YAxis instead of the default. + YAxisLabel string +} + +func (ps *PlotStyle) Defaults() { + ps.Legend.Defaults() + ps.Axis.Defaults() + ps.TitleStyle.Size.Dp(24) + ps.Background = colors.Scheme.Surface + ps.Scale = 1 +} From 967e939cc27c86aee0dba9dc5b255fc42c54cafe Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 7 Nov 2024 11:59:03 -0800 Subject: [PATCH 267/311] new styling plan in place for how to combine plot and element level styling --- plot/README.md | 21 ++++++++ plot/axis.go | 2 +- plot/draw.go | 28 +++++++++-- plot/legend.go | 2 +- plot/plot.go | 99 ++++++++++++++++++------------------ plot/plots/xy.go | 5 +- plot/plotter.go | 11 ++-- plot/style.go | 9 +++- plot/text.go | 2 +- plot/typegen.go | 128 +++++++++++++++++++++++++++++++++++++++++++++-- 10 files changed, 239 insertions(+), 68 deletions(-) diff --git a/plot/README.md b/plot/README.md index c5fe23479b..7d8a6ce6ad 100644 --- a/plot/README.md +++ b/plot/README.md @@ -4,6 +4,27 @@ The `plot` package generates 2D plots of data using the Cogent Core `paint` rend * `Plot` is just a wrapper around a `plot.Plot`, for manually-configured plots. * `PlotEditor` is an interactive plot viewer that supports selection of which data to plot, and configuration of many plot parameters. +# Styling + +`plot.Style` contains the full set of styling parameters, which can be set using Styler functions that are attached to individual plot elements that drive the content of what is actually plotted (based on the `Plotter` interface). + +Each such plot element defines the `Styler` method, eg: + +```Go +plt := plot.NewPlot() +ln := plots.AddLine.Styler(func(s *plot.Style) { + s.Plot.Title = "My Plot" + s.Line.Color = colors.Uniform(colors.Red) +}) +plt.Add(ln) +``` + +The `Plot` field (of type `PlotStyle`) contains all the settings that apply to the plot as a whole. Each element can set these values, and they are applied in the order the elements are added, so the last one gets final say. Typically you want to just set these plot-level styles on one element only and avoid any conflicts. + +The rest of the style properties (e.g., `Line`, `Point`) apply to the element in question. There are also some default plot-level settings in `Plot` that apply to all elements, and the plot-level styles are updated first, so in this way it is possible to have plot-wide settings applied from one styler, that affect all plots (e.g., the line width, and whether lines and / or points are plotted or not). + +# History + The code is adapted from the [gonum plot](https://github.com/gonum/plot) package (which in turn was adapted from google's [plotinum](https://code.google.com/archive/p/plotinum/), to use the Cogent Core [styles](../styles) and [paint](../paint) rendering framework, which also supports SVG output of the rendering. Here is the copyright notice for that package: diff --git a/plot/axis.go b/plot/axis.go index 7547681243..dc3ce4e1fd 100644 --- a/plot/axis.go +++ b/plot/axis.go @@ -16,7 +16,7 @@ import ( ) // AxisStyle has style properties for the axis. -type AxisStyle struct { +type AxisStyle struct { //types:add -setters // Text has the text style parameters for the text label. Text TextStyle diff --git a/plot/draw.go b/plot/draw.go index 985c43ce5b..b614ab5aac 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -51,11 +51,25 @@ func (pt *Plot) SVGToFile(filename string) error { return bw.Flush() } -// drawConfig configures everything for drawing -func (pt *Plot) drawConfig() { - pt.Resize(pt.Size) // ensure +// applyStyle applies all the style parameters +func (pt *Plot) applyStyle() { + // first update the global plot style settings + var st Style + st.Defaults() + st.Plot = pt.Style for _, plt := range pt.Plotters { - plt.ApplyStyle() + stlr := plt.Stylers() + stlr.Run(&st) + } + pt.Style = st.Plot + // then apply to elements + for _, plt := range pt.Plotters { + plt.ApplyStyle(&pt.Style) + } + // now style plot: + pt.Title.Style = pt.Style.TitleStyle + if pt.Style.Title != "" { + pt.Title.Text = pt.Style.Title } pt.Legend.Style = pt.Style.Legend pt.Legend.Style.Text.openFont(pt) @@ -67,6 +81,12 @@ func (pt *Plot) drawConfig() { pt.Y.TickText.Style = pt.Style.Axis.TickText pt.Y.Label.Style.Rotation = -90 pt.Y.Style.TickText.Align = styles.End +} + +// drawConfig configures everything for drawing, applying styles etc. +func (pt *Plot) drawConfig() { + pt.Resize(pt.Size) // ensure + pt.applyStyle() pt.Paint.ToDots() } diff --git a/plot/legend.go b/plot/legend.go index ee86b38451..dc825b8a12 100644 --- a/plot/legend.go +++ b/plot/legend.go @@ -13,7 +13,7 @@ import ( ) // LegendStyle has the styling properties for the Legend. -type LegendStyle struct { +type LegendStyle struct { //types:add -setters // Column is for table-based plotting, specifying the column with legend values. Column string diff --git a/plot/plot.go b/plot/plot.go index fb756c434b..0ceec6d877 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -21,6 +21,55 @@ import ( "cogentcore.org/core/styles" ) +// PlotStyle has overall plot level styling parameters. +type PlotStyle struct { //types:add -setters + + // Title is the overall title of the plot. + Title string + + // TitleStyle is the text styling parameters for the title. + TitleStyle TextStyle + + // Background is the background of the plot. + // The default is [colors.Scheme.Surface]. + Background image.Image + + // Scale multiplies the plot DPI value, to change the overall scale + // of the rendered plot. Larger numbers produce larger scaling. + // Typically use larger numbers when generating plots for inclusion in + // documents or other cases where the overall plot size will be small. + Scale float32 `default:"1,2"` + + // Legend has the styling properties for the Legend. + Legend LegendStyle + + // Axis has the styling properties for the Axes. + Axis AxisStyle + + // XAxis specifies the column to use for the common X axis in a table based plot. + // if empty or not found, the row number is used. + // This optional for Bar plots, if present and Legend is also present, + // then an extra space will be put between X values. + XAxis string + + // XAxisRotation is the rotation of the X Axis labels, in degrees. + XAxisRotation float32 + + // XAxisLabel is the optional label to use for the XAxis instead of the default. + XAxisLabel string + + // YAxisLabel is the optional label to use for the YAxis instead of the default. + YAxisLabel string +} + +func (ps *PlotStyle) Defaults() { + ps.Legend.Defaults() + ps.Axis.Defaults() + ps.TitleStyle.Size.Dp(24) + ps.Background = colors.Scheme.Surface + ps.Scale = 1 +} + // Plot is the basic type representing a plot. // It renders into its own image.RGBA Pixels image, // and can also save a corresponding SVG version. @@ -237,53 +286,3 @@ func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float } return } - -//////// PlotStyle - -// PlotStyle has overall plot level styling parameters. -type PlotStyle struct { - - // Title is the overall title of the plot. - Title string - - // TitleStyle is the text styling parameters for the title. - TitleStyle TextStyle - - // Background is the background of the plot. - // The default is [colors.Scheme.Surface]. - Background image.Image - - // Scale multiplies the plot DPI value, to change the overall scale - // of the rendered plot. Larger numbers produce larger scaling. - // Typically use larger numbers when generating plots for inclusion in - // documents or other cases where the overall plot size will be small. - Scale float32 `default:"1,2"` - - // Legend has the styling properties for the Legend. - Legend LegendStyle - - // Axis has the styling properties for the Axes. - Axis AxisStyle - - // what column to use for the common X axis. if empty or not found, - // the row number is used. This optional for Bar plots, if present and - // Legend is also present, then an extra space will be put between X values. - XAxis string - - // XAxisRotation is the rotation of the X Axis labels, in degrees. - XAxisRotation float32 - - // XAxisLabel is the optional label to use for the XAxis instead of the default. - XAxisLabel string - - // YAxisLabel is the optional label to use for the YAxis instead of the default. - YAxisLabel string -} - -func (ps *PlotStyle) Defaults() { - ps.Legend.Defaults() - ps.Axis.Defaults() - ps.TitleStyle.Size.Dp(24) - ps.Background = colors.Scheme.Surface - ps.Scale = 1 -} diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 4fc79585ba..a6d2a34c05 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -81,7 +81,10 @@ func (ln *XY) Styler(f func(s *plot.Style)) *XY { return ln } -func (ln *XY) ApplyStyle() { ln.stylers.Run(&ln.Style) } +func (ln *XY) Stylers() { return &ln.stylers } +func (ln *XY) ApplyStyle(plotStyle *plot.Style) { + ln.stylers.Run(&ln.Style) +} func (ln *XY) XYData() (data plot.XYer, pixels plot.XYer) { data = ln.XYs diff --git a/plot/plotter.go b/plot/plotter.go index b888b8b1c7..9d5f91812e 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -5,7 +5,7 @@ package plot // Plotter is an interface that wraps the Plot method. -// Some standard implementations of Plotter can be found in plotters. +// Standard implementations of Plotter are in the [plots] package. type Plotter interface { // Plot draws the data to the Plot Paint Plot(pt *Plot) @@ -15,8 +15,13 @@ type Plotter interface { // This allows gui interface to inspect data etc. XYData() (data XYer, pixels XYer) - // ApplyStyle applies any stylers to this element. - ApplyStyle() + // Stylers returns the styler functions for this element. + Stylers() *Stylers + + // ApplyStyle applies any stylers to this element, + // first initializing from the given global plot style, which has + // already been styled with defaults and all the plot element stylers. + ApplyStyle(plotStyle *PlotStyle) } // DataRanger wraps the DataRange method. diff --git a/plot/style.go b/plot/style.go index cac64f3c64..4a57c78295 100644 --- a/plot/style.go +++ b/plot/style.go @@ -10,9 +10,14 @@ import ( ) // Style contains the plot styling properties relevant across -// most plot types. These properties apply both to individual plot elements -// and to the plot as a whole. +// most plot types. These properties apply to individual plot elements +// while the Plot properties applies to the overall plot itself. type Style struct { //types:add -setters + + // Plot has overall plot-level properties, which can be set by any + // plot element, and are updated first, before applying element-wise styles. + Plot PlotStyle + // On specifies whether to plot this item, for cases where it can be turned off. On DefaultOffOn diff --git a/plot/text.go b/plot/text.go index 5c14245168..98f65a99d3 100644 --- a/plot/text.go +++ b/plot/text.go @@ -17,7 +17,7 @@ import ( var DefaultFontFamily = "" // TextStyle specifies styling parameters for Text elements -type TextStyle struct { +type TextStyle struct { //types:add -setters styles.FontRender // how to align text along the relevant dimension for the text element diff --git a/plot/typegen.go b/plot/typegen.go index 4f5869f1a6..02414b35f8 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -6,13 +6,43 @@ import ( "image" "cogentcore.org/core/math32/minmax" + "cogentcore.org/core/styles" "cogentcore.org/core/styles/units" "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Normalizer", IDName: "normalizer", Doc: "Normalizer rescales values from the data coordinate system to the\nnormalized coordinate system.", Methods: []types.Method{{Name: "Normalize", Doc: "Normalize transforms a value x in the data coordinate system to\nthe normalized coordinate system.", Args: []string{"min", "max", "x"}, Returns: []string{"float32"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) + +// SetText sets the [AxisStyle.Text]: +// Text has the text style parameters for the text label. +func (t *AxisStyle) SetText(v TextStyle) *AxisStyle { t.Text = v; return t } + +// SetLine sets the [AxisStyle.Line]: +// Line has styling properties for the axis line. +func (t *AxisStyle) SetLine(v LineStyle) *AxisStyle { t.Line = v; return t } + +// SetPadding sets the [AxisStyle.Padding]: +// Padding between the axis line and the data. Having +// non-zero padding ensures that the data is never drawn +// on the axis, thus making it easier to see. +func (t *AxisStyle) SetPadding(v units.Value) *AxisStyle { t.Padding = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Axis", IDName: "axis", Doc: "Axis represents either a horizontal or vertical\naxis of a plot.", Fields: []types.Field{{Name: "Min", Doc: "Min and Max are the minimum and maximum data\nvalues represented by the axis."}, {Name: "Max", Doc: "Min and Max are the minimum and maximum data\nvalues represented by the axis."}, {Name: "Axis", Doc: "specifies which axis this is: X or Y"}, {Name: "Label", Doc: "Label for the axis"}, {Name: "Line", Doc: "Line styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "TickText", Doc: "has the text style for rendering tick labels, and is shared for actual rendering"}, {Name: "TickLine", Doc: "line style for drawing tick lines"}, {Name: "TickLength", Doc: "length of tick lines"}, {Name: "Ticker", Doc: "Ticker generates the tick marks. Any tick marks\nreturned by the Marker function that are not in\nrange of the axis are not drawn."}, {Name: "Scale", Doc: "Scale transforms a value given in the data coordinate system\nto the normalized coordinate system of the axis—its distance\nalong the axis as a fraction of the axis range."}, {Name: "AutoRescale", Doc: "AutoRescale enables an axis to automatically adapt its minimum\nand maximum boundaries, according to its underlying Ticker."}, {Name: "ticks", Doc: "cached list of ticks, set in size"}}}) +// SetTickText sets the [AxisStyle.TickText]: +// TickText has the text style for rendering tick labels, +// and is shared for actual rendering. +func (t *AxisStyle) SetTickText(v TextStyle) *AxisStyle { t.TickText = v; return t } + +// SetTickLine sets the [AxisStyle.TickLine]: +// TickLine has line style for drawing tick lines. +func (t *AxisStyle) SetTickLine(v LineStyle) *AxisStyle { t.TickLine = v; return t } + +// SetTickLength sets the [AxisStyle.TickLength]: +// TickLength is the length of tick lines. +func (t *AxisStyle) SetTickLength(v units.Value) *AxisStyle { t.TickLength = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Axis", IDName: "axis", Doc: "Axis represents either a horizontal or vertical\naxis of a plot.", Fields: []types.Field{{Name: "Min", Doc: "Min and Max are the minimum and maximum data\nvalues represented by the axis."}, {Name: "Max", Doc: "Min and Max are the minimum and maximum data\nvalues represented by the axis."}, {Name: "Axis", Doc: "specifies which axis this is: X or Y"}, {Name: "Label", Doc: "Label for the axis"}, {Name: "Style", Doc: "Style has the style parameters for the Axis."}, {Name: "TickText", Doc: "TickText is used for rendering the tick text labels."}, {Name: "Ticker", Doc: "Ticker generates the tick marks. Any tick marks\nreturned by the Marker function that are not in\nrange of the axis are not drawn."}, {Name: "Scale", Doc: "Scale transforms a value given in the data coordinate system\nto the normalized coordinate system of the axis—its distance\nalong the axis as a fraction of the axis range."}, {Name: "AutoRescale", Doc: "AutoRescale enables an axis to automatically adapt its minimum\nand maximum boundaries, according to its underlying Ticker."}, {Name: "ticks", Doc: "cached list of ticks, set in size"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Normalizer", IDName: "normalizer", Doc: "Normalizer rescales values from the data coordinate system to the\nnormalized coordinate system.", Methods: []types.Method{{Name: "Normalize", Doc: "Normalize transforms a value x in the data coordinate system to\nthe normalized coordinate system.", Args: []string{"min", "max", "x"}, Returns: []string{"float32"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LinearScale", IDName: "linear-scale", Doc: "LinearScale an be used as the value of an Axis.Scale function to\nset the axis to a standard linear scale."}) @@ -50,9 +80,32 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.selection", ID var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.weights", IDName: "weights", Doc: "weights is a helper type to calcuate the labelling scheme's total score.", Fields: []types.Field{{Name: "simplicity"}, {Name: "coverage"}, {Name: "density"}, {Name: "legibility"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LegendStyle", IDName: "legend-style", Doc: "LegendStyle has the styling properties for the Legend.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Column", Doc: "Column is for table-based plotting, specifying the column with legend values."}, {Name: "Text", Doc: "Text is the style given to the legend entry texts."}, {Name: "Position", Doc: "position of the legend"}, {Name: "ThumbnailWidth", Doc: "ThumbnailWidth is the width of legend thumbnails."}, {Name: "Fill", Doc: "Fill specifies the background fill color for the legend box,\nif non-nil."}}}) + +// SetColumn sets the [LegendStyle.Column]: +// Column is for table-based plotting, specifying the column with legend values. +func (t *LegendStyle) SetColumn(v string) *LegendStyle { t.Column = v; return t } + +// SetText sets the [LegendStyle.Text]: +// Text is the style given to the legend entry texts. +func (t *LegendStyle) SetText(v TextStyle) *LegendStyle { t.Text = v; return t } + +// SetPosition sets the [LegendStyle.Position]: +// position of the legend +func (t *LegendStyle) SetPosition(v LegendPosition) *LegendStyle { t.Position = v; return t } + +// SetThumbnailWidth sets the [LegendStyle.ThumbnailWidth]: +// ThumbnailWidth is the width of legend thumbnails. +func (t *LegendStyle) SetThumbnailWidth(v units.Value) *LegendStyle { t.ThumbnailWidth = v; return t } + +// SetFill sets the [LegendStyle.Fill]: +// Fill specifies the background fill color for the legend box, +// if non-nil. +func (t *LegendStyle) SetFill(v image.Image) *LegendStyle { t.Fill = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LegendPosition", IDName: "legend-position", Doc: "LegendPosition specifies where to put the legend", Fields: []types.Field{{Name: "Top", Doc: "Top and Left specify the location of the legend."}, {Name: "Left", Doc: "Top and Left specify the location of the legend."}, {Name: "XOffs", Doc: "XOffs and YOffs are added to the legend's final position,\nrelative to the relevant anchor position"}, {Name: "YOffs", Doc: "XOffs and YOffs are added to the legend's final position,\nrelative to the relevant anchor position"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Legend", IDName: "legend", Doc: "A Legend gives a description of the meaning of different\ndata elements of the plot. Each legend entry has a name\nand a thumbnail, where the thumbnail shows a small\nsample of the display style of the corresponding data.", Fields: []types.Field{{Name: "TextStyle", Doc: "TextStyle is the style given to the legend entry texts."}, {Name: "Position", Doc: "position of the legend"}, {Name: "ThumbnailWidth", Doc: "ThumbnailWidth is the width of legend thumbnails."}, {Name: "Fill", Doc: "Fill specifies the background fill color for the legend box,\nif non-nil."}, {Name: "Entries", Doc: "Entries are all of the LegendEntries described by this legend."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Legend", IDName: "legend", Doc: "A Legend gives a description of the meaning of different\ndata elements of the plot. Each legend entry has a name\nand a thumbnail, where the thumbnail shows a small\nsample of the display style of the corresponding data.", Fields: []types.Field{{Name: "Style", Doc: "Style has the legend styling parameters."}, {Name: "Entries", Doc: "Entries are all of the LegendEntries described by this legend."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Thumbnailer", IDName: "thumbnailer", Doc: "Thumbnailer wraps the Thumbnail method, which\ndraws the small image in a legend representing the\nstyle of data.", Methods: []types.Method{{Name: "Thumbnail", Doc: "Thumbnail draws an thumbnail representing\na legend entry. The thumbnail will usually show\na smaller representation of the style used\nto plot the corresponding data.", Args: []string{"pt"}}}}) @@ -98,7 +151,56 @@ func (t *LineStyle) SetStep(v StepKind) *LineStyle { t.Step = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.StepKind", IDName: "step-kind", Doc: "StepKind specifies a form of a connection of two consecutive points."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.\nThe Axis ranges are updated automatically when plots\nare added, so setting a fixed range should happen\nafter that point. See [UpdateRange] method as well.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Y", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotStyle", IDName: "plot-style", Doc: "PlotStyle has overall plot level styling parameters.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Title", Doc: "Title is the overall title of the plot."}, {Name: "TitleStyle", Doc: "TitleStyle is the text styling parameters for the title."}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Legend", Doc: "Legend has the styling properties for the Legend."}, {Name: "Axis", Doc: "Axis has the styling properties for the Axes."}, {Name: "XAxis", Doc: "XAxis specifies the column to use for the common X axis in a table based plot.\nif empty or not found, the row number is used.\nThis optional for Bar plots, if present and Legend is also present,\nthen an extra space will be put between X values."}, {Name: "XAxisRotation", Doc: "XAxisRotation is the rotation of the X Axis labels, in degrees."}, {Name: "XAxisLabel", Doc: "XAxisLabel is the optional label to use for the XAxis instead of the default."}, {Name: "YAxisLabel", Doc: "YAxisLabel is the optional label to use for the YAxis instead of the default."}}}) + +// SetTitle sets the [PlotStyle.Title]: +// Title is the overall title of the plot. +func (t *PlotStyle) SetTitle(v string) *PlotStyle { t.Title = v; return t } + +// SetTitleStyle sets the [PlotStyle.TitleStyle]: +// TitleStyle is the text styling parameters for the title. +func (t *PlotStyle) SetTitleStyle(v TextStyle) *PlotStyle { t.TitleStyle = v; return t } + +// SetBackground sets the [PlotStyle.Background]: +// Background is the background of the plot. +// The default is [colors.Scheme.Surface]. +func (t *PlotStyle) SetBackground(v image.Image) *PlotStyle { t.Background = v; return t } + +// SetScale sets the [PlotStyle.Scale]: +// Scale multiplies the plot DPI value, to change the overall scale +// of the rendered plot. Larger numbers produce larger scaling. +// Typically use larger numbers when generating plots for inclusion in +// documents or other cases where the overall plot size will be small. +func (t *PlotStyle) SetScale(v float32) *PlotStyle { t.Scale = v; return t } + +// SetLegend sets the [PlotStyle.Legend]: +// Legend has the styling properties for the Legend. +func (t *PlotStyle) SetLegend(v LegendStyle) *PlotStyle { t.Legend = v; return t } + +// SetAxis sets the [PlotStyle.Axis]: +// Axis has the styling properties for the Axes. +func (t *PlotStyle) SetAxis(v AxisStyle) *PlotStyle { t.Axis = v; return t } + +// SetXAxis sets the [PlotStyle.XAxis]: +// XAxis specifies the column to use for the common X axis in a table based plot. +// if empty or not found, the row number is used. +// This optional for Bar plots, if present and Legend is also present, +// then an extra space will be put between X values. +func (t *PlotStyle) SetXAxis(v string) *PlotStyle { t.XAxis = v; return t } + +// SetXAxisRotation sets the [PlotStyle.XAxisRotation]: +// XAxisRotation is the rotation of the X Axis labels, in degrees. +func (t *PlotStyle) SetXAxisRotation(v float32) *PlotStyle { t.XAxisRotation = v; return t } + +// SetXAxisLabel sets the [PlotStyle.XAxisLabel]: +// XAxisLabel is the optional label to use for the XAxis instead of the default. +func (t *PlotStyle) SetXAxisLabel(v string) *PlotStyle { t.XAxisLabel = v; return t } + +// SetYAxisLabel sets the [PlotStyle.YAxisLabel]: +// YAxisLabel is the optional label to use for the YAxis instead of the default. +func (t *PlotStyle) SetYAxisLabel(v string) *PlotStyle { t.YAxisLabel = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.\nThe Axis ranges are updated automatically when plots\nare added, so setting a fixed range should happen\nafter that point. See [UpdateRange] method as well.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Y", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nSome standard implementations of Plotter can be found in plotters.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element."}}}) @@ -206,7 +308,23 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Stylers", IDNa var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DefaultOffOn", IDName: "default-off-on", Doc: "DefaultOffOn specifies whether to use the default value for a bool option,\nor to override the default and set Off or On."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TextStyle", IDName: "text-style", Doc: "TextStyle specifies styling parameters for Text elements", Embeds: []types.Field{{Name: "FontRender"}}, Fields: []types.Field{{Name: "Align", Doc: "how to align text along the relevant dimension for the text element"}, {Name: "Padding", Doc: "Padding is used in a case-dependent manner to add space around text elements"}, {Name: "Rotation", Doc: "rotation of the text, in Degrees"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TextStyle", IDName: "text-style", Doc: "TextStyle specifies styling parameters for Text elements", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Embeds: []types.Field{{Name: "FontRender"}}, Fields: []types.Field{{Name: "Align", Doc: "how to align text along the relevant dimension for the text element"}, {Name: "Padding", Doc: "Padding is used in a case-dependent manner to add space around text elements"}, {Name: "Rotation", Doc: "rotation of the text, in Degrees"}, {Name: "Offset", Doc: "Offset is added directly to the final label location."}}}) + +// SetAlign sets the [TextStyle.Align]: +// how to align text along the relevant dimension for the text element +func (t *TextStyle) SetAlign(v styles.Aligns) *TextStyle { t.Align = v; return t } + +// SetPadding sets the [TextStyle.Padding]: +// Padding is used in a case-dependent manner to add space around text elements +func (t *TextStyle) SetPadding(v units.Value) *TextStyle { t.Padding = v; return t } + +// SetRotation sets the [TextStyle.Rotation]: +// rotation of the text, in Degrees +func (t *TextStyle) SetRotation(v float32) *TextStyle { t.Rotation = v; return t } + +// SetOffset sets the [TextStyle.Offset]: +// Offset is added directly to the final label location. +func (t *TextStyle) SetOffset(v units.XY) *TextStyle { t.Offset = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Text", IDName: "text", Doc: "Text specifies a single text element in a plot", Fields: []types.Field{{Name: "Text", Doc: "text string, which can use HTML formatting"}, {Name: "Style", Doc: "styling for this text element"}, {Name: "PaintText", Doc: "PaintText is the [paint.Text] for the text."}}}) From 75e27b8ae924fb89966fea253ec9774414b026ac Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 8 Nov 2024 01:13:02 -0800 Subject: [PATCH 268/311] plot: all styling in place, elements updated, plots replicating. next is styling tests. --- base/iox/imagex/testing.go | 26 +++++++++ plot/README.md | 10 ++-- plot/draw.go | 32 ----------- plot/plot.go | 95 +++++++++++++++++++++++++++++---- plot/plots/barchart.go | 105 +++++++++++++++++++------------------ plot/plots/errbars.go | 14 ++++- plot/plots/labels.go | 12 +++-- plot/plots/plot_test.go | 1 - plot/plots/xy.go | 6 ++- plot/point.go | 2 +- plot/style.go | 8 +-- 11 files changed, 202 insertions(+), 109 deletions(-) diff --git a/base/iox/imagex/testing.go b/base/iox/imagex/testing.go index 1e2f3a91c5..662804c70a 100644 --- a/base/iox/imagex/testing.go +++ b/base/iox/imagex/testing.go @@ -12,6 +12,8 @@ import ( "os" "path/filepath" "strings" + + "cogentcore.org/core/base/num" ) // TestingT is an interface wrapper around *testing.T @@ -56,6 +58,25 @@ func CompareColors(cc, ic color.RGBA, tol int) bool { return true } +// DiffImage returns the difference between two images, +// with pixels having the abs of the difference between pixels. +func DiffImage(a, b image.Image) image.Image { + ab := a.Bounds() + di := image.NewRGBA(ab) + for y := ab.Min.Y; y < ab.Max.Y; y++ { + for x := ab.Min.X; x < ab.Max.X; x++ { + cc := color.RGBAModel.Convert(a.At(x, y)).(color.RGBA) + ic := color.RGBAModel.Convert(b.At(x, y)).(color.RGBA) + r := uint8(num.Abs(int(cc.R) - int(ic.R))) + g := uint8(num.Abs(int(cc.G) - int(ic.G))) + b := uint8(num.Abs(int(cc.B) - int(ic.B))) + c := color.RGBA{r, g, b, 255} + di.Set(x, y, c) + } + } + return di +} + // Assert asserts that the given image is equivalent // to the image stored at the given filename in the testdata directory, // with ".png" added to the filename if there is no extension @@ -77,6 +98,7 @@ func Assert(t TestingT, img image.Image, filename string) { ext := filepath.Ext(filename) failFilename := strings.TrimSuffix(filename, ext) + ".fail" + ext + diffFilename := strings.TrimSuffix(filename, ext) + ".diff" + ext if UpdateTestImages { err := Save(img, filename) @@ -133,6 +155,10 @@ func Assert(t TestingT, img image.Image, filename string) { if err != nil { t.Errorf("AssertImage: error saving fail image: %v", err) } + err = Save(DiffImage(img, fimg), diffFilename) + if err != nil { + t.Errorf("AssertImage: error saving diff image: %v", err) + } } else { err := os.RemoveAll(failFilename) if err != nil { diff --git a/plot/README.md b/plot/README.md index 7d8a6ce6ad..6a434e0614 100644 --- a/plot/README.md +++ b/plot/README.md @@ -6,20 +6,20 @@ The `plot` package generates 2D plots of data using the Cogent Core `paint` rend # Styling -`plot.Style` contains the full set of styling parameters, which can be set using Styler functions that are attached to individual plot elements that drive the content of what is actually plotted (based on the `Plotter` interface). +`plot.Style` contains the full set of styling parameters, which can be set using Styler functions that are attached to individual plot elements (e.g., lines, points etc) that drive the content of what is actually plotted (based on the `Plotter` interface). -Each such plot element defines the `Styler` method, eg: +Each such plot element defines a `Styler` method, e.g.,: ```Go plt := plot.NewPlot() ln := plots.AddLine.Styler(func(s *plot.Style) { - s.Plot.Title = "My Plot" - s.Line.Color = colors.Uniform(colors.Red) + s.Plot.Title = "My Plot" // overall Plot styles + s.Line.Color = colors.Uniform(colors.Red) // line-specific styles }) plt.Add(ln) ``` -The `Plot` field (of type `PlotStyle`) contains all the settings that apply to the plot as a whole. Each element can set these values, and they are applied in the order the elements are added, so the last one gets final say. Typically you want to just set these plot-level styles on one element only and avoid any conflicts. +The `Plot` field (of type `PlotStyle`) contains all the properties that apply to the plot as a whole. Each element can set these values, and they are applied in the order the elements are added, so the last one gets final say. Typically you want to just set these plot-level styles on one element only and avoid any conflicts. The rest of the style properties (e.g., `Line`, `Point`) apply to the element in question. There are also some default plot-level settings in `Plot` that apply to all elements, and the plot-level styles are updated first, so in this way it is possible to have plot-wide settings applied from one styler, that affect all plots (e.g., the line width, and whether lines and / or points are plotted or not). diff --git a/plot/draw.go b/plot/draw.go index b614ab5aac..72d7e6b70e 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -51,38 +51,6 @@ func (pt *Plot) SVGToFile(filename string) error { return bw.Flush() } -// applyStyle applies all the style parameters -func (pt *Plot) applyStyle() { - // first update the global plot style settings - var st Style - st.Defaults() - st.Plot = pt.Style - for _, plt := range pt.Plotters { - stlr := plt.Stylers() - stlr.Run(&st) - } - pt.Style = st.Plot - // then apply to elements - for _, plt := range pt.Plotters { - plt.ApplyStyle(&pt.Style) - } - // now style plot: - pt.Title.Style = pt.Style.TitleStyle - if pt.Style.Title != "" { - pt.Title.Text = pt.Style.Title - } - pt.Legend.Style = pt.Style.Legend - pt.Legend.Style.Text.openFont(pt) - pt.X.Style = pt.Style.Axis - pt.Y.Style = pt.Style.Axis - pt.X.Label.Style = pt.Style.Axis.Text - pt.Y.Label.Style = pt.Style.Axis.Text - pt.X.TickText.Style = pt.Style.Axis.TickText - pt.Y.TickText.Style = pt.Style.Axis.TickText - pt.Y.Label.Style.Rotation = -90 - pt.Y.Style.TickText.Align = styles.End -} - // drawConfig configures everything for drawing, applying styles etc. func (pt *Plot) drawConfig() { pt.Resize(pt.Size) // ensure diff --git a/plot/plot.go b/plot/plot.go index 0ceec6d877..265ec54812 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -19,9 +19,12 @@ import ( "cogentcore.org/core/math32" "cogentcore.org/core/paint" "cogentcore.org/core/styles" + "cogentcore.org/core/styles/units" ) -// PlotStyle has overall plot level styling parameters. +// PlotStyle has overall plot level styling properties. +// Some properties provide defaults for individual elements, which can +// then be overwritten by element-level properties. type PlotStyle struct { //types:add -setters // Title is the overall title of the plot. @@ -60,14 +63,56 @@ type PlotStyle struct { //types:add -setters // YAxisLabel is the optional label to use for the YAxis instead of the default. YAxisLabel string + + // LinesOn determines whether lines are plotted by default, + // for elements that plot lines (e.g., plots.XY). + LinesOn DefaultOffOn + + // LineWidth sets the default line width for data plotting lines. + LineWidth units.Value + + // PointsOn determines whether points are plotted by default, + // for elements that plot points (e.g., plots.XY). + PointsOn DefaultOffOn + + // PointSize sets the default point size. + PointSize units.Value + + // LabelSize sets the default label text size. + LabelSize units.Value + + // BarWidth for Bar plot sets the default width of the bars, + // which should be less than the Stride (1 typically) to prevent + // bar overlap. Defaults to .8. + BarWidth float32 } func (ps *PlotStyle) Defaults() { - ps.Legend.Defaults() - ps.Axis.Defaults() + ps.TitleStyle.Defaults() ps.TitleStyle.Size.Dp(24) ps.Background = colors.Scheme.Surface - ps.Scale = 1 + ps.Scale = 2 + ps.Legend.Defaults() + ps.Axis.Defaults() + ps.LineWidth.Pt(1) + ps.PointSize.Pt(4) + ps.LabelSize.Dp(16) + ps.BarWidth = .8 +} + +// SetElementStyle sets the properties for given element's style +// based on the global default settings in this PlotStyle. +func (ps *PlotStyle) SetElementStyle(es *Style) { + if ps.LinesOn != Default { + es.Line.On = ps.LinesOn + } + if ps.PointsOn != Default { + es.Point.On = ps.PointsOn + } + es.Line.Width = ps.LineWidth + es.Point.Size = ps.PointSize + es.Width.Width = ps.BarWidth + es.Text.Size = ps.LabelSize } // Plot is the basic type representing a plot. @@ -115,6 +160,13 @@ type Plot struct { PlotBox math32.Box2 } +// New returns a new plot with some reasonable default settings. +func New() *Plot { + pt := &Plot{} + pt.Defaults() + return pt +} + // Defaults sets defaults func (pt *Plot) Defaults() { pt.Style.Defaults() @@ -129,11 +181,36 @@ func (pt *Plot) Defaults() { pt.StandardTextStyle.WhiteSpace = styles.WhiteSpaceNowrap } -// New returns a new plot with some reasonable default settings. -func New() *Plot { - pt := &Plot{} - pt.Defaults() - return pt +// applyStyle applies all the style parameters +func (pt *Plot) applyStyle() { + // first update the global plot style settings + var st Style + st.Defaults() + st.Plot = pt.Style + for _, plt := range pt.Plotters { + stlr := plt.Stylers() + stlr.Run(&st) + } + pt.Style = st.Plot + // then apply to elements + for _, plt := range pt.Plotters { + plt.ApplyStyle(&pt.Style) + } + // now style plot: + pt.Title.Style = pt.Style.TitleStyle + if pt.Style.Title != "" { + pt.Title.Text = pt.Style.Title + } + pt.Legend.Style = pt.Style.Legend + pt.Legend.Style.Text.openFont(pt) + pt.X.Style = pt.Style.Axis + pt.Y.Style = pt.Style.Axis + pt.X.Label.Style = pt.Style.Axis.Text + pt.Y.Label.Style = pt.Style.Axis.Text + pt.X.TickText.Style = pt.Style.Axis.TickText + pt.Y.TickText.Style = pt.Style.Axis.TickText + pt.Y.Label.Style.Rotation = -90 + pt.Y.Style.TickText.Align = styles.End } // Add adds a Plotters to the plot. diff --git a/plot/plots/barchart.go b/plot/plots/barchart.go index ad4d822656..f43f69093d 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/barchart.go @@ -71,12 +71,12 @@ func NewBarChart(vs, ers plot.Valuer) *BarChart { return nil } } - b := &BarChart{ + bc := &BarChart{ Values: values, Errors: errs, } - b.Defaults() - return b + bc.Defaults() + return bc } // NewBarChartTensor returns a new bar chart with a single bar for each value. @@ -91,36 +91,41 @@ func NewBarChartTensor(vs, ers tensor.Tensor) *BarChart { return NewBarChart(vt, plot.TensorValues{ers}) } -func (b *BarChart) Defaults() { - b.Style.Defaults() +func (bc *BarChart) Defaults() { + bc.Style.Defaults() } -func (b *BarChart) Styler(f func(s *plot.Style)) *BarChart { - b.stylers.Add(f) - return b +func (bc *BarChart) Styler(f func(s *plot.Style)) *BarChart { + bc.stylers.Add(f) + return bc } -func (b *BarChart) ApplyStyle() { b.stylers.Run(&b.Style) } +func (bc *BarChart) ApplyStyle(ps *plot.PlotStyle) { + ps.SetElementStyle(&bc.Style) + bc.stylers.Run(&bc.Style) +} + +func (bc *BarChart) Stylers() *plot.Stylers { return &bc.stylers } -func (b *BarChart) XYData() (data plot.XYer, pixels plot.XYer) { - data = b.XYs - pixels = b.PXYs +func (bc *BarChart) XYData() (data plot.XYer, pixels plot.XYer) { + data = bc.XYs + pixels = bc.PXYs return } // BarHeight returns the maximum y value of the // ith bar, taking into account any bars upon // which it is stacked. -func (b *BarChart) BarHeight(i int) float32 { +func (bc *BarChart) BarHeight(i int) float32 { ht := float32(0.0) - if b == nil { + if bc == nil { return 0 } - if i >= 0 && i < len(b.Values) { - ht += b.Values[i] + if i >= 0 && i < len(bc.Values) { + ht += bc.Values[i] } - if b.StackedOn != nil { - ht += b.StackedOn.BarHeight(i) + if bc.StackedOn != nil { + ht += bc.StackedOn.BarHeight(i) } return ht } @@ -128,48 +133,48 @@ func (b *BarChart) BarHeight(i int) float32 { // StackOn stacks a bar chart on top of another, // and sets the bar positioning options to that of the // chart upon which it is being stacked. -func (b *BarChart) StackOn(on *BarChart) { - b.Style.Width = on.Style.Width - b.StackedOn = on +func (bc *BarChart) StackOn(on *BarChart) { + bc.Style.Width = on.Style.Width + bc.StackedOn = on } // Plot implements the plot.Plotter interface. -func (b *BarChart) Plot(plt *plot.Plot) { +func (bc *BarChart) Plot(plt *plot.Plot) { pc := plt.Paint - b.Style.Line.SetStroke(plt) - pc.FillStyle.Color = b.Style.Line.Fill - bw := b.Style.Width + bc.Style.Line.SetStroke(plt) + pc.FillStyle.Color = bc.Style.Line.Fill + bw := bc.Style.Width - nv := len(b.Values) - b.XYs = make(plot.XYs, nv) - b.PXYs = make(plot.XYs, nv) + nv := len(bc.Values) + bc.XYs = make(plot.XYs, nv) + bc.PXYs = make(plot.XYs, nv) hw := 0.5 * bw.Width ew := bw.Width / 3 - for i, ht := range b.Values { + for i, ht := range bc.Values { cat := bw.Offset + float32(i)*bw.Stride var bottom, catVal, catMin, catMax, valMin, valMax float32 var box math32.Box2 - if b.Horizontal { + if bc.Horizontal { catVal = plt.PY(cat) catMin = plt.PY(cat - hw) catMax = plt.PY(cat + hw) - bottom = b.StackedOn.BarHeight(i) // nil safe + bottom = bc.StackedOn.BarHeight(i) // nil safe valMin = plt.PX(bottom) valMax = plt.PX(bottom + ht) - b.XYs[i] = math32.Vec2(bottom+ht, cat) - b.PXYs[i] = math32.Vec2(valMax, catVal) + bc.XYs[i] = math32.Vec2(bottom+ht, cat) + bc.PXYs[i] = math32.Vec2(valMax, catVal) box.Min.Set(valMin, catMin) box.Max.Set(valMax, catMax) } else { catVal = plt.PX(cat) catMin = plt.PX(cat - hw) catMax = plt.PX(cat + hw) - bottom = b.StackedOn.BarHeight(i) // nil safe + bottom = bc.StackedOn.BarHeight(i) // nil safe valMin = plt.PY(bottom) valMax = plt.PY(bottom + ht) - b.XYs[i] = math32.Vec2(cat, bottom+ht) - b.PXYs[i] = math32.Vec2(catVal, valMax) + bc.XYs[i] = math32.Vec2(cat, bottom+ht) + bc.PXYs[i] = math32.Vec2(catVal, valMax) box.Min.Set(catMin, valMin) box.Max.Set(catMax, valMax) } @@ -177,9 +182,9 @@ func (b *BarChart) Plot(plt *plot.Plot) { pc.DrawRectangle(box.Min.X, box.Min.Y, box.Size().X, box.Size().Y) pc.FillStrokeClear() - if i < len(b.Errors) { - errval := b.Errors[i] - if b.Horizontal { + if i < len(bc.Errors) { + errval := bc.Errors[i] + if bc.Horizontal { eVal := plt.PX(bottom + ht + math32.Abs(errval)) pc.MoveTo(valMax, catVal) pc.LineTo(eVal, catVal) @@ -199,33 +204,33 @@ func (b *BarChart) Plot(plt *plot.Plot) { } // DataRange implements the plot.DataRanger interface. -func (b *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - bw := b.Style.Width +func (bc *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { + bw := bc.Style.Width catMin := bw.Offset - bw.Pad - catMax := bw.Offset + float32(len(b.Values)-1)*bw.Stride + bw.Pad + catMax := bw.Offset + float32(len(bc.Values)-1)*bw.Stride + bw.Pad valMin := math32.Inf(1) valMax := math32.Inf(-1) - for i, val := range b.Values { - valBot := b.StackedOn.BarHeight(i) + for i, val := range bc.Values { + valBot := bc.StackedOn.BarHeight(i) valTop := valBot + val - if i < len(b.Errors) { - valTop += math32.Abs(b.Errors[i]) + if i < len(bc.Errors) { + valTop += math32.Abs(bc.Errors[i]) } valMin = math32.Min(valMin, math32.Min(valBot, valTop)) valMax = math32.Max(valMax, math32.Max(valBot, valTop)) } - if !b.Horizontal { + if !bc.Horizontal { return catMin, catMax, valMin, valMax } return valMin, valMax, catMin, catMax } // Thumbnail fulfills the plot.Thumbnailer interface. -func (b *BarChart) Thumbnail(plt *plot.Plot) { +func (bc *BarChart) Thumbnail(plt *plot.Plot) { pc := plt.Paint - b.Style.Line.SetStroke(plt) - pc.FillStyle.Color = b.Style.Line.Fill + bc.Style.Line.SetStroke(plt) + pc.FillStyle.Color = bc.Style.Line.Fill ptb := pc.Bounds pc.DrawRectangle(float32(ptb.Min.X), float32(ptb.Min.Y), float32(ptb.Size().X), float32(ptb.Size().Y)) pc.FillStrokeClear() diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index a43bd95078..8182c1e198 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -104,7 +104,12 @@ func (eb *YErrorBars) Styler(f func(s *plot.Style)) *YErrorBars { return eb } -func (eb *YErrorBars) ApplyStyle() { eb.stylers.Run(&eb.Style) } +func (eb *YErrorBars) ApplyStyle(ps *plot.PlotStyle) { + ps.SetElementStyle(&eb.Style) + eb.stylers.Run(&eb.Style) +} + +func (eb *YErrorBars) Stylers() *plot.Stylers { return &eb.stylers } func (eb *YErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { data = eb.XYs @@ -217,7 +222,12 @@ func (eb *XErrorBars) Styler(f func(s *plot.Style)) *XErrorBars { return eb } -func (eb *XErrorBars) ApplyStyle() { eb.stylers.Run(&eb.Style) } +func (eb *XErrorBars) ApplyStyle(ps *plot.PlotStyle) { + ps.SetElementStyle(&eb.Style) + eb.stylers.Run(&eb.Style) +} + +func (eb *XErrorBars) Stylers() *plot.Stylers { return &eb.stylers } // Plot implements the Plotter interface, drawing labels. func (eb *XErrorBars) Plot(plt *plot.Plot) { diff --git a/plot/plots/labels.go b/plot/plots/labels.go index 2d5467f16e..ddf3a8704d 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -50,21 +50,27 @@ func NewLabels(d XYLabeler) *Labels { } lb := &Labels{XYs: xys, Labels: strs} - lb.Style.Defaults() + lb.Defaults() return lb } +func (lb *Labels) Defaults() { + lb.Style.Defaults() +} + // Styler adds a style function to set style parameters. func (lb *Labels) Styler(f func(s *plot.Style)) *Labels { lb.stylers.Add(f) return lb } -func (lb *Labels) ApplyStyle() { - st := lb.stylers.NewStyle() +func (lb *Labels) ApplyStyle(ps *plot.PlotStyle) { + st := lb.stylers.NewStyle(ps) lb.Style = st.Text } +func (lb *Labels) Stylers() *plot.Stylers { return &lb.stylers } + func (lb *Labels) XYData() (data plot.XYer, pixels plot.XYer) { data = lb.XYs pixels = lb.PXYs diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 841c514c60..b1d048aebc 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -99,7 +99,6 @@ func TestLine(t *testing.T) { l1.Style.Line.NegativeX = true plt.Draw() imagex.Assert(t, plt.Pixels, "line-negx.png") - } func TestScatter(t *testing.T) { diff --git a/plot/plots/xy.go b/plot/plots/xy.go index a6d2a34c05..1ca0297d93 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -81,8 +81,10 @@ func (ln *XY) Styler(f func(s *plot.Style)) *XY { return ln } -func (ln *XY) Stylers() { return &ln.stylers } -func (ln *XY) ApplyStyle(plotStyle *plot.Style) { +func (ln *XY) Stylers() *plot.Stylers { return &ln.stylers } + +func (ln *XY) ApplyStyle(ps *plot.PlotStyle) { + ps.SetElementStyle(&ln.Style) ln.stylers.Run(&ln.Style) } diff --git a/plot/point.go b/plot/point.go index 3b9269553d..e78c80e2cb 100644 --- a/plot/point.go +++ b/plot/point.go @@ -30,7 +30,7 @@ type PointStyle struct { //types:add -setters // Use nil to disable filling. Fill image.Image - // Width is the line width, with a default of 1 Pt (point). + // Width is the line width for point glyphs, with a default of 1 Pt (point). // Setting to 0 turns line off. Width units.Value diff --git a/plot/style.go b/plot/style.go index 4a57c78295..2560d8456e 100644 --- a/plot/style.go +++ b/plot/style.go @@ -73,9 +73,8 @@ type WidthStyle struct { //types:add -setters Stride float32 // Width for Bar plot is the width of the bars, which should be less than - // the Stride to prevent bar overlap. - // Defaults to .8 - Width float32 + // the Stride (1 typically) to prevent bar overlap. Defaults to .8. + Width float32 `min:"0.01" max:"1" default:"0.8"` // Pad for Bar plot is additional space at start / end of data range, // to keep bars from overflowing ends. This amount is subtracted from Offset @@ -111,8 +110,9 @@ func (st *Stylers) Run(s *Style) { // NewStyle returns a new Style object with styling functions applied // on top of Style defaults. -func (st *Stylers) NewStyle() *Style { +func (st *Stylers) NewStyle(ps *PlotStyle) *Style { s := NewStyle() + ps.SetElementStyle(s) st.Run(s) return s } From 003f14903e82c216f6f775002174001f6bcb43a9 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 8 Nov 2024 03:39:27 -0800 Subject: [PATCH 269/311] plot styling fully working including range; imagex/testing now saves a .diff image so you can see what is different. --- base/iox/imagex/testing.go | 2 + math32/minmax/range.go | 28 ++++++-- plot/data.go | 28 ++++++-- plot/plot.go | 44 ++++++++----- plot/plots/barchart.go | 2 + plot/plots/errbars.go | 2 +- plot/plots/plot_test.go | 132 +++++++++++++++++++++++-------------- plot/plots/xy.go | 2 +- plot/typegen.go | 88 ++++++++++++++++++------- 9 files changed, 227 insertions(+), 101 deletions(-) diff --git a/base/iox/imagex/testing.go b/base/iox/imagex/testing.go index 662804c70a..28cec1db34 100644 --- a/base/iox/imagex/testing.go +++ b/base/iox/imagex/testing.go @@ -109,6 +109,7 @@ func Assert(t TestingT, img image.Image, filename string) { if err != nil { t.Errorf("AssertImage: error removing old fail image: %v", err) } + os.RemoveAll(diffFilename) return } @@ -164,5 +165,6 @@ func Assert(t TestingT, img image.Image, filename string) { if err != nil { t.Errorf("AssertImage: error removing old fail image: %v", err) } + os.RemoveAll(diffFilename) } } diff --git a/math32/minmax/range.go b/math32/minmax/range.go index 2d73b938f5..83b92a2850 100644 --- a/math32/minmax/range.go +++ b/math32/minmax/range.go @@ -6,8 +6,6 @@ package minmax // Range32 represents a range of values for plotting, where the min or max can optionally be fixed type Range32 struct { - - // Min and Max range values F32 // fix the minimum end of the range @@ -34,13 +32,23 @@ func (rr *Range32) Range() float32 { return rr.Max - rr.Min } +// Clamp returns min, max values clamped according to Fixed min / max of range. +func (rr *Range32) Clamp(mnIn, mxIn float32) (mn, mx float32) { + mn, mx = mnIn, mxIn + if rr.FixMin && rr.Min < mn { + mn = rr.Min + } + if rr.FixMax && rr.Max > mx { + mx = rr.Max + } + return +} + /////////////////////////////////////////////////////////////////////// // 64 // Range64 represents a range of values for plotting, where the min or max can optionally be fixed type Range64 struct { - - // Min and Max range values F64 // fix the minimum end of the range @@ -66,3 +74,15 @@ func (rr *Range64) SetMax(mx float64) { func (rr *Range64) Range() float64 { return rr.Max - rr.Min } + +// Clamp returns min, max values clamped according to Fixed min / max of range. +func (rr *Range64) Clamp(mnIn, mxIn float64) (mn, mx float64) { + mn, mx = mnIn, mxIn + if rr.FixMin && rr.Min < mn { + mn = rr.Min + } + if rr.FixMax && rr.Max > mx { + mx = rr.Max + } + return +} diff --git a/plot/data.go b/plot/data.go index ad51b9b10c..fe9be936fa 100644 --- a/plot/data.go +++ b/plot/data.go @@ -13,6 +13,7 @@ import ( "errors" "cogentcore.org/core/math32" + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/tensor" ) @@ -67,20 +68,27 @@ type Valuer interface { } // Range returns the minimum and maximum values. -func Range(vs Valuer) (min, max float32) { - min = math32.Inf(1) - max = math32.Inf(-1) +func Range(vs Valuer) (mn, mx float32) { + mn = math32.Inf(1) + mx = math32.Inf(-1) for i := 0; i < vs.Len(); i++ { v := vs.Value(i) if math32.IsNaN(v) { continue } - min = math32.Min(min, v) - max = math32.Max(max, v) + mn = math32.Min(mn, v) + mx = math32.Max(mx, v) } return } +// RangeClamp returns the minimum and maximum values clamped by given range. +func RangeClamp(vs Valuer, rng *minmax.Range32) (mn, mx float32) { + mn, mx = Range(vs) + mn, mx = rng.Clamp(mn, mx) + return +} + // Values implements the Valuer interface. type Values []float32 @@ -139,14 +147,20 @@ type XYer interface { XY(i int) (x, y float32) } -// XYRange returns the minimum and maximum -// x and y values. +// XYRange returns the minimum and maximum x and y values. func XYRange(xys XYer) (xmin, xmax, ymin, ymax float32) { xmin, xmax = Range(XValues{xys}) ymin, ymax = Range(YValues{xys}) return } +// XYRangeClamp returns the data range with Range clamped for Y axis. +func XYRangeClamp(xys XYer, rng *minmax.Range32) (xmin, xmax, ymin, ymax float32) { + xmin, xmax, ymin, ymax = XYRange(xys) + ymin, ymax = rng.Clamp(ymin, ymax) + return +} + // XYs implements the XYer interface. type XYs []math32.Vector2 diff --git a/plot/plot.go b/plot/plot.go index 265ec54812..f86cd97103 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -17,11 +17,30 @@ import ( "cogentcore.org/core/base/iox/imagex" "cogentcore.org/core/colors" "cogentcore.org/core/math32" + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/paint" "cogentcore.org/core/styles" "cogentcore.org/core/styles/units" ) +// XAxisStyle has overall plot level styling properties for the XAxis. +type XAxisStyle struct { //types:add -setters + // Column specifies the column to use for the common X axis in a table based plot. + // if empty or not found, the row number is used. + // This optional for Bar plots, if present and Legend is also present, + // then an extra space will be put between X values. + Column string + + // Rotation is the rotation of the X Axis labels, in degrees. + Rotation float32 + + // Label is the optional label to use for the XAxis instead of the default. + Label string + + // Range is the effective range of XAxis data to plot, where either end can be fixed. + Range minmax.Range32 `display:"inline"` +} + // PlotStyle has overall plot level styling properties. // Some properties provide defaults for individual elements, which can // then be overwritten by element-level properties. @@ -49,17 +68,8 @@ type PlotStyle struct { //types:add -setters // Axis has the styling properties for the Axes. Axis AxisStyle - // XAxis specifies the column to use for the common X axis in a table based plot. - // if empty or not found, the row number is used. - // This optional for Bar plots, if present and Legend is also present, - // then an extra space will be put between X values. - XAxis string - - // XAxisRotation is the rotation of the X Axis labels, in degrees. - XAxisRotation float32 - - // XAxisLabel is the optional label to use for the XAxis instead of the default. - XAxisLabel string + // XAxis has plot-level XAxis style properties. + XAxis XAxisStyle // YAxisLabel is the optional label to use for the YAxis instead of the default. YAxisLabel string @@ -91,7 +101,7 @@ func (ps *PlotStyle) Defaults() { ps.TitleStyle.Defaults() ps.TitleStyle.Size.Dp(24) ps.Background = colors.Scheme.Surface - ps.Scale = 2 + ps.Scale = 1 ps.Legend.Defaults() ps.Axis.Defaults() ps.LineWidth.Pt(1) @@ -118,9 +128,6 @@ func (ps *PlotStyle) SetElementStyle(es *Style) { // Plot is the basic type representing a plot. // It renders into its own image.RGBA Pixels image, // and can also save a corresponding SVG version. -// The Axis ranges are updated automatically when plots -// are added, so setting a fixed range should happen -// after that point. See [UpdateRange] method as well. type Plot struct { // Title of the plot Title Text @@ -197,6 +204,7 @@ func (pt *Plot) applyStyle() { plt.ApplyStyle(&pt.Style) } // now style plot: + pt.DPI *= pt.Style.Scale pt.Title.Style = pt.Style.TitleStyle if pt.Style.Title != "" { pt.Title.Text = pt.Style.Title @@ -205,12 +213,16 @@ func (pt *Plot) applyStyle() { pt.Legend.Style.Text.openFont(pt) pt.X.Style = pt.Style.Axis pt.Y.Style = pt.Style.Axis + pt.X.Label.Text = pt.Style.XAxis.Label + pt.Y.Label.Text = pt.Style.YAxisLabel pt.X.Label.Style = pt.Style.Axis.Text pt.Y.Label.Style = pt.Style.Axis.Text pt.X.TickText.Style = pt.Style.Axis.TickText + pt.X.TickText.Style.Rotation = pt.Style.XAxis.Rotation pt.Y.TickText.Style = pt.Style.Axis.TickText pt.Y.Label.Style.Rotation = -90 pt.Y.Style.TickText.Align = styles.End + pt.UpdateRange() } // Add adds a Plotters to the plot. @@ -232,7 +244,6 @@ func (pt *Plot) SetPixels(img *image.RGBA) { pt.Paint = paint.NewContextFromImage(pt.Pixels) pt.Paint.UnitContext.DPI = pt.DPI pt.Size = pt.Pixels.Bounds().Size() - pt.UpdateRange() // needs context, to automatically update for labels } // Resize sets the size of the output image to given size. @@ -326,6 +337,7 @@ func (pt *Plot) UpdateRangeFromPlotter(d Plotter) { pt.Y.Min = math32.Min(pt.Y.Min, ymin) pt.Y.Max = math32.Max(pt.Y.Max, ymax) } + pt.X.Min, pt.X.Max = pt.Style.XAxis.Range.Clamp(pt.X.Min, pt.X.Max) } // PX returns the X-axis plotting coordinate for given raw data value diff --git a/plot/plots/barchart.go b/plot/plots/barchart.go index f43f69093d..d2f3c045ab 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/barchart.go @@ -221,8 +221,10 @@ func (bc *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { valMax = math32.Max(valMax, math32.Max(valBot, valTop)) } if !bc.Horizontal { + valMin, valMax = bc.Style.Range.Clamp(valMin, valMax) return catMin, catMax, valMin, valMax } + catMin, catMax = bc.Style.Range.Clamp(catMin, catMax) return valMin, valMax, catMin, catMax } diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 8182c1e198..1ba1ab3edd 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -149,7 +149,7 @@ func (eb *YErrorBars) Plot(plt *plot.Plot) { // DataRange implements the plot.DataRanger interface. func (eb *YErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - xmin, xmax = plot.Range(plot.XValues{eb}) + xmin, xmax = plot.RangeClamp(plot.XValues{eb}, &eb.Style.Range) ymin = math32.Inf(1) ymax = math32.Inf(-1) for i, err := range eb.YErrors { diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index b1d048aebc..103c7906ea 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -44,17 +44,8 @@ func TestMain(m *testing.M) { os.Exit(m.Run()) } -func TestLine(t *testing.T) { - plt := plot.New() - plt.Title.Text = "Test Line" - plt.X.Min = 0 - plt.X.Max = 100 - plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 - plt.Y.Label.Text = "Y Axis" - - // note: making two overlapping series +// sinCosWrapData returns overlapping sin / cos curves in one sequence. +func sinCosWrapData() plot.XYs { data := make(plot.XYs, 42) for i := range data { x := float32(i % 21) @@ -65,6 +56,47 @@ func TestLine(t *testing.T) { data[i].Y = float32(50) + 40*math32.Cos((x/8)*math32.Pi) } } + return data +} + +func sinDataXY() plot.XYs { + data := make(plot.XYs, 21) + for i := range data { + data[i].X = float32(i * 5) + data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) + } + return data +} + +func sinData() plot.Values { + sin := make(plot.Values, 21) + for i := range sin { + x := float32(i % 21) + sin[i] = float32(50) + 40*math32.Sin((x/8)*math32.Pi) + } + return sin +} + +func cosData() plot.Values { + cos := make(plot.Values, 21) + for i := range cos { + x := float32(i % 21) + cos[i] = float32(50) + 40*math32.Cos((x/8)*math32.Pi) + } + return cos +} + +func TestLine(t *testing.T) { + data := sinCosWrapData() + + plt := plot.New() + plt.Title.Text = "Test Line" + plt.X.Min = 0 + plt.X.Max = 100 + plt.X.Label.Text = "X Axis" + plt.Y.Min = 0 + plt.Y.Max = 100 + plt.Y.Label.Text = "Y Axis" l1 := NewLine(data) if l1 == nil { @@ -102,6 +134,8 @@ func TestLine(t *testing.T) { } func TestScatter(t *testing.T) { + data := sinDataXY() + plt := plot.New() plt.Title.Text = "Test Scatter" plt.X.Min = 0 @@ -111,12 +145,6 @@ func TestScatter(t *testing.T) { plt.Y.Max = 100 plt.Y.Label.Text = "Y Axis" - data := make(plot.XYs, 21) - for i := range data { - data[i].X = float32(i * 5) - data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) - } - l1 := NewScatter(data) if l1 == nil { t.Error("bad data") @@ -178,17 +206,8 @@ func TestBarChart(t *testing.T) { plt.Y.Max = 100 plt.Y.Label.Text = "Y Axis" - data := make(plot.Values, 21) - for i := range data { - x := float32(i % 21) - data[i] = float32(50) + 40*math32.Sin((x/8)*math32.Pi) - } - - cos := make(plot.Values, 21) - for i := range data { - x := float32(i % 21) - cos[i] = float32(50) + 40*math32.Cos((x/8)*math32.Pi) - } + data := sinData() + cos := cosData() l1 := NewBarChart(data, nil) if l1 == nil { @@ -226,17 +245,8 @@ func TestBarChartErr(t *testing.T) { plt.Y.Max = 100 plt.Y.Label.Text = "Y Axis" - data := make(plot.Values, 21) - for i := range data { - x := float32(i % 21) - data[i] = float32(50) + 40*math32.Sin((x/8)*math32.Pi) - } - - cos := make(plot.Values, 21) - for i := range data { - x := float32(i % 21) - cos[i] = float32(5) + 4*math32.Cos((x/8)*math32.Pi) - } + data := sinData() + cos := cosData() l1 := NewBarChart(data, cos) if l1 == nil { @@ -266,17 +276,8 @@ func TestBarChartStack(t *testing.T) { plt.Y.Max = 100 plt.Y.Label.Text = "Y Axis" - data := make(plot.Values, 21) - for i := range data { - x := float32(i % 21) - data[i] = float32(50) + 40*math32.Sin((x/8)*math32.Pi) - } - - cos := make(plot.Values, 21) - for i := range data { - x := float32(i % 21) - cos[i] = float32(5) + 4*math32.Cos((x/8)*math32.Pi) - } + data := sinData() + cos := cosData() l1 := NewBarChart(data, nil) if l1 == nil { @@ -346,3 +347,34 @@ func TestErrBar(t *testing.T) { plt.Draw() imagex.Assert(t, plt.Pixels, "errbar.png") } + +func TestStyle(t *testing.T) { + data := sinCosWrapData() + + plt := plot.New() + l1 := NewLine(data).Styler(func(s *plot.Style) { + s.Plot.Title = "Test Line" + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plot.XAxis.Range.SetMax(105) + s.Plot.LineWidth.Pt(2) + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) + s.Plot.TitleStyle.Size.Dp(48) + s.Plot.Legend.Position.Left = true + s.Plot.Legend.Text.Size.Dp(24) + s.Plot.Axis.Text.Size.Dp(32) + s.Plot.Axis.TickText.Size.Dp(24) + s.Plot.XAxis.Rotation = -45 + // s.Line.On = plot.Off + s.Line.Color = colors.Uniform(colors.Red) + s.Point.Color = colors.Uniform(colors.Blue) + s.Range.SetMax(100) + }) + plt.Add(l1) + plt.Legend.Add("Sine", l1) + plt.Legend.Add("Cos", l1) + + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "style_line_point.png") +} diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 1ca0297d93..82dcdc4836 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -198,7 +198,7 @@ func (ln *XY) Plot(plt *plot.Plot) { // DataRange returns the minimum and maximum // x and y values, implementing the plot.DataRanger interface. func (ln *XY) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - return plot.XYRange(ln) + return plot.XYRangeClamp(ln, &ln.Style.Range) } // Thumbnail returns the thumbnail, implementing the plot.Thumbnailer interface. diff --git a/plot/typegen.go b/plot/typegen.go index 02414b35f8..0e086d62ea 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -151,7 +151,28 @@ func (t *LineStyle) SetStep(v StepKind) *LineStyle { t.Step = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.StepKind", IDName: "step-kind", Doc: "StepKind specifies a form of a connection of two consecutive points."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotStyle", IDName: "plot-style", Doc: "PlotStyle has overall plot level styling parameters.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Title", Doc: "Title is the overall title of the plot."}, {Name: "TitleStyle", Doc: "TitleStyle is the text styling parameters for the title."}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Legend", Doc: "Legend has the styling properties for the Legend."}, {Name: "Axis", Doc: "Axis has the styling properties for the Axes."}, {Name: "XAxis", Doc: "XAxis specifies the column to use for the common X axis in a table based plot.\nif empty or not found, the row number is used.\nThis optional for Bar plots, if present and Legend is also present,\nthen an extra space will be put between X values."}, {Name: "XAxisRotation", Doc: "XAxisRotation is the rotation of the X Axis labels, in degrees."}, {Name: "XAxisLabel", Doc: "XAxisLabel is the optional label to use for the XAxis instead of the default."}, {Name: "YAxisLabel", Doc: "YAxisLabel is the optional label to use for the YAxis instead of the default."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XAxisStyle", IDName: "x-axis-style", Doc: "XAxisStyle has overall plot level styling properties for the XAxis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Column", Doc: "Column specifies the column to use for the common X axis in a table based plot.\nif empty or not found, the row number is used.\nThis optional for Bar plots, if present and Legend is also present,\nthen an extra space will be put between X values."}, {Name: "Rotation", Doc: "Rotation is the rotation of the X Axis labels, in degrees."}, {Name: "Label", Doc: "Label is the optional label to use for the XAxis instead of the default."}, {Name: "Range", Doc: "Range is the effective range of XAxis data to plot, where either end can be fixed."}}}) + +// SetColumn sets the [XAxisStyle.Column]: +// Column specifies the column to use for the common X axis in a table based plot. +// if empty or not found, the row number is used. +// This optional for Bar plots, if present and Legend is also present, +// then an extra space will be put between X values. +func (t *XAxisStyle) SetColumn(v string) *XAxisStyle { t.Column = v; return t } + +// SetRotation sets the [XAxisStyle.Rotation]: +// Rotation is the rotation of the X Axis labels, in degrees. +func (t *XAxisStyle) SetRotation(v float32) *XAxisStyle { t.Rotation = v; return t } + +// SetLabel sets the [XAxisStyle.Label]: +// Label is the optional label to use for the XAxis instead of the default. +func (t *XAxisStyle) SetLabel(v string) *XAxisStyle { t.Label = v; return t } + +// SetRange sets the [XAxisStyle.Range]: +// Range is the effective range of XAxis data to plot, where either end can be fixed. +func (t *XAxisStyle) SetRange(v minmax.Range32) *XAxisStyle { t.Range = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotStyle", IDName: "plot-style", Doc: "PlotStyle has overall plot level styling properties.\nSome properties provide defaults for individual elements, which can\nthen be overwritten by element-level properties.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Title", Doc: "Title is the overall title of the plot."}, {Name: "TitleStyle", Doc: "TitleStyle is the text styling parameters for the title."}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Legend", Doc: "Legend has the styling properties for the Legend."}, {Name: "Axis", Doc: "Axis has the styling properties for the Axes."}, {Name: "XAxis", Doc: "XAxis has plot-level XAxis style properties."}, {Name: "YAxisLabel", Doc: "YAxisLabel is the optional label to use for the YAxis instead of the default."}, {Name: "LinesOn", Doc: "LinesOn determines whether lines are plotted by default,\nfor elements that plot lines (e.g., plots.XY)."}, {Name: "LineWidth", Doc: "LineWidth sets the default line width for data plotting lines."}, {Name: "PointsOn", Doc: "PointsOn determines whether points are plotted by default,\nfor elements that plot points (e.g., plots.XY)."}, {Name: "PointSize", Doc: "PointSize sets the default point size."}, {Name: "LabelSize", Doc: "LabelSize sets the default label text size."}, {Name: "BarWidth", Doc: "BarWidth for Bar plot sets the default width of the bars,\nwhich should be less than the Stride (1 typically) to prevent\nbar overlap. Defaults to .8."}}}) // SetTitle sets the [PlotStyle.Title]: // Title is the overall title of the plot. @@ -182,31 +203,48 @@ func (t *PlotStyle) SetLegend(v LegendStyle) *PlotStyle { t.Legend = v; return t func (t *PlotStyle) SetAxis(v AxisStyle) *PlotStyle { t.Axis = v; return t } // SetXAxis sets the [PlotStyle.XAxis]: -// XAxis specifies the column to use for the common X axis in a table based plot. -// if empty or not found, the row number is used. -// This optional for Bar plots, if present and Legend is also present, -// then an extra space will be put between X values. -func (t *PlotStyle) SetXAxis(v string) *PlotStyle { t.XAxis = v; return t } - -// SetXAxisRotation sets the [PlotStyle.XAxisRotation]: -// XAxisRotation is the rotation of the X Axis labels, in degrees. -func (t *PlotStyle) SetXAxisRotation(v float32) *PlotStyle { t.XAxisRotation = v; return t } - -// SetXAxisLabel sets the [PlotStyle.XAxisLabel]: -// XAxisLabel is the optional label to use for the XAxis instead of the default. -func (t *PlotStyle) SetXAxisLabel(v string) *PlotStyle { t.XAxisLabel = v; return t } +// XAxis has plot-level XAxis style properties. +func (t *PlotStyle) SetXAxis(v XAxisStyle) *PlotStyle { t.XAxis = v; return t } // SetYAxisLabel sets the [PlotStyle.YAxisLabel]: // YAxisLabel is the optional label to use for the YAxis instead of the default. func (t *PlotStyle) SetYAxisLabel(v string) *PlotStyle { t.YAxisLabel = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.\nThe Axis ranges are updated automatically when plots\nare added, so setting a fixed range should happen\nafter that point. See [UpdateRange] method as well.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Y", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) +// SetLinesOn sets the [PlotStyle.LinesOn]: +// LinesOn determines whether lines are plotted by default, +// for elements that plot lines (e.g., plots.XY). +func (t *PlotStyle) SetLinesOn(v DefaultOffOn) *PlotStyle { t.LinesOn = v; return t } + +// SetLineWidth sets the [PlotStyle.LineWidth]: +// LineWidth sets the default line width for data plotting lines. +func (t *PlotStyle) SetLineWidth(v units.Value) *PlotStyle { t.LineWidth = v; return t } + +// SetPointsOn sets the [PlotStyle.PointsOn]: +// PointsOn determines whether points are plotted by default, +// for elements that plot points (e.g., plots.XY). +func (t *PlotStyle) SetPointsOn(v DefaultOffOn) *PlotStyle { t.PointsOn = v; return t } + +// SetPointSize sets the [PlotStyle.PointSize]: +// PointSize sets the default point size. +func (t *PlotStyle) SetPointSize(v units.Value) *PlotStyle { t.PointSize = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nSome standard implementations of Plotter can be found in plotters.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element."}}}) +// SetLabelSize sets the [PlotStyle.LabelSize]: +// LabelSize sets the default label text size. +func (t *PlotStyle) SetLabelSize(v units.Value) *PlotStyle { t.LabelSize = v; return t } + +// SetBarWidth sets the [PlotStyle.BarWidth]: +// BarWidth for Bar plot sets the default width of the bars, +// which should be less than the Stride (1 typically) to prevent +// bar overlap. Defaults to .8. +func (t *PlotStyle) SetBarWidth(v float32) *PlotStyle { t.BarWidth = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Y", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DataRanger", IDName: "data-ranger", Doc: "DataRanger wraps the DataRange method.", Methods: []types.Method{{Name: "DataRange", Doc: "DataRange returns the range of X and Y values.", Args: []string{"pt"}, Returns: []string{"xmin", "xmax", "ymin", "ymax"}}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PointStyle", IDName: "point-style", Doc: "PointStyle has style properties for drawing points as different shapes.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On indicates whether to plot points."}, {Name: "Shape", Doc: "Shape to draw."}, {Name: "Color", Doc: "Color is the stroke color image specification.\nSetting to nil turns line off."}, {Name: "Fill", Doc: "Fill is the color to fill solid regions, in a plot-specific\nway (e.g., the area below a Line plot, the bar color).\nUse nil to disable filling."}, {Name: "Width", Doc: "Width is the line width, with a default of 1 Pt (point).\nSetting to 0 turns line off."}, {Name: "Size", Doc: "Size of shape to draw for each point.\nDefaults to 4 Pt (point)."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PointStyle", IDName: "point-style", Doc: "PointStyle has style properties for drawing points as different shapes.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On indicates whether to plot points."}, {Name: "Shape", Doc: "Shape to draw."}, {Name: "Color", Doc: "Color is the stroke color image specification.\nSetting to nil turns line off."}, {Name: "Fill", Doc: "Fill is the color to fill solid regions, in a plot-specific\nway (e.g., the area below a Line plot, the bar color).\nUse nil to disable filling."}, {Name: "Width", Doc: "Width is the line width for point glyphs, with a default of 1 Pt (point).\nSetting to 0 turns line off."}, {Name: "Size", Doc: "Size of shape to draw for each point.\nDefaults to 4 Pt (point)."}}}) // SetOn sets the [PointStyle.On]: // On indicates whether to plot points. @@ -228,7 +266,7 @@ func (t *PointStyle) SetColor(v image.Image) *PointStyle { t.Color = v; return t func (t *PointStyle) SetFill(v image.Image) *PointStyle { t.Fill = v; return t } // SetWidth sets the [PointStyle.Width]: -// Width is the line width, with a default of 1 Pt (point). +// Width is the line width for point glyphs, with a default of 1 Pt (point). // Setting to 0 turns line off. func (t *PointStyle) SetWidth(v units.Value) *PointStyle { t.Width = v; return t } @@ -239,7 +277,14 @@ func (t *PointStyle) SetSize(v units.Value) *PointStyle { t.Size = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Shapes", IDName: "shapes"}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply both to individual plot elements\nand to the plot as a whole.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On specifies whether to plot this item, for cases where it can be turned off."}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for cases where it can be turned off."}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) + +// SetPlot sets the [Style.Plot]: +// +// Plot has overall plot-level properties, which can be set by any +// +// plot element, and are updated first, before applying element-wise styles. +func (t *Style) SetPlot(v PlotStyle) *Style { t.Plot = v; return t } // SetOn sets the [Style.On]: // On specifies whether to plot this item, for cases where it can be turned off. @@ -273,7 +318,7 @@ func (t *Style) SetText(v TextStyle) *Style { t.Text = v; return t } // Width has various plot width properties. func (t *Style) SetWidth(v WidthStyle) *Style { t.Width = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.WidthStyle", IDName: "width-style", Doc: "WidthStyle contains various plot width properties relevant across\ndifferent plot types.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Cap", Doc: "Cap is the width of the caps drawn at the top of error bars.\nThe default is 10dp"}, {Name: "Offset", Doc: "Offset for Bar plot is the offset added to each X axis value\nrelative to the Stride computed value (X = offset + index * Stride)\nDefaults to 1."}, {Name: "Stride", Doc: "Stride for Bar plot is distance between bars. Defaults to 1."}, {Name: "Width", Doc: "Width for Bar plot is the width of the bars, which should be less than\nthe Stride to prevent bar overlap.\nDefaults to .8"}, {Name: "Pad", Doc: "Pad for Bar plot is additional space at start / end of data range,\nto keep bars from overflowing ends. This amount is subtracted from Offset\nand added to (len(Values)-1)*Stride -- no other accommodation for bar\nwidth is provided, so that should be built into this value as well.\nDefaults to 1."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.WidthStyle", IDName: "width-style", Doc: "WidthStyle contains various plot width properties relevant across\ndifferent plot types.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Cap", Doc: "Cap is the width of the caps drawn at the top of error bars.\nThe default is 10dp"}, {Name: "Offset", Doc: "Offset for Bar plot is the offset added to each X axis value\nrelative to the Stride computed value (X = offset + index * Stride)\nDefaults to 1."}, {Name: "Stride", Doc: "Stride for Bar plot is distance between bars. Defaults to 1."}, {Name: "Width", Doc: "Width for Bar plot is the width of the bars, which should be less than\nthe Stride (1 typically) to prevent bar overlap. Defaults to .8."}, {Name: "Pad", Doc: "Pad for Bar plot is additional space at start / end of data range,\nto keep bars from overflowing ends. This amount is subtracted from Offset\nand added to (len(Values)-1)*Stride -- no other accommodation for bar\nwidth is provided, so that should be built into this value as well.\nDefaults to 1."}}}) // SetCap sets the [WidthStyle.Cap]: // Cap is the width of the caps drawn at the top of error bars. @@ -292,8 +337,7 @@ func (t *WidthStyle) SetStride(v float32) *WidthStyle { t.Stride = v; return t } // SetWidth sets the [WidthStyle.Width]: // Width for Bar plot is the width of the bars, which should be less than -// the Stride to prevent bar overlap. -// Defaults to .8 +// the Stride (1 typically) to prevent bar overlap. Defaults to .8. func (t *WidthStyle) SetWidth(v float32) *WidthStyle { t.Width = v; return t } // SetPad sets the [WidthStyle.Pad]: From b95d5031166c1ab77172399b114ea95671c0148b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 03:33:47 -0800 Subject: [PATCH 270/311] new plot data framework all in place and building, but tests are failing --- math32/minmax/minmax32.go | 23 +- math32/minmax/minmax64.go | 33 ++- plot/README.md | 103 ++++++++- plot/axis.go | 50 ++-- plot/data.go | 354 +++++++++++------------------ plot/draw.go | 4 +- plot/labelling.go | 80 +++---- plot/plot.go | 61 ++--- plot/plotcore/barplot.go | 2 +- plot/plotcore/tablexy.go | 2 +- plot/plots/{barchart.go => bar.go} | 174 +++++++------- plot/plots/doc.go | 3 +- plot/plots/errbars.go | 244 ++++++++------------ plot/plots/labels.go | 104 ++++----- plot/plots/plot_test.go | 204 +++++++++-------- plot/plots/table.go | 64 +++--- plot/plots/xy.go | 185 ++++++++------- plot/plotter.go | 62 ++++- plot/point.go | 1 + plot/style.go | 18 +- plot/tick.go | 47 ++-- plot/typegen.go | 62 +++-- 22 files changed, 949 insertions(+), 931 deletions(-) rename plot/plots/{barchart.go => bar.go} (53%) diff --git a/math32/minmax/minmax32.go b/math32/minmax/minmax32.go index eaa7e68f99..d7c96b742f 100644 --- a/math32/minmax/minmax32.go +++ b/math32/minmax/minmax32.go @@ -4,7 +4,11 @@ package minmax -import "fmt" +import ( + "fmt" + + "cogentcore.org/core/math32" +) //gosl:start @@ -139,3 +143,20 @@ func (mr *F32) FitInRange(oth F32) bool { } return adj } + +// Sanitize ensures that the Min / Max range is not infinite or contradictory. +func (mr *F32) Sanitize() { + if math32.IsInf(mr.Min, 0) { + mr.Min = 0 + } + if math32.IsInf(mr.Max, 0) { + mr.Max = 0 + } + if mr.Min > mr.Max { + mr.Min, mr.Max = mr.Max, mr.Min + } + if mr.Min == mr.Max { + mr.Min-- + mr.Max++ + } +} diff --git a/math32/minmax/minmax64.go b/math32/minmax/minmax64.go index 046c7c3958..011db38328 100644 --- a/math32/minmax/minmax64.go +++ b/math32/minmax/minmax64.go @@ -5,6 +5,8 @@ // Package minmax provides a struct that holds Min and Max values. package minmax +import "math" + //go:generate core generate const ( @@ -26,38 +28,38 @@ func (mr *F64) Set(mn, mx float64) { } // SetInfinity sets the Min to +MaxFloat, Max to -MaxFloat -- suitable for -// iteratively calling Fit*InRange +// iteratively calling Fit*InRange. func (mr *F64) SetInfinity() { mr.Min = MaxFloat64 mr.Max = -MaxFloat64 } -// IsValid returns true if Min <= Max +// IsValid returns true if Min <= Max. func (mr *F64) IsValid() bool { return mr.Min <= mr.Max } -// InRange tests whether value is within the range (>= Min and <= Max) +// InRange tests whether value is within the range (>= Min and <= Max). func (mr *F64) InRange(val float64) bool { return ((val >= mr.Min) && (val <= mr.Max)) } -// IsLow tests whether value is lower than the minimum +// IsLow tests whether value is lower than the minimum. func (mr *F64) IsLow(val float64) bool { return (val < mr.Min) } -// IsHigh tests whether value is higher than the maximum +// IsHigh tests whether value is higher than the maximum. func (mr *F64) IsHigh(val float64) bool { return (val > mr.Min) } -// Range returns Max - Min +// Range returns Max - Min. func (mr *F64) Range() float64 { return mr.Max - mr.Min } -// Scale returns 1 / Range -- if Range = 0 then returns 0 +// Scale returns 1 / Range -- if Range = 0 then returns 0. func (mr *F64) Scale() float64 { r := mr.Range() if r != 0 { @@ -135,3 +137,20 @@ func (mr *F64) FitInRange(oth F64) bool { } return adj } + +// Sanitize ensures that the Min / Max range is not infinite or contradictory. +func (mr *F64) Sanitize() { + if math.IsInf(mr.Min, 0) { + mr.Min = 0 + } + if math.IsInf(mr.Max, 0) { + mr.Max = 0 + } + if mr.Min > mr.Max { + mr.Min, mr.Max = mr.Max, mr.Min + } + if mr.Min == mr.Max { + mr.Min-- + mr.Max++ + } +} diff --git a/plot/README.md b/plot/README.md index 6a434e0614..bf7ba363cb 100644 --- a/plot/README.md +++ b/plot/README.md @@ -1,8 +1,21 @@ # Plot The `plot` package generates 2D plots of data using the Cogent Core `paint` rendering system. The `plotcore` sub-package has Cogent Core Widgets that can be used in applications. -* `Plot` is just a wrapper around a `plot.Plot`, for manually-configured plots. -* `PlotEditor` is an interactive plot viewer that supports selection of which data to plot, and configuration of many plot parameters. +* `Plot` is just a wrapper around a `plot.Plot`, for code-generated plots. +* `PlotEditor` is an interactive plot viewer that supports selection of which data to plot, and GUI configuration of plot parameters. + +`plot` is designed to work in two potentially-conflicting ways: +* Code-based creation of a specific plot with specific data. +* GUI-based configuration of plots based on a `tensor.Table` of data columns (via `PlotEditor`). + +The GUI constraint requires a more systematic, factorial organization of the space of possible plot data and how it is organized to create a plot, so that it can be configured with a relatively simple set of GUI settings. The overall logic is as follows: +* The overall plot has a single shared range of X, Y and optionally Z coordinate ranges, that defines where a data value in any plot type is plotted. These ranges are set based on the DataRanger interface. +* Plot content is driven by `Plotter` elements that each consume one or more sets of data, which is provided by a `Data` interface that maps onto a minimal subset of the `tensor.Tensor` interface, so a tensor directly satisfies the interface. +* Each `Plotter` element can generally handle multiple different data elements, that are index-aligned. For example, the basic `XY` plotter requires `X` and `Y` Valuers, and optionally `Size` or `Color` Valuers that apply to the Point elements, while `Bar` gets at least a `Y` but also optionally a `High` Valuer for an error bar. + +The table-driven plotting case uses a `Group` name along with the `Roles` type (`X`, `Y` etc) and Plotter type names to organize different plots based on `Style` settings. Columns with the same Group name all provide data to the same plotter using their different Roles, making it easy to configure various statistical plots of multiple series of grouped data. + +Different plotter types (including custom ones) are registered along with their accepted input roles, to allow any type of plot to be generated. # Styling @@ -23,6 +36,88 @@ The `Plot` field (of type `PlotStyle`) contains all the properties that apply to The rest of the style properties (e.g., `Line`, `Point`) apply to the element in question. There are also some default plot-level settings in `Plot` that apply to all elements, and the plot-level styles are updated first, so in this way it is possible to have plot-wide settings applied from one styler, that affect all plots (e.g., the line width, and whether lines and / or points are plotted or not). +# Plot Types + +The following are the builtin standard plot types, in the `plots` package: + +## 1D and 2D XY Data + +### XY + +`XY` is the workhorse standard Plotter, taking at least `X` and `Y` inputs, and plotting lines and / or points at each X, Y point. + +Optionally `Size` and / or `Color` inputs can be provided, which apply to the points. Thus, by using a `Point.Shape` of `Ring` or `Circle`, you can create a bubble plot by providing Size and Color data. + +### Bar + +`Bar` takes `Y` inputs, and draws bars of corresponding height. + +An optional `High` input can be provided to also plot error bars above each bar. + +To create a plot with multiple error bars, multiple Bar Plotters are created, with `Style.Width` parameters that have a shared `Stride = 1 / number of bars` and `Offset` that increments for each bar added. The `plots.NewBars` function handles this directly. + +### ErrorBar + +`XErrorBar` and `YErrorBar` take `X`, `Y`, `Low`, and `High` inputs, and draws an `I` shaped error bar at the X, Y coordinate with the error "handles" around it. + +### Labels + +`Labels` takes `X`, `Y` and `Labels` string inputs and plots labels at the given coordinates. + +### Box + +`Box` takes `X`, `Y` (median line), `U`, `V` (box first and 3rd quartile values), and `Low`, `High` (Min, Max) inputs, and renders a box plot with error bars. + +### XFill, YFill + +`XFill` and `YFill` are used to draw filled regions between pairs of X or Y points, using the `X`, `Y`, and `BoxLow`, `BoxHigh` values to specify the center point (X, Y) and the region below / left and above / right to fill around that central point. + +XFill along with an XY line can be used to draw the equivalent of the [matplotlib fill_between](https://matplotlib.org/stable/plot_types/basic/fill_between.html#sphx-glr-plot-types-basic-fill-between-py) plot. + +YFill can be used to draw the equivalent of the [matplotlib violin plot](https://matplotlib.org/stable/plot_types/stats/violin.html#sphx-glr-plot-types-stats-violin-py). + +### Pie + +TODO + +## 2D Grid-based + +### ColorGrid + +Input = Values and X, Y size + +### Contour + +?? + +### Vector + +X,Y,U,V + +Quiver? + +## 3D + +TODO: use math32 3D projection math and you can just take each 3d point and reduce to 2D. For stuff you want to actually be able to use in SVG, it needs to ultimately be 2D, so it makes sense to support basic versions here, including XYZ (points, lines), Bar3D, wireframe. + +Could also have a separate plot3d package based on `xyz` that is true 3D for interactive 3D plots of surfaces or things that don't make sense in this more limited 2D world. + +# Statistical plots + +The `statplot` package provides functions taking `tensor` data that produce statistical plots of the data, including Quartiles (Box with Median, Quartile, Min, Max), Histogram (Bar), Violin (YFill), Range (XFill), Cluster... + +TODO: add a Data scatter that plots points to overlay on top of Violin or Box. + +## Histogram + +## Quartiles + +## Violin + +## Range + +## Cluster + # History The code is adapted from the [gonum plot](https://github.com/gonum/plot) package (which in turn was adapted from google's [plotinum](https://code.google.com/archive/p/plotinum/), to use the Cogent Core [styles](../styles) and [paint](../paint) rendering framework, which also supports SVG output of the rendering. @@ -34,3 +129,7 @@ Here is the copyright notice for that package: // license that can be found in the LICENSE file. ``` +# TODO + +* Grid? + diff --git a/plot/axis.go b/plot/axis.go index dc3ce4e1fd..56207427f4 100644 --- a/plot/axis.go +++ b/plot/axis.go @@ -10,7 +10,10 @@ package plot import ( + "math" + "cogentcore.org/core/math32" + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/styles" "cogentcore.org/core/styles/units" ) @@ -55,14 +58,13 @@ func (ax *AxisStyle) Defaults() { // Axis represents either a horizontal or vertical // axis of a plot. type Axis struct { - // Min and Max are the minimum and maximum data - // values represented by the axis. - Min, Max float32 + // Range has the Min, Max range of values for the axis (in raw data units.) + Range minmax.F64 - // specifies which axis this is: X or Y + // specifies which axis this is: X, Y or Z. Axis math32.Dims - // Label for the axis + // Label for the axis. Label Text // Style has the style parameters for the Axis. @@ -93,8 +95,6 @@ type Axis struct { // value is less than Min and greater than Max. func (ax *Axis) Defaults(dim math32.Dims) { ax.Style.Defaults() - ax.Min = math32.Inf(+1) - ax.Max = math32.Inf(-1) ax.Axis = dim if dim == math32.Y { ax.Label.Style.Rotation = -90 @@ -106,25 +106,11 @@ func (ax *Axis) Defaults(dim math32.Dims) { // SanitizeRange ensures that the range of the axis makes sense. func (ax *Axis) SanitizeRange() { - if math32.IsInf(ax.Min, 0) { - ax.Min = 0 - } - if math32.IsInf(ax.Max, 0) { - ax.Max = 0 - } - if ax.Min > ax.Max { - ax.Min, ax.Max = ax.Max, ax.Min - } - if ax.Min == ax.Max { - ax.Min-- - ax.Max++ - } - + ax.Range.Sanitize() if ax.AutoRescale { - marks := ax.Ticker.Ticks(ax.Min, ax.Max) + marks := ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max) for _, t := range marks { - ax.Min = math32.Min(ax.Min, t.Value) - ax.Max = math32.Max(ax.Max, t.Value) + ax.Range.FitValInRange(t.Value) } } } @@ -134,7 +120,7 @@ func (ax *Axis) SanitizeRange() { type Normalizer interface { // Normalize transforms a value x in the data coordinate system to // the normalized coordinate system. - Normalize(min, max, x float32) float32 + Normalize(min, max, x float64) float64 } // LinearScale an be used as the value of an Axis.Scale function to @@ -144,7 +130,7 @@ type LinearScale struct{} var _ Normalizer = LinearScale{} // Normalize returns the fractional distance of x between min and max. -func (LinearScale) Normalize(min, max, x float32) float32 { +func (LinearScale) Normalize(min, max, x float64) float64 { return (x - min) / (max - min) } @@ -156,12 +142,12 @@ var _ Normalizer = LogScale{} // Normalize returns the fractional logarithmic distance of // x between min and max. -func (LogScale) Normalize(min, max, x float32) float32 { +func (LogScale) Normalize(min, max, x float64) float64 { if min <= 0 || max <= 0 || x <= 0 { panic("Values must be greater than 0 for a log scale.") } - logMin := math32.Log(min) - return (math32.Log(x) - logMin) / (math32.Log(max) - logMin) + logMin := math.Log(min) + return (math.Log(x) - logMin) / (math.Log(max) - logMin) } // InvertedScale can be used as the value of an Axis.Scale function to @@ -171,7 +157,7 @@ type InvertedScale struct{ Normalizer } var _ Normalizer = InvertedScale{} // Normalize returns a normalized [0, 1] value for the position of x. -func (is InvertedScale) Normalize(min, max, x float32) float32 { +func (is InvertedScale) Normalize(min, max, x float64) float64 { return is.Normalizer.Normalize(max, min, x) } @@ -179,6 +165,6 @@ func (is InvertedScale) Normalize(min, max, x float32) float32 { // system, normalized to its distance as a fraction of the // range of this axis. For example, if x is a.Min then the return // value is 0, and if x is a.Max then the return value is 1. -func (ax *Axis) Norm(x float32) float32 { - return ax.Scale.Normalize(ax.Min, ax.Max, x) +func (ax *Axis) Norm(x float64) float64 { + return ax.Scale.Normalize(ax.Range.Min, ax.Range.Max, x) } diff --git a/plot/data.go b/plot/data.go index fe9be936fa..720ac89bbc 100644 --- a/plot/data.go +++ b/plot/data.go @@ -10,30 +10,87 @@ package plot import ( - "errors" + "log/slog" + "math" + "strconv" - "cogentcore.org/core/math32" + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32/minmax" - "cogentcore.org/core/tensor" ) -// data defines the main data interfaces for plotting. -// Other more specific types of plots define their own interfaces. -// unlike gonum/plot, NaN values are treated as missing data here. +// data defines the main data interfaces for plotting +// and the different Roles for data. var ( ErrInfinity = errors.New("plotter: infinite data point") ErrNoData = errors.New("plotter: no data points") ) +// Data is the data interface for plotting, supporting either +// float64 or string representations. It is satisfied by the tensor.Tensor +// interface, so a tensor can be used directly for plot Data. +type Data interface { + // Len returns the number of values. + Len() int + + // Float1D(i int) returns float64 value at given index. + Float1D(i int) float64 + + // String1D(i int) returns string value at given index. + String1D(i int) string +} + +// Roles are the roles that a given set of data values can play, +// designed to be sufficiently generalizable across all different +// types of plots, even if sometimes it is a bit of a stretch. +type Roles int32 //enums:enum + +const ( + // NoRole is the default no-role specified case. + NoRole Roles = iota + + // X axis + X + + // Y axis + Y + + // Z axis + Z + + // U is the X component of a vector or first quartile in Box plot, etc. + U + + // V is the Y component of a vector or third quartile in a Box plot, etc. + V + + // W is the Z component of a vector + W + + // Low is a lower error bar or region. + Low + + // High is an upper error bar or region. + High + + // Size controls the size of points etc. + Size + + // Color controls the color of points or other elements. + Color + + // Label renders a label, typically from string data, but can also be used for values. + Label +) + // CheckFloats returns an error if any of the arguments are Infinity. // or if there are no non-NaN data points available for plotting. -func CheckFloats(fs ...float32) error { +func CheckFloats(fs ...float64) error { n := 0 for _, f := range fs { switch { - case math32.IsNaN(f): - case math32.IsInf(f, 0): + case math.IsNaN(f): + case math.IsInf(f, 0): return ErrInfinity default: n++ @@ -46,268 +103,129 @@ func CheckFloats(fs ...float32) error { } // CheckNaNs returns true if any of the floats are NaN -func CheckNaNs(fs ...float32) bool { +func CheckNaNs(fs ...float64) bool { for _, f := range fs { - if math32.IsNaN(f) { + if math.IsNaN(f) { return true } } return false } -////////////////////////////////////////////////// -// Valuer - -// Valuer provides an interface for a list of scalar values -type Valuer interface { - // Len returns the number of values. - Len() int - - // Value returns a value. - Value(i int) float32 -} - -// Range returns the minimum and maximum values. -func Range(vs Valuer) (mn, mx float32) { - mn = math32.Inf(1) - mx = math32.Inf(-1) - for i := 0; i < vs.Len(); i++ { - v := vs.Value(i) - if math32.IsNaN(v) { +// Range updates given Range with values from data. +func Range(data Data, rng *minmax.F64) { + for i := 0; i < data.Len(); i++ { + v := data.Float1D(i) + if math.IsNaN(v) { continue } - mn = math32.Min(mn, v) - mx = math32.Max(mx, v) + rng.FitValInRange(v) } - return } -// RangeClamp returns the minimum and maximum values clamped by given range. -func RangeClamp(vs Valuer, rng *minmax.Range32) (mn, mx float32) { - mn, mx = Range(vs) - mn, mx = rng.Clamp(mn, mx) - return +// RangeClamp updates the given axis Min, Max range values based +// on the range of values in the given [Data], and the given style range. +func RangeClamp(data Data, axisRng *minmax.F64, styleRng *minmax.Range64) { + Range(data, axisRng) + axisRng.Min, axisRng.Max = styleRng.Clamp(axisRng.Min, axisRng.Max) } -// Values implements the Valuer interface. -type Values []float32 +// Values provides a minimal implementation of the Data interface +// using a slice of float64. +type Values []float64 func (vs Values) Len() int { return len(vs) } -func (vs Values) Value(i int) float32 { +func (vs Values) Float1D(i int) float64 { return vs[i] } -// TensorValues provides a Valuer interface wrapper for a tensor. -type TensorValues struct { - tensor.Tensor -} - -func (vs TensorValues) Len() int { - return vs.Tensor.Len() -} - -func (vs TensorValues) Value(i int) float32 { - return float32(vs.Tensor.Float1D(i)) +func (vs Values) String1D(i int) string { + return strconv.FormatFloat(vs[i], 'g', -1, 64) } // CopyValues returns a Values that is a copy of the values -// from a Valuer, or an error if there are no values, or if one of +// from Data, or an error if there are no values, or if one of // the copied values is a Infinity. // NaN values are skipped in the copying process. -func CopyValues(vs Valuer) (Values, error) { - if vs.Len() == 0 { +func CopyValues(data Data) (Values, error) { + if data == nil { return nil, ErrNoData } - cpy := make(Values, 0, vs.Len()) - for i := 0; i < vs.Len(); i++ { - v := vs.Value(i) - if math32.IsNaN(v) { - continue - } - if err := CheckFloats(v); err != nil { - return nil, err - } - cpy = append(cpy, v) - } - return cpy, nil -} - -////////////////////////////////////////////////// -// XYer - -// XYer provides an interface for a list of X,Y data pairs -type XYer interface { - // Len returns the number of x, y pairs. - Len() int - - // XY returns an x, y pair. - XY(i int) (x, y float32) -} - -// XYRange returns the minimum and maximum x and y values. -func XYRange(xys XYer) (xmin, xmax, ymin, ymax float32) { - xmin, xmax = Range(XValues{xys}) - ymin, ymax = Range(YValues{xys}) - return -} - -// XYRangeClamp returns the data range with Range clamped for Y axis. -func XYRangeClamp(xys XYer, rng *minmax.Range32) (xmin, xmax, ymin, ymax float32) { - xmin, xmax, ymin, ymax = XYRange(xys) - ymin, ymax = rng.Clamp(ymin, ymax) - return -} - -// XYs implements the XYer interface. -type XYs []math32.Vector2 - -func (xys XYs) Len() int { - return len(xys) -} - -func (xys XYs) XY(i int) (float32, float32) { - return xys[i].X, xys[i].Y -} - -// TensorXYs provides a XYer interface wrapper for a tensor. -type TensorXYs struct { - X, Y tensor.Tensor -} - -func (xys TensorXYs) Len() int { - return xys.X.Len() -} - -func (xys TensorXYs) XY(i int) (float32, float32) { - return float32(xys.X.Float1D(i)), float32(xys.Y.Float1D(i)) -} - -// CopyXYs returns an XYs that is a copy of the x and y values from -// an XYer, or an error if one of the data points contains a NaN or -// Infinity. -func CopyXYs(data XYer) (XYs, error) { if data.Len() == 0 { return nil, ErrNoData } - cpy := make(XYs, 0, data.Len()) - for i := range data.Len() { - x, y := data.XY(i) - if CheckNaNs(x, y) { + cpy := make(Values, 0, data.Len()) + for i := 0; i < data.Len(); i++ { + v := data.Float1D(i) + if math.IsNaN(v) { continue } - if err := CheckFloats(x, y); err != nil { + if err := CheckFloats(v); err != nil { return nil, err } - cpy = append(cpy, math32.Vec2(x, y)) + cpy = append(cpy, v) } return cpy, nil } -// PlotXYs returns plot coordinates for given set of XYs -func PlotXYs(plt *Plot, data XYs) XYs { - ps := make(XYs, len(data)) - for i := range data { - ps[i].X, ps[i].Y = plt.PX(data[i].X), plt.PY(data[i].Y) +// MustCopyRole returns Values copy of given role from given data map, +// logging an error and returning nil if not present. +func MustCopyRole(data map[Roles]Data, role Roles) Values { + d, ok := data[role] + if !ok { + slog.Error("plot Data role not present, but is required", "role:", role) + return nil } - return ps -} - -// XValues implements the Valuer interface, -// returning the x value from an XYer. -type XValues struct { - XYer -} - -func (xs XValues) Value(i int) float32 { - x, _ := xs.XY(i) - return x -} - -// YValues implements the Valuer interface, -// returning the y value from an XYer. -type YValues struct { - XYer + v, _ := CopyValues(d) + return v } -func (ys YValues) Value(i int) float32 { - _, y := ys.XY(i) - return y +// CopyRole returns Values copy of given role from given data map, +// returning nil if role not present. +func CopyRole(data map[Roles]Data, role Roles) Values { + d, ok := data[role] + if !ok { + return nil + } + v, _ := CopyValues(d) + return v } -////////////////////////////////////////////////// -// XYer - -// XYZer provides an interface for a list of X,Y,Z data triples. -// It also satisfies the XYer interface for the X,Y pairs. -type XYZer interface { - // Len returns the number of x, y, z triples. - Len() int - - // XYZ returns an x, y, z triple. - XYZ(i int) (float32, float32, float32) - - // XY returns an x, y pair. - XY(i int) (float32, float32) +// PlotX returns plot pixel X coordinate values for given data. +func PlotX(plt *Plot, data Data) []float32 { + px := make([]float32, data.Len()) + for i := range px { + px[i] = plt.PX(data.Float1D(i)) + } + return px } -// XYZs implements the XYZer interface using a slice. -type XYZs []XYZ - -// XYZ is an x, y and z value. -type XYZ struct{ X, Y, Z float32 } - -// Len implements the Len method of the XYZer interface. -func (xyz XYZs) Len() int { - return len(xyz) +// PlotY returns plot pixel Y coordinate values for given data. +func PlotY(plt *Plot, data Data) []float32 { + py := make([]float32, data.Len()) + for i := range py { + py[i] = plt.PY(data.Float1D(i)) + } + return py } -// XYZ implements the XYZ method of the XYZer interface. -func (xyz XYZs) XYZ(i int) (float32, float32, float32) { - return xyz[i].X, xyz[i].Y, xyz[i].Z -} +//////// Labels -// XY implements the XY method of the XYer interface. -func (xyz XYZs) XY(i int) (float32, float32) { - return xyz[i].X, xyz[i].Y -} +// Labels provides a minimal implementation of the Data interface +// using a slice of string. It always returns 0 for Float1D. +type Labels []string -// CopyXYZs copies an XYZer. -func CopyXYZs(data XYZer) (XYZs, error) { - if data.Len() == 0 { - return nil, ErrNoData - } - cpy := make(XYZs, 0, data.Len()) - for i := range data.Len() { - x, y, z := data.XYZ(i) - if CheckNaNs(x, y, z) { - continue - } - if err := CheckFloats(x, y, z); err != nil { - return nil, err - } - cpy = append(cpy, XYZ{X: x, Y: y, Z: z}) - } - return cpy, nil +func (lb Labels) Len() int { + return len(lb) } -// XYValues implements the XYer interface, returning -// the x and y values from an XYZer. -type XYValues struct{ XYZer } - -// XY implements the XY method of the XYer interface. -func (xy XYValues) XY(i int) (float32, float32) { - x, y, _ := xy.XYZ(i) - return x, y +func (lb Labels) Float1D(i int) float64 { + return 0 } -////////////////////////////////////////////////// -// Labeler - -// Labeler provides an interface for a list of string labels -type Labeler interface { - // Label returns a label. - Label(i int) string +func (lb Labels) String1D(i int) string { + return lb[i] } diff --git a/plot/draw.go b/plot/draw.go index 72d7e6b70e..52dd9b836e 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -139,7 +139,7 @@ func (ax *Axis) sizeX(pt *Plot, axw float32) (ht, lpad, rpad int) { pc := pt.Paint uc := &pc.UnitContext ax.Style.TickLength.ToDots(uc) - ax.ticks = ax.Ticker.Ticks(ax.Min, ax.Max) + ax.ticks = ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max) h := float32(0) if ax.Label.Text != "" { // We assume that the label isn't rotated. ax.Label.Config(pt) @@ -231,7 +231,7 @@ func (ax *Axis) longestTickLabel() string { func (ax *Axis) sizeY(pt *Plot) (ywidth, tickWidth, tpad, bpad int) { pc := pt.Paint uc := &pc.UnitContext - ax.ticks = ax.Ticker.Ticks(ax.Min, ax.Max) + ax.ticks = ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max) ax.Style.TickLength.ToDots(uc) w := float32(0) diff --git a/plot/labelling.go b/plot/labelling.go index 29b2b1cc64..606cac3a2c 100644 --- a/plot/labelling.go +++ b/plot/labelling.go @@ -14,7 +14,7 @@ package plot -import "cogentcore.org/core/math32" +import "math" const ( // dlamchE is the machine epsilon. For IEEE this is 2^{-53}. @@ -49,7 +49,7 @@ const ( // By default, when nil, legbility will set the legibility score for each candidate // labelling scheme to 1. // See the paper for an explanation of the function of Q, w and legibility. -func talbotLinHanrahan(dMin, dMax float32, want int, containment int, Q []float32, w *weights, legibility func(lMin, lMax, lStep float32) float32) (values []float32, step, q float32, magnitude int) { +func talbotLinHanrahan(dMin, dMax float64, want int, containment int, Q []float64, w *weights, legibility func(lMin, lMax, lStep float64) float64) (values []float64, step, q float64, magnitude int) { const eps = dlamchP * 100 if dMin > dMax { @@ -57,7 +57,7 @@ func talbotLinHanrahan(dMin, dMax float32, want int, containment int, Q []float3 } if Q == nil { - Q = []float32{1, 5, 2, 2.5, 4, 3} + Q = []float64{1, 5, 2, 2.5, 4, 3} } if w == nil { w = &weights{ @@ -72,10 +72,10 @@ func talbotLinHanrahan(dMin, dMax float32, want int, containment int, Q []float3 } if r := dMax - dMin; r < eps { - l := make([]float32, want) - step := r / float32(want-1) + l := make([]float64, want) + step := r / float64(want-1) for i := range l { - l[i] = dMin + float32(i)*step + l[i] = dMin + float64(i)*step } magnitude = minAbsMag(dMin, dMax) return l, step, 0, magnitude @@ -87,9 +87,9 @@ func talbotLinHanrahan(dMin, dMax float32, want int, containment int, Q []float3 // lMin and lMax are the selected min // and max label values. lq is the q // chosen. - lMin, lMax, lStep, lq float32 + lMin, lMax, lStep, lq float64 // score is the score for the selection. - score float32 + score float64 // magnitude is the magnitude of the // label step distance. magnitude int @@ -110,22 +110,22 @@ outer: break } - delta := (dMax - dMin) / float32(have+1) / float32(skip) / q + delta := (dMax - dMin) / float64(have+1) / float64(skip) / q const maxExp = 309 - for mag := int(math32.Ceil(math32.Log10(delta))); mag < maxExp; mag++ { - step := float32(skip) * q * math32.Pow10(mag) + for mag := int(math.Ceil(math.Log10(delta))); mag < maxExp; mag++ { + step := float64(skip) * q * math.Pow10(mag) - cm := maxCoverage(dMin, dMax, step*float32(have-1)) + cm := maxCoverage(dMin, dMax, step*float64(have-1)) if w.score(sm, cm, dm, 1) < best.score { break } - fracStep := step / float32(skip) - kStep := step * float32(have-1) + fracStep := step / float64(skip) + kStep := step * float64(have-1) - minStart := (math32.Floor(dMax/step) - float32(have-1)) * float32(skip) - maxStart := math32.Ceil(dMax/step) * float32(skip) + minStart := (math.Floor(dMax/step) - float64(have-1)) * float64(skip) + maxStart := math.Ceil(dMax/step) * float64(skip) for start := minStart; start <= maxStart && start != start-1; start++ { lMin := start * fracStep lMax := lMin + kStep @@ -154,7 +154,7 @@ outer: n: have, lMin: lMin, lMax: lMax, - lStep: float32(skip) * q, + lStep: float64(skip) * q, lq: q, score: score, magnitude: mag, @@ -167,51 +167,51 @@ outer: } if best.score == -2 { - l := make([]float32, want) - step := (dMax - dMin) / float32(want-1) + l := make([]float64, want) + step := (dMax - dMin) / float64(want-1) for i := range l { - l[i] = dMin + float32(i)*step + l[i] = dMin + float64(i)*step } magnitude = minAbsMag(dMin, dMax) return l, step, 0, magnitude } - l := make([]float32, best.n) - step = best.lStep * math32.Pow10(best.magnitude) + l := make([]float64, best.n) + step = best.lStep * math.Pow10(best.magnitude) for i := range l { - l[i] = best.lMin + float32(i)*step + l[i] = best.lMin + float64(i)*step } return l, best.lStep, best.lq, best.magnitude } // minAbsMag returns the minumum magnitude of the absolute values of a and b. -func minAbsMag(a, b float32) int { - return int(math32.Min(math32.Floor(math32.Log10(math32.Abs(a))), (math32.Floor(math32.Log10(math32.Abs(b)))))) +func minAbsMag(a, b float64) int { + return int(math.Min(math.Floor(math.Log10(math.Abs(a))), (math.Floor(math.Log10(math.Abs(b)))))) } // simplicity returns the simplicity score for how will the curent q, lMin, lMax, // lStep and skip match the given nice numbers, Q. -func simplicity(q float32, Q []float32, skip int, lMin, lMax, lStep float32) float32 { +func simplicity(q float64, Q []float64, skip int, lMin, lMax, lStep float64) float64 { const eps = dlamchP * 100 for i, v := range Q { if v == q { - m := math32.Mod(lMin, lStep) + m := math.Mod(lMin, lStep) v = 0 if (m < eps || lStep-m < eps) && lMin <= 0 && 0 <= lMax { v = 1 } - return 1 - float32(i)/(float32(len(Q))-1) - float32(skip) + v + return 1 - float64(i)/(float64(len(Q))-1) - float64(skip) + v } } panic("labelling: invalid q for Q") } // maxSimplicity returns the maximum simplicity for q, Q and skip. -func maxSimplicity(q float32, Q []float32, skip int) float32 { +func maxSimplicity(q float64, Q []float64, skip int) float64 { for i, v := range Q { if v == q { - return 1 - float32(i)/(float32(len(Q))-1) - float32(skip) + 1 + return 1 - float64(i)/(float64(len(Q))-1) - float64(skip) + 1 } } panic("labelling: invalid q for Q") @@ -220,7 +220,7 @@ func maxSimplicity(q float32, Q []float32, skip int) float32 { // coverage returns the coverage score for based on the average // squared distance between the extreme labels, lMin and lMax, and // the extreme data points, dMin and dMax. -func coverage(dMin, dMax, lMin, lMax float32) float32 { +func coverage(dMin, dMax, lMin, lMax float64) float64 { r := 0.1 * (dMax - dMin) max := dMax - lMax min := dMin - lMin @@ -229,7 +229,7 @@ func coverage(dMin, dMax, lMin, lMax float32) float32 { // maxCoverage returns the maximum coverage achievable for the data // range. -func maxCoverage(dMin, dMax, span float32) float32 { +func maxCoverage(dMin, dMax, span float64) float64 { r := dMax - dMin if span <= r { return 1 @@ -242,9 +242,9 @@ func maxCoverage(dMin, dMax, span float32) float32 { // density returns the density score which measures the goodness of // the labelling density compared to the user defined target // based on the want parameter given to talbotLinHanrahan. -func density(have, want int, dMin, dMax, lMin, lMax float32) float32 { - rho := float32(have-1) / (lMax - lMin) - rhot := float32(want-1) / (math32.Max(lMax, dMax) - math32.Min(dMin, lMin)) +func density(have, want int, dMin, dMax, lMin, lMax float64) float64 { + rho := float64(have-1) / (lMax - lMin) + rhot := float64(want-1) / (math.Max(lMax, dMax) - math.Min(dMin, lMin)) if d := rho / rhot; d >= 1 { return 2 - d } @@ -252,26 +252,26 @@ func density(have, want int, dMin, dMax, lMin, lMax float32) float32 { } // maxDensity returns the maximum density score achievable for have and want. -func maxDensity(have, want int) float32 { +func maxDensity(have, want int) float64 { if have < want { return 1 } - return 2 - float32(have-1)/float32(want-1) + return 2 - float64(have-1)/float64(want-1) } // unitLegibility returns a default legibility score ignoring label // spacing. -func unitLegibility(_, _, _ float32) float32 { +func unitLegibility(_, _, _ float64) float64 { return 1 } // weights is a helper type to calcuate the labelling scheme's total score. type weights struct { - simplicity, coverage, density, legibility float32 + simplicity, coverage, density, legibility float64 } // score returns the score for a labelling scheme with simplicity, s, // coverage, c, density, d and legibility l. -func (w *weights) score(s, c, d, l float32) float32 { +func (w *weights) score(s, c, d, l float64) float64 { return w.simplicity*s + w.coverage*c + w.density*d + w.legibility*l } diff --git a/plot/plot.go b/plot/plot.go index f86cd97103..3e4eeb9581 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -94,7 +94,7 @@ type PlotStyle struct { //types:add -setters // BarWidth for Bar plot sets the default width of the bars, // which should be less than the Stride (1 typically) to prevent // bar overlap. Defaults to .8. - BarWidth float32 + BarWidth float64 } func (ps *PlotStyle) Defaults() { @@ -138,9 +138,9 @@ type Plot struct { // standard text style with default options StandardTextStyle styles.Text - // X and Y are the horizontal and vertical axes + // X, Y, and Z are the horizontal, vertical, and depth axes // of the plot respectively. - X, Y Axis + X, Y, Z Axis // Legend is the plot's legend. Legend Legend @@ -225,20 +225,14 @@ func (pt *Plot) applyStyle() { pt.UpdateRange() } -// Add adds a Plotters to the plot. -// -// If the plotters implements DataRanger then the -// minimum and maximum values of the X and Y -// axes are changed if necessary to fit the range of -// the data. -// +// Add adds Plotter element(s) to the plot. // When drawing the plot, Plotters are drawn in the // order in which they were added to the plot. func (pt *Plot) Add(ps ...Plotter) { pt.Plotters = append(pt.Plotters, ps...) } -// SetPixels sets the backing pixels image to given image.RGBA +// SetPixels sets the backing pixels image to given image.RGBA. func (pt *Plot) SetPixels(img *image.RGBA) { pt.Pixels = img pt.Paint = paint.NewContextFromImage(pt.Pixels) @@ -278,7 +272,7 @@ func (pt *Plot) NominalX(names ...string) { // pt.Y.Padding.Pt(pt.X.Style.Tick.Label.Width(names[0]) / 2) ticks := make([]Tick, len(names)) for i, name := range names { - ticks[i] = Tick{float32(i), name} + ticks[i] = Tick{float64(i), name} } pt.X.Ticker = ConstantTicks(ticks) } @@ -311,7 +305,7 @@ func (pt *Plot) NominalY(names ...string) { // pt.X.Padding = pt.Y.Tick.Label.Height(names[0]) / 2 ticks := make([]Tick, len(names)) for i, name := range names { - ticks[i] = Tick{float32(i), name} + ticks[i] = Tick{float64(i), name} } pt.Y.Ticker = ConstantTicks(ticks) } @@ -320,46 +314,34 @@ func (pt *Plot) NominalY(names ...string) { // This first resets the range so any fixed additional range values should // be set after this point. func (pt *Plot) UpdateRange() { - pt.X.Min = math32.Inf(+1) - pt.X.Max = math32.Inf(-1) - pt.Y.Min = math32.Inf(+1) - pt.Y.Max = math32.Inf(-1) - for _, d := range pt.Plotters { - pt.UpdateRangeFromPlotter(d) + pt.X.Range.SetInfinity() + pt.Y.Range.SetInfinity() + pt.Z.Range.SetInfinity() + for _, pl := range pt.Plotters { + pl.UpdateRange(pt, &pt.X.Range, &pt.Y.Range, &pt.Z.Range) } } -func (pt *Plot) UpdateRangeFromPlotter(d Plotter) { - if x, ok := d.(DataRanger); ok { - xmin, xmax, ymin, ymax := x.DataRange(pt) - pt.X.Min = math32.Min(pt.X.Min, xmin) - pt.X.Max = math32.Max(pt.X.Max, xmax) - pt.Y.Min = math32.Min(pt.Y.Min, ymin) - pt.Y.Max = math32.Max(pt.Y.Max, ymax) - } - pt.X.Min, pt.X.Max = pt.Style.XAxis.Range.Clamp(pt.X.Min, pt.X.Max) -} - // PX returns the X-axis plotting coordinate for given raw data value // using the current plot bounding region -func (pt *Plot) PX(v float32) float32 { - return pt.PlotBox.ProjectX(pt.X.Norm(v)) +func (pt *Plot) PX(v float64) float32 { + return pt.PlotBox.ProjectX(float32(pt.X.Norm(v))) } // PY returns the Y-axis plotting coordinate for given raw data value -func (pt *Plot) PY(v float32) float32 { - return pt.PlotBox.ProjectY(1 - pt.Y.Norm(v)) +func (pt *Plot) PY(v float64) float32 { + return pt.PlotBox.ProjectY(float32(1 - pt.Y.Norm(v))) } // ClosestDataToPixel returns the Plotter data point closest to given pixel point, // in the Pixels image. -func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float32, data, pixel math32.Vector2, legend string) { +func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float32, pixel math32.Vector2, data map[Roles]Data, legend string) { tp := math32.Vec2(float32(px), float32(py)) dist = float32(math32.MaxFloat32) for _, p := range pt.Plotters { - dts, pxls := p.XYData() - for i := range pxls.Len() { - ptx, pty := pxls.XY(i) + dts, pxX, pxY := p.Data() + for i, ptx := range pxX { + pty := pxY[i] pxy := math32.Vec2(ptx, pty) d := pxy.DistanceTo(tp) if d < dist { @@ -367,8 +349,7 @@ func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float pixel = pxy plt = p idx = i - dx, dy := dts.XY(i) - data = math32.Vec2(dx, dy) + data = dts legend = pt.Legend.LegendForPlotter(p) } } diff --git a/plot/plotcore/barplot.go b/plot/plotcore/barplot.go index ba20f1a0df..fe0461bf3d 100644 --- a/plot/plotcore/barplot.go +++ b/plot/plotcore/barplot.go @@ -169,7 +169,7 @@ func (pl *PlotEditor) genPlotBar() { xyl.Labels = make([]string, n) for i := range xview.Indexes { - y := firstXY.Value(i) + y := firstXY.Float1D(i) x := float32(mid + (i%maxx)*stride) xyl.XYs[i] = math32.Vec2(x, y) xyl.Labels[i] = xy.Label(i) diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index 060daeac37..403a2a74d7 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -213,7 +213,7 @@ func (txy *tableXY) XY(row int) (x, y float32) { if txy.table == nil { return 0, 0 } - x = txy.xValue(row) + x = txy.Value(row) y = txy.Value(row) return } diff --git a/plot/plots/barchart.go b/plot/plots/bar.go similarity index 53% rename from plot/plots/barchart.go rename to plot/plots/bar.go index d2f3c045ab..d8e09c8ae7 100644 --- a/plot/plots/barchart.go +++ b/plot/plots/bar.go @@ -12,33 +12,39 @@ package plots import ( - "cogentcore.org/core/base/errors" + "math" + "cogentcore.org/core/math32" + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" - "cogentcore.org/core/tensor" ) -// A BarChart presents ordinally-organized data with rectangular bars +// BarType is be used for specifying the type name. +const BarType = "Bar" + +func init() { + plot.RegisterPlotter(BarType, "A Bar presents ordinally-organized data with rectangular bars with lengths proportional to the data values, and an optional error bar at the top of the bar using the High data role.", []plot.Roles{plot.Y}, []plot.Roles{plot.High}, func(data map[plot.Roles]plot.Data) plot.Plotter { + return NewBar(data) + }) +} + +// A Bar presents ordinally-organized data with rectangular bars // with lengths proportional to the data values, and an optional -// error bar ("handle") at the top of the bar using given error value -// (single value, like a standard deviation etc, not drawn below the bar). +// error bar ("handle") at the top of the bar using the High data role. // // Bars are plotted centered at integer multiples of Stride plus Start offset. // Full data range also includes Pad value to extend range beyond edge bar centers. // Bar Width is in data units, e.g., should be <= Stride. // Defaults provide a unit-spaced plot. -type BarChart struct { - // Values are the plotted values - Values plot.Values - - // YErrors is a copy of the Y errors for each point. - Errors plot.Values +type Bar struct { + // copies of data + Y, Err plot.Values - // XYs is the actual pixel plotting coordinates for each value. - XYs plot.XYs + // actual plotting X, Y values in data coordinates, taking into account stacking etc. + X, Yp plot.Values - // PXYs is the actual pixel plotting coordinates for each value. - PXYs plot.XYs + // PX, PY are the actual pixel plotting coordinates for each XY value. + PX, PY []float32 // Style has the properties used to render the bars. Style plot.Style @@ -50,79 +56,64 @@ type BarChart struct { Horizontal bool // stackedOn is the bar chart upon which this bar chart is stacked. - StackedOn *BarChart + StackedOn *Bar stylers plot.Stylers } -// NewBarChart returns a new bar chart with a single bar for each value. +// NewBar returns a new bar chart with a single bar for each value. // The bars heights correspond to the values and their x locations correspond // to the index of their value in the Valuer. -// Optional error-bar values can be provided. -func NewBarChart(vs, ers plot.Valuer) *BarChart { - values, err := plot.CopyValues(vs) - if errors.Log(err) != nil { +// Optional error-bar values can be provided using the High data role. +func NewBar(data map[plot.Roles]plot.Data) *Bar { + bc := &Bar{} + bc.Y = plot.MustCopyRole(data, plot.Y) + if bc.Y == nil { return nil } - var errs plot.Values - if ers != nil { - errs, err = plot.CopyValues(ers) - if errors.Log(err) != nil { - return nil - } - } - bc := &BarChart{ - Values: values, - Errors: errs, - } + bc.Err = plot.CopyRole(data, plot.High) bc.Defaults() return bc } -// NewBarChartTensor returns a new bar chart with a single bar for each value. -// The bars heights correspond to the values and their x locations correspond -// to the index of their value in the Valuer. -// Optional error-bar values can be provided. -func NewBarChartTensor(vs, ers tensor.Tensor) *BarChart { - vt := plot.TensorValues{vs} - if ers == nil { - return NewBarChart(vt, nil) - } - return NewBarChart(vt, plot.TensorValues{ers}) -} - -func (bc *BarChart) Defaults() { +func (bc *Bar) Defaults() { bc.Style.Defaults() } -func (bc *BarChart) Styler(f func(s *plot.Style)) *BarChart { +func (bc *Bar) Styler(f func(s *plot.Style)) *Bar { bc.stylers.Add(f) return bc } -func (bc *BarChart) ApplyStyle(ps *plot.PlotStyle) { +func (bc *Bar) ApplyStyle(ps *plot.PlotStyle) { ps.SetElementStyle(&bc.Style) bc.stylers.Run(&bc.Style) } -func (bc *BarChart) Stylers() *plot.Stylers { return &bc.stylers } +func (bc *Bar) Stylers() *plot.Stylers { return &bc.stylers } -func (bc *BarChart) XYData() (data plot.XYer, pixels plot.XYer) { - data = bc.XYs - pixels = bc.PXYs +func (bc *Bar) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { + pixX = bc.PX + pixY = bc.PY + data = map[plot.Roles]plot.Data{} + data[plot.X] = bc.X + data[plot.Y] = bc.Y + if bc.Err != nil { + data[plot.High] = bc.Err + } return } // BarHeight returns the maximum y value of the // ith bar, taking into account any bars upon // which it is stacked. -func (bc *BarChart) BarHeight(i int) float32 { - ht := float32(0.0) +func (bc *Bar) BarHeight(i int) float64 { + ht := float64(0.0) if bc == nil { return 0 } - if i >= 0 && i < len(bc.Values) { - ht += bc.Values[i] + if i >= 0 && i < len(bc.Y) { + ht += bc.Y[i] } if bc.StackedOn != nil { ht += bc.StackedOn.BarHeight(i) @@ -133,27 +124,28 @@ func (bc *BarChart) BarHeight(i int) float32 { // StackOn stacks a bar chart on top of another, // and sets the bar positioning options to that of the // chart upon which it is being stacked. -func (bc *BarChart) StackOn(on *BarChart) { +func (bc *Bar) StackOn(on *Bar) { bc.Style.Width = on.Style.Width bc.StackedOn = on } // Plot implements the plot.Plotter interface. -func (bc *BarChart) Plot(plt *plot.Plot) { +func (bc *Bar) Plot(plt *plot.Plot) { pc := plt.Paint bc.Style.Line.SetStroke(plt) pc.FillStyle.Color = bc.Style.Line.Fill bw := bc.Style.Width - nv := len(bc.Values) - bc.XYs = make(plot.XYs, nv) - bc.PXYs = make(plot.XYs, nv) + nv := len(bc.Y) + bc.X = make(plot.Values, nv) + bc.Yp = make(plot.Values, nv) hw := 0.5 * bw.Width ew := bw.Width / 3 - for i, ht := range bc.Values { - cat := bw.Offset + float32(i)*bw.Stride - var bottom, catVal, catMin, catMax, valMin, valMax float32 + for i, ht := range bc.Y { + cat := bw.Offset + float64(i)*bw.Stride + var bottom float64 + var catVal, catMin, catMax, valMin, valMax float32 var box math32.Box2 if bc.Horizontal { catVal = plt.PY(cat) @@ -162,8 +154,10 @@ func (bc *BarChart) Plot(plt *plot.Plot) { bottom = bc.StackedOn.BarHeight(i) // nil safe valMin = plt.PX(bottom) valMax = plt.PX(bottom + ht) - bc.XYs[i] = math32.Vec2(bottom+ht, cat) - bc.PXYs[i] = math32.Vec2(valMax, catVal) + bc.X[i] = bottom + ht + bc.Yp[i] = cat + bc.PX[i] = valMax + bc.PY[i] = catVal box.Min.Set(valMin, catMin) box.Max.Set(valMax, catMax) } else { @@ -173,8 +167,10 @@ func (bc *BarChart) Plot(plt *plot.Plot) { bottom = bc.StackedOn.BarHeight(i) // nil safe valMin = plt.PY(bottom) valMax = plt.PY(bottom + ht) - bc.XYs[i] = math32.Vec2(cat, bottom+ht) - bc.PXYs[i] = math32.Vec2(catVal, valMax) + bc.X[i] = cat + bc.Yp[i] = bottom + ht + bc.PX[i] = catVal + bc.PY[i] = valMax box.Min.Set(catMin, valMin) box.Max.Set(catMax, valMax) } @@ -182,16 +178,16 @@ func (bc *BarChart) Plot(plt *plot.Plot) { pc.DrawRectangle(box.Min.X, box.Min.Y, box.Size().X, box.Size().Y) pc.FillStrokeClear() - if i < len(bc.Errors) { - errval := bc.Errors[i] + if i < len(bc.Err) { + errval := bc.Err[i] if bc.Horizontal { - eVal := plt.PX(bottom + ht + math32.Abs(errval)) + eVal := plt.PX(bottom + ht + math.Abs(errval)) pc.MoveTo(valMax, catVal) pc.LineTo(eVal, catVal) pc.MoveTo(eVal, plt.PY(cat-ew)) pc.LineTo(eVal, plt.PY(cat+ew)) } else { - eVal := plt.PY(bottom + ht + math32.Abs(errval)) + eVal := plt.PY(bottom + ht + math.Abs(errval)) pc.MoveTo(catVal, valMax) pc.LineTo(catVal, eVal) pc.MoveTo(plt.PX(cat-ew), eVal) @@ -203,33 +199,37 @@ func (bc *BarChart) Plot(plt *plot.Plot) { pc.FillStyle.Color = nil } -// DataRange implements the plot.DataRanger interface. -func (bc *BarChart) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { +// UpdateRange updates the given ranges. +func (bc *Bar) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { bw := bc.Style.Width catMin := bw.Offset - bw.Pad - catMax := bw.Offset + float32(len(bc.Values)-1)*bw.Stride + bw.Pad + catMax := bw.Offset + float64(len(bc.Y)-1)*bw.Stride + bw.Pad - valMin := math32.Inf(1) - valMax := math32.Inf(-1) - for i, val := range bc.Values { + for i, val := range bc.Y { valBot := bc.StackedOn.BarHeight(i) valTop := valBot + val - if i < len(bc.Errors) { - valTop += math32.Abs(bc.Errors[i]) + if i < len(bc.Err) { + valTop += math.Abs(bc.Err[i]) + } + if bc.Horizontal { + x.FitValInRange(valBot) + x.FitValInRange(valTop) + } else { + y.FitValInRange(valBot) + y.FitValInRange(valTop) } - valMin = math32.Min(valMin, math32.Min(valBot, valTop)) - valMax = math32.Max(valMax, math32.Max(valBot, valTop)) } - if !bc.Horizontal { - valMin, valMax = bc.Style.Range.Clamp(valMin, valMax) - return catMin, catMax, valMin, valMax + if bc.Horizontal { + x.Min, x.Max = bc.Style.Range.Clamp(x.Min, x.Max) + y.FitInRange(minmax.F64{catMin, catMax}) + } else { + y.Min, y.Max = bc.Style.Range.Clamp(y.Min, y.Max) + x.FitInRange(minmax.F64{catMin, catMax}) } - catMin, catMax = bc.Style.Range.Clamp(catMin, catMax) - return valMin, valMax, catMin, catMax } // Thumbnail fulfills the plot.Thumbnailer interface. -func (bc *BarChart) Thumbnail(plt *plot.Plot) { +func (bc *Bar) Thumbnail(plt *plot.Plot) { pc := plt.Paint bc.Style.Line.SetStroke(plt) pc.FillStyle.Color = bc.Style.Line.Fill diff --git a/plot/plots/doc.go b/plot/plots/doc.go index 072b3aef99..f4b79d3f06 100644 --- a/plot/plots/doc.go +++ b/plot/plots/doc.go @@ -18,6 +18,5 @@ // data points, and are just skipped over. // // New* functions return an error if the data contains Inf or is -// empty. Some of the New* functions return other plotter-specific errors -// too. +// empty. Some of the New* functions return other plotter-specific errors too. package plots diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 1ba1ab3edd..2db6586df2 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -5,58 +5,20 @@ package plots import ( - "cogentcore.org/core/base/errors" - "cogentcore.org/core/math32" + "math" + + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" ) -////////////////////////////////////////////////// -// XErrorer - -// XErrorer provides an interface for a list of Low, High error bar values. -// This is used in addition to an XYer interface, if implemented. -type XErrorer interface { - // XError returns Low, High error values for X data. - XError(i int) (low, high float32) -} - -// Errors is a slice of low and high error values. -type Errors []struct{ Low, High float32 } - -// XErrors implements the XErrorer interface. -type XErrors Errors - -func (xe XErrors) XError(i int) (low, high float32) { - return xe[i].Low, xe[i].High -} - -// YErrorer provides an interface for YError method. -// This is used in addition to an XYer interface, if implemented. -type YErrorer interface { - // YError returns two error values for Y data. - YError(i int) (float32, float32) -} - -// YErrors implements the YErrorer interface. -type YErrors Errors - -func (ye YErrors) YError(i int) (float32, float32) { - return ye[i].Low, ye[i].High -} - -// YErrorBars implements the plot.Plotter, plot.DataRanger, -// and plot.GlyphBoxer interfaces, drawing vertical error -// bars, denoting error in Y values. +// YErrorBars draws vertical error bars, denoting error in Y values, +// using Low, High data roles for error deviations around X, Y coordinates. type YErrorBars struct { - // XYs is a copy of the points for this line. - plot.XYs + // copies of data for this line + X, Y, Low, High plot.Values - // YErrors is a copy of the Y errors for each point. - YErrors - - // PXYs is the actual pixel plotting coordinates for each XY value, - // representing the high, center value of the error bar. - PXYs plot.XYs + // PX, PY are the actual pixel plotting coordinates for each XY value. + PX, PY []float32 // Style is the style for plotting. Style plot.Style @@ -68,32 +30,17 @@ func (eb *YErrorBars) Defaults() { eb.Style.Defaults() } -// NewYErrorBars returns a new YErrorBars plotter, or an error on failure. -// The error values from the YErrorer interface are interpreted as relative -// to the corresponding Y value. The errors for a given Y value are computed -// by taking the absolute value of the error returned by the YErrorer -// and subtracting the first and adding the second to the Y value. -func NewYErrorBars(yerrs interface { - plot.XYer - YErrorer -}) *YErrorBars { - - errs := make(YErrors, yerrs.Len()) - for i := range errs { - errs[i].Low, errs[i].High = yerrs.YError(i) - if err := plot.CheckFloats(errs[i].Low, errs[i].High); errors.Log(err) != nil { - return nil - } - } - xys, err := plot.CopyXYs(yerrs) - if errors.Log(err) != nil { +// NewYErrorBars returns a new YErrorBars plotter, +// using Low, High data roles for error deviations around X, Y coordinates. +func NewYErrorBars(data map[plot.Roles]plot.Data) *YErrorBars { + eb := &YErrorBars{} + eb.X = plot.MustCopyRole(data, plot.X) + eb.Y = plot.MustCopyRole(data, plot.Y) + eb.Low = plot.MustCopyRole(data, plot.Low) + eb.High = plot.MustCopyRole(data, plot.High) + if eb.X == nil || eb.Y == nil || eb.Low == nil || eb.High == nil { return nil } - - eb := &YErrorBars{ - XYs: xys, - YErrors: errs, - } eb.Defaults() return eb } @@ -111,29 +58,34 @@ func (eb *YErrorBars) ApplyStyle(ps *plot.PlotStyle) { func (eb *YErrorBars) Stylers() *plot.Stylers { return &eb.stylers } -func (eb *YErrorBars) XYData() (data plot.XYer, pixels plot.XYer) { - data = eb.XYs - pixels = eb.PXYs +func (eb *YErrorBars) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { + pixX = eb.PX + pixY = eb.PY + data = map[plot.Roles]plot.Data{} + data[plot.X] = eb.X + data[plot.Y] = eb.Y + data[plot.Low] = eb.Low + data[plot.High] = eb.High return } -// Plot implements the Plotter interface, drawing labels. func (eb *YErrorBars) Plot(plt *plot.Plot) { pc := plt.Paint uc := &pc.UnitContext eb.Style.Width.Cap.ToDots(uc) cw := 0.5 * eb.Style.Width.Cap.Dots - nv := len(eb.YErrors) - eb.PXYs = make(plot.XYs, nv) + nv := len(eb.X) + eb.PX = make([]float32, nv) + eb.PY = make([]float32, nv) eb.Style.Line.SetStroke(plt) - for i, err := range eb.YErrors { - x := plt.PX(eb.XYs[i].X) - ylow := plt.PY(eb.XYs[i].Y - math32.Abs(err.Low)) - yhigh := plt.PY(eb.XYs[i].Y + math32.Abs(err.High)) + for i, y := range eb.Y { + x := plt.PX(eb.X.Float1D(i)) + ylow := plt.PY(y - math.Abs(eb.Low[i])) + yhigh := plt.PY(y + math.Abs(eb.High[i])) - eb.PXYs[i].X = x - eb.PXYs[i].Y = yhigh + eb.PX[i] = x + eb.PY[i] = yhigh pc.MoveTo(x, ylow) pc.LineTo(x, yhigh) @@ -147,34 +99,27 @@ func (eb *YErrorBars) Plot(plt *plot.Plot) { } } -// DataRange implements the plot.DataRanger interface. -func (eb *YErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - xmin, xmax = plot.RangeClamp(plot.XValues{eb}, &eb.Style.Range) - ymin = math32.Inf(1) - ymax = math32.Inf(-1) - for i, err := range eb.YErrors { - y := eb.XYs[i].Y - ylow := y - math32.Abs(err.Low) - yhigh := y + math32.Abs(err.High) - ymin = math32.Min(math32.Min(math32.Min(ymin, y), ylow), yhigh) - ymax = math32.Max(math32.Max(math32.Max(ymax, y), ylow), yhigh) +// UpdateRange updates the given ranges. +func (eb *YErrorBars) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { + plot.Range(eb.X, x) + for i, yv := range eb.Y { + ylow := yv - math.Abs(eb.Low[i]) + yhigh := yv + math.Abs(eb.High[i]) + y.FitInRange(minmax.F64{ylow, yhigh}) } return } -// XErrorBars implements the plot.Plotter, plot.DataRanger, -// and plot.GlyphBoxer interfaces, drawing horizontal error -// bars, denoting error in Y values. -type XErrorBars struct { - // XYs is a copy of the points for this line. - plot.XYs +//////// XErrorBars - // XErrors is a copy of the X errors for each point. - XErrors +// XErrorBars draws horizontal error bars, denoting error in X values, +// using Low, High data roles for error deviations around X, Y coordinates. +type XErrorBars struct { + // copies of data for this line + X, Y, Low, High plot.Values - // PXYs is the actual pixel plotting coordinates for each XY value, - // representing the high, center value of the error bar. - PXYs plot.XYs + // PX, PY are the actual pixel plotting coordinates for each XY value. + PX, PY []float32 // Style is the style for plotting. Style plot.Style @@ -182,40 +127,25 @@ type XErrorBars struct { stylers plot.Stylers } -// Returns a new XErrorBars plotter, or an error on failure. The error values -// from the XErrorer interface are interpreted as relative to the corresponding -// X value. The errors for a given X value are computed by taking the absolute -// value of the error returned by the XErrorer and subtracting the first and -// adding the second to the X value. -func NewXErrorBars(xerrs interface { - plot.XYer - XErrorer -}) *XErrorBars { - - errs := make(XErrors, xerrs.Len()) - for i := range errs { - errs[i].Low, errs[i].High = xerrs.XError(i) - if err := plot.CheckFloats(errs[i].Low, errs[i].High); errors.Log(err) != nil { - return nil - } - } - xys, err := plot.CopyXYs(xerrs) - if errors.Log(err) != nil { - return nil - } +func (eb *XErrorBars) Defaults() { + eb.Style.Defaults() +} - eb := &XErrorBars{ - XYs: xys, - XErrors: errs, +// NewXErrorBars returns a new XErrorBars plotter, +// using Low, High data roles for error deviations around X, Y coordinates. +func NewXErrorBars(data map[plot.Roles]plot.Data) *XErrorBars { + eb := &XErrorBars{} + eb.X = plot.MustCopyRole(data, plot.X) + eb.Y = plot.MustCopyRole(data, plot.Y) + eb.Low = plot.MustCopyRole(data, plot.Low) + eb.High = plot.MustCopyRole(data, plot.High) + if eb.X == nil || eb.Y == nil || eb.Low == nil || eb.High == nil { + return nil } eb.Defaults() return eb } -func (eb *XErrorBars) Defaults() { - eb.Style.Defaults() -} - // Styler adds a style function to set style parameters. func (eb *XErrorBars) Styler(f func(s *plot.Style)) *XErrorBars { eb.stylers.Add(f) @@ -229,24 +159,34 @@ func (eb *XErrorBars) ApplyStyle(ps *plot.PlotStyle) { func (eb *XErrorBars) Stylers() *plot.Stylers { return &eb.stylers } -// Plot implements the Plotter interface, drawing labels. +func (eb *XErrorBars) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { + pixX = eb.PX + pixY = eb.PY + data = map[plot.Roles]plot.Data{} + data[plot.X] = eb.X + data[plot.Y] = eb.Y + data[plot.Low] = eb.Low + data[plot.High] = eb.High + return +} + func (eb *XErrorBars) Plot(plt *plot.Plot) { pc := plt.Paint uc := &pc.UnitContext eb.Style.Width.Cap.ToDots(uc) cw := 0.5 * eb.Style.Width.Cap.Dots - - nv := len(eb.XErrors) - eb.PXYs = make(plot.XYs, nv) + nv := len(eb.X) + eb.PX = make([]float32, nv) + eb.PY = make([]float32, nv) eb.Style.Line.SetStroke(plt) - for i, err := range eb.XErrors { - y := plt.PY(eb.XYs[i].Y) - xlow := plt.PX(eb.XYs[i].X - math32.Abs(err.Low)) - xhigh := plt.PX(eb.XYs[i].X + math32.Abs(err.High)) + for i, x := range eb.X { + y := plt.PY(eb.Y.Float1D(i)) + xlow := plt.PX(x - math.Abs(eb.Low[i])) + xhigh := plt.PX(x + math.Abs(eb.High[i])) - eb.PXYs[i].X = xhigh - eb.PXYs[i].Y = y + eb.PX[i] = xhigh + eb.PY[i] = y pc.MoveTo(xlow, y) pc.LineTo(xhigh, y) @@ -260,17 +200,13 @@ func (eb *XErrorBars) Plot(plt *plot.Plot) { } } -// DataRange implements the plot.DataRanger interface. -func (eb *XErrorBars) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - ymin, ymax = plot.Range(plot.YValues{eb}) - xmin = math32.Inf(1) - xmax = math32.Inf(-1) - for i, err := range eb.XErrors { - x := eb.XYs[i].X - xlow := x - math32.Abs(err.Low) - xhigh := x + math32.Abs(err.High) - xmin = math32.Min(math32.Min(math32.Min(xmin, x), xlow), xhigh) - xmax = math32.Max(math32.Max(math32.Max(xmax, x), xlow), xhigh) +// UpdateRange updates the given ranges. +func (eb *XErrorBars) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { + plot.Range(eb.Y, y) + for i, xv := range eb.X { + xlow := xv - math.Abs(eb.Low[i]) + xhigh := xv + math.Abs(eb.High[i]) + x.FitInRange(minmax.F64{xlow, xhigh}) } return } diff --git a/plot/plots/labels.go b/plot/plots/labels.go index ddf3a8704d..fa047b8ca6 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -7,22 +7,20 @@ package plots import ( "image" - "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" ) // Labels implements the Plotter interface, // drawing a set of labels at specified points. type Labels struct { - // XYs is a copy of the points for labels - plot.XYs + // copies of data for this line + X, Y plot.Values + Labels plot.Labels - // PXYs is the actual pixel plotting coordinates for each XY value. - PXYs plot.XYs - - // Labels is the set of labels corresponding to each point. - Labels []string + // PX, PY are the actual pixel plotting coordinates for each XY value. + PX, PY []float32 // Style is the style of the label text. Style plot.TextStyle @@ -33,23 +31,28 @@ type Labels struct { } // NewLabels returns a new Labels using defaults -func NewLabels(d XYLabeler) *Labels { - xys, err := plot.CopyXYs(d) - if errors.Log(err) != nil { +func NewLabels(data map[plot.Roles]plot.Data) *Labels { + lb := &Labels{} + lb.X = plot.MustCopyRole(data, plot.X) + lb.Y = plot.MustCopyRole(data, plot.Y) + if lb.X == nil || lb.Y == nil { return nil } - - if d.Len() != len(xys) { - errors.Log(errors.New("plotter: number of points does not match the number of labels")) + ld := data[plot.Label] + if ld == nil { return nil } - strs := make([]string, d.Len()) - for i := range strs { - strs[i] = d.Label(i) + // todo: in general need length checking on all data maps! + // if d.Len() != len(xys) { + // errors.Log(errors.New("plotter: number of points does not match the number of labels")) + // return nil + // } + lb.Labels = make(plot.Labels, lb.X.Len()) + for i := range ld.Len() { + lb.Labels[i] = ld.String1D(i) } - lb := &Labels{XYs: xys, Labels: strs} lb.Defaults() return lb } @@ -71,17 +74,22 @@ func (lb *Labels) ApplyStyle(ps *plot.PlotStyle) { func (lb *Labels) Stylers() *plot.Stylers { return &lb.stylers } -func (lb *Labels) XYData() (data plot.XYer, pixels plot.XYer) { - data = lb.XYs - pixels = lb.PXYs +func (lb *Labels) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { + pixX = lb.PX + pixY = lb.PY + data = map[plot.Roles]plot.Data{} + data[plot.X] = lb.X + data[plot.Y] = lb.Y + data[plot.Label] = lb.Labels return } // Plot implements the Plotter interface, drawing labels. func (lb *Labels) Plot(plt *plot.Plot) { - ps := plot.PlotXYs(plt, lb.XYs) pc := plt.Paint uc := &pc.UnitContext + lb.PX = plot.PlotX(plt, lb.X) + lb.PY = plot.PlotX(plt, lb.Y) lb.Style.Offset.ToDots(uc) lb.Style.ToDots(uc) var ltxt plot.Text @@ -93,16 +101,18 @@ func (lb *Labels) Plot(plt *plot.Plot) { ltxt.Text = label ltxt.Config(plt) tht := ltxt.PaintText.BBox.Size().Y - ltxt.Draw(plt, math32.Vec2(ps[i].X+lb.Style.Offset.X.Dots, ps[i].Y+lb.Style.Offset.Y.Dots-tht)) + ltxt.Draw(plt, math32.Vec2(lb.PX[i]+lb.Style.Offset.X.Dots, lb.PY[i]+lb.Style.Offset.Y.Dots-tht)) } } -// DataRange returns the minimum and maximum X and Y values -func (lb *Labels) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - xmin, xmax, ymin, ymax = plot.XYRange(lb) // first get basic numerical range +// UpdateRange updates the given ranges. +func (lb *Labels) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { + // todo: include point sizes! + plot.Range(lb.X, x) + plot.Range(lb.Y, y) pxToData := math32.FromPoint(plt.Size) - pxToData.X = (xmax - xmin) / pxToData.X - pxToData.Y = (ymax - ymin) / pxToData.Y + pxToData.X = float32(x.Range()) / pxToData.X + pxToData.Y = float32(y.Range()) / pxToData.Y var ltxt plot.Text ltxt.Style = lb.Style for i, label := range lb.Labels { @@ -113,35 +123,13 @@ func (lb *Labels) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { ltxt.Config(plt) tht := pxToData.Y * ltxt.PaintText.BBox.Size().Y twd := 1.1 * pxToData.X * ltxt.PaintText.BBox.Size().X - x, y := lb.XY(i) - minx := x - maxx := x + pxToData.X*lb.Style.Offset.X.Dots + twd - miny := y - maxy := y + pxToData.Y*lb.Style.Offset.Y.Dots + tht // y is up here - xmin = min(xmin, minx) - xmax = max(xmax, maxx) - ymin = min(ymin, miny) - ymax = max(ymax, maxy) + xv := lb.X[i] + yv := lb.Y[i] + minx := xv + maxx := xv + float64(pxToData.X*lb.Style.Offset.X.Dots+twd) + miny := yv + maxy := yv + float64(pxToData.Y*lb.Style.Offset.Y.Dots+tht) // y is up here + x.FitInRange(minmax.F64{minx, maxx}) + y.FitInRange(minmax.F64{miny, maxy}) } - return -} - -// XYLabeler combines the [plot.XYer] and [plot.Labeler] types. -type XYLabeler interface { - plot.XYer - plot.Labeler } - -// XYLabels holds XY data with labels. -// The ith label corresponds to the ith XY. -type XYLabels struct { - plot.XYs - Labels []string -} - -// Label returns the label for point index i. -func (lb XYLabels) Label(i int) string { - return lb.Labels[i] -} - -var _ XYLabeler = (*XYLabels)(nil) diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 103c7906ea..9dfbb10651 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -7,36 +7,36 @@ package plots import ( "fmt" "image" + "math" "os" "testing" "cogentcore.org/core/base/iox/imagex" "cogentcore.org/core/colors" - "cogentcore.org/core/math32" "cogentcore.org/core/paint" "cogentcore.org/core/plot" ) func ExampleLine() { - data := make(plot.XYs, 42) - for i := range data { - x := float32(i % 21) - data[i].X = x * 5 - if i < 21 { - data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) - } else { - data[i].Y = float32(50) + 40*math32.Cos((x/8)*math32.Pi) - } - } - - plt := plot.New() - plt.Add(NewLine(data).Styler(func(s *plot.Style) { - s.Line.Color = colors.Uniform(colors.Red) - s.Line.Width.Pt(2) - })) - plt.Draw() - imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") - // Output: + // data := make(plot.XYs, 42) + // for i := range data { + // x := float32(i % 21) + // data[i].X = x * 5 + // if i < 21 { + // data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) + // } else { + // data[i].Y = float32(50) + 40*math32.Cos((x/8)*math32.Pi) + // } + // } + // + // plt := plot.New() + // plt.Add(NewLine(data).Styler(func(s *plot.Style) { + // s.Line.Color = colors.Uniform(colors.Red) + // s.Line.Width.Pt(2) + // })) + // plt.Draw() + // imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") + // // Output: } func TestMain(m *testing.M) { @@ -45,45 +45,55 @@ func TestMain(m *testing.M) { } // sinCosWrapData returns overlapping sin / cos curves in one sequence. -func sinCosWrapData() plot.XYs { - data := make(plot.XYs, 42) - for i := range data { - x := float32(i % 21) - data[i].X = x * 5 +func sinCosWrapData() map[plot.Roles]plot.Data { + xd, yd := make(plot.Values, 42), make(plot.Values, 42) + for i := range xd { + x := float64(i % 21) + xd[i] = x * 5 if i < 21 { - data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) + yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) } else { - data[i].Y = float32(50) + 40*math32.Cos((x/8)*math32.Pi) + yd[i] = float64(50) + 40*math.Cos((x/8)*math.Pi) } } + data := map[plot.Roles]plot.Data{} + data[plot.X] = xd + data[plot.Y] = yd return data } -func sinDataXY() plot.XYs { - data := make(plot.XYs, 21) - for i := range data { - data[i].X = float32(i * 5) - data[i].Y = float32(50) + 40*math32.Sin((float32(i)/8)*math32.Pi) +func sinDataXY() map[plot.Roles]plot.Data { + xd, yd := make(plot.Values, 21), make(plot.Values, 21) + for i := range xd { + xd[i] = float64(i * 5) + xd[i] = float64(50) + 40*math.Sin((float64(i)/8)*math.Pi) } + data := map[plot.Roles]plot.Data{} + data[plot.X] = xd + data[plot.Y] = yd return data } -func sinData() plot.Values { - sin := make(plot.Values, 21) - for i := range sin { - x := float32(i % 21) - sin[i] = float32(50) + 40*math32.Sin((x/8)*math32.Pi) +func sinData() map[plot.Roles]plot.Data { + yd := make(plot.Values, 21) + for i := range yd { + x := float64(i % 21) + yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) } - return sin + data := map[plot.Roles]plot.Data{} + data[plot.Y] = yd + return data } -func cosData() plot.Values { - cos := make(plot.Values, 21) - for i := range cos { - x := float32(i % 21) - cos[i] = float32(50) + 40*math32.Cos((x/8)*math32.Pi) +func cosData() map[plot.Roles]plot.Data { + yd := make(plot.Values, 21) + for i := range yd { + x := float64(i % 21) + yd[i] = float64(50) + 40*math.Cos((x/8)*math.Pi) } - return cos + data := map[plot.Roles]plot.Data{} + data[plot.Y] = yd + return data } func TestLine(t *testing.T) { @@ -91,11 +101,11 @@ func TestLine(t *testing.T) { plt := plot.New() plt.Title.Text = "Test Line" - plt.X.Min = 0 - plt.X.Max = 100 + plt.X.Range.Min = 0 + plt.X.Range.Max = 100 plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 + plt.Y.Range.Min = 0 + plt.Y.Range.Max = 100 plt.Y.Label.Text = "Y Axis" l1 := NewLine(data) @@ -138,11 +148,11 @@ func TestScatter(t *testing.T) { plt := plot.New() plt.Title.Text = "Test Scatter" - plt.X.Min = 0 - plt.X.Max = 100 + plt.X.Range.Min = 0 + plt.X.Range.Max = 100 plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 + plt.Y.Range.Min = 0 + plt.Y.Range.Max = 100 plt.Y.Label.Text = "Y Axis" l1 := NewScatter(data) @@ -168,14 +178,18 @@ func TestLabels(t *testing.T) { plt.Y.Label.Text = "Y Axis" // note: making two overlapping series - data := make(plot.XYs, 12) - labels := make([]string, 12) - for i := range data { - x := float32(i % 21) - data[i].X = x * 5 - data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) - labels[i] = fmt.Sprintf("%7.4g", data[i].Y) + xd, yd := make(plot.Values, 12), make(plot.Values, 12) + labels := make(plot.Labels, 12) + for i := range xd { + x := float64(i % 21) + xd[i] = x * 5 + yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) + labels[i] = fmt.Sprintf("%7.4g", yd[i]) } + data := map[plot.Roles]plot.Data{} + data[plot.X] = xd + data[plot.Y] = yd + data[plot.Label] = labels l1 := NewLine(data) if l1 == nil { @@ -185,7 +199,7 @@ func TestLabels(t *testing.T) { plt.Add(l1) plt.Legend.Add("Sine", l1) - l2 := NewLabels(XYLabels{XYs: data, Labels: labels}) + l2 := NewLabels(data) if l2 == nil { t.Error("bad data") } @@ -198,18 +212,18 @@ func TestLabels(t *testing.T) { imagex.Assert(t, plt.Pixels, "labels.png") } -func TestBarChart(t *testing.T) { +func TestBar(t *testing.T) { plt := plot.New() plt.Title.Text = "Test Bar Chart" plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 + plt.Y.Range.Min = 0 + plt.Y.Range.Max = 100 plt.Y.Label.Text = "Y Axis" data := sinData() cos := cosData() - l1 := NewBarChart(data, nil) + l1 := NewBar(data) if l1 == nil { t.Error("bad data") } @@ -221,7 +235,7 @@ func TestBarChart(t *testing.T) { plt.Draw() imagex.Assert(t, plt.Pixels, "bar.png") - l2 := NewBarChart(cos, nil) + l2 := NewBar(cos) if l2 == nil { t.Error("bad data") } @@ -237,18 +251,19 @@ func TestBarChart(t *testing.T) { imagex.Assert(t, plt.Pixels, "bar-cos.png") } -func TestBarChartErr(t *testing.T) { +func TestBarErr(t *testing.T) { plt := plot.New() plt.Title.Text = "Test Bar Chart Errors" plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 + plt.Y.Range.Min = 0 + plt.Y.Range.Max = 100 plt.Y.Label.Text = "Y Axis" data := sinData() cos := cosData() + data[plot.High] = cos[plot.Y] - l1 := NewBarChart(data, cos) + l1 := NewBar(data) if l1 == nil { t.Error("bad data") } @@ -262,24 +277,24 @@ func TestBarChartErr(t *testing.T) { l1.Horizontal = true plt.UpdateRange() - plt.X.Min = 0 - plt.X.Max = 100 + plt.X.Range.Min = 0 + plt.X.Range.Max = 100 plt.Draw() imagex.Assert(t, plt.Pixels, "bar-err-horiz.png") } -func TestBarChartStack(t *testing.T) { +func TestBarStack(t *testing.T) { plt := plot.New() plt.Title.Text = "Test Bar Chart Stacked" plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 + plt.Y.Range.Min = 0 + plt.Y.Range.Max = 100 plt.Y.Label.Text = "Y Axis" data := sinData() cos := cosData() - l1 := NewBarChart(data, nil) + l1 := NewBar(data) if l1 == nil { t.Error("bad data") } @@ -287,7 +302,7 @@ func TestBarChartStack(t *testing.T) { plt.Add(l1) plt.Legend.Add("Sine", l1) - l2 := NewBarChart(cos, nil) + l2 := NewBar(cos) if l2 == nil { t.Error("bad data") } @@ -301,34 +316,33 @@ func TestBarChartStack(t *testing.T) { imagex.Assert(t, plt.Pixels, "bar-stacked.png") } -type XYErr struct { - plot.XYs - YErrors -} - func TestErrBar(t *testing.T) { plt := plot.New() plt.Title.Text = "Test Line Errors" plt.X.Label.Text = "X Axis" - plt.Y.Min = 0 - plt.Y.Max = 100 + plt.Y.Range.Min = 0 + plt.Y.Range.Max = 100 plt.Y.Label.Text = "Y Axis" - data := make(plot.XYs, 21) - for i := range data { - x := float32(i % 21) - data[i].X = x * 5 - data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) + xd, yd := make(plot.Values, 21), make(plot.Values, 21) + for i := range xd { + x := float64(i % 21) + xd[i] = x * 5 + yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) } - yerr := make(YErrors, 21) - for i := range yerr { - x := float32(i % 21) - yerr[i].High = float32(5) + 4*math32.Cos((x/8)*math32.Pi) - yerr[i].Low = -yerr[i].High + low, high := make(plot.Values, 21), make(plot.Values, 21) + for i := range low { + x := float64(i % 21) + high[i] = float64(5) + 4*math.Cos((x/8)*math.Pi) + low[i] = -high[i] } - xyerr := XYErr{XYs: data, YErrors: yerr} + data := map[plot.Roles]plot.Data{} + data[plot.X] = xd + data[plot.Y] = yd + data[plot.Low] = low + data[plot.High] = high l1 := NewLine(data) if l1 == nil { @@ -337,7 +351,7 @@ func TestErrBar(t *testing.T) { plt.Add(l1) plt.Legend.Add("Sine", l1) - l2 := NewYErrorBars(xyerr) + l2 := NewYErrorBars(data) if l2 == nil { t.Error("bad data") } diff --git a/plot/plots/table.go b/plot/plots/table.go index 70ab4fec38..6403edcbb0 100644 --- a/plot/plots/table.go +++ b/plot/plots/table.go @@ -4,8 +4,6 @@ package plots -import "cogentcore.org/core/plot" - // Table is an interface for tabular data for plotting, // with columns of values. type Table interface { @@ -31,34 +29,34 @@ func TableColumnByIndex(tab Table, name string) int { return -1 } -// TableXYer is an interface for providing XY access to Table data -type TableXYer struct { - Table Table - - // the indexes of the tensor columns to use for the X and Y data, respectively - XColumn, YColumn int -} - -func NewTableXYer(tab Table, xcolumn, ycolumn int) *TableXYer { - txy := &TableXYer{Table: tab, XColumn: xcolumn, YColumn: ycolumn} - return txy -} - -func (dt *TableXYer) Len() int { - return dt.Table.NumRows() -} - -func (dt *TableXYer) XY(i int) (x, y float32) { - return dt.Table.PlotData(dt.XColumn, i), dt.Table.PlotData(dt.YColumn, i) -} - -// AddTableLine adds XY Line with given x, y columns from given tabular data. -func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *XY { - txy := NewTableXYer(tab, xcolumn, ycolumn) - ln := NewLine(txy) - if ln == nil { - return nil - } - plt.Add(ln) - return ln -} +// // TableXYer is an interface for providing XY access to Table data +// type TableXYer struct { +// Table Table +// +// // the indexes of the tensor columns to use for the X and Y data, respectively +// XColumn, YColumn int +// } +// +// func NewTableXYer(tab Table, xcolumn, ycolumn int) *TableXYer { +// txy := &TableXYer{Table: tab, XColumn: xcolumn, YColumn: ycolumn} +// return txy +// } +// +// func (dt *TableXYer) Len() int { +// return dt.Table.NumRows() +// } +// +// func (dt *TableXYer) XY(i int) (x, y float32) { +// return dt.Table.PlotData(dt.XColumn, i), dt.Table.PlotData(dt.YColumn, i) +// } +// +// // AddTableLine adds XY Line with given x, y columns from given tabular data. +// func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *XY { +// txy := NewTableXYer(tab, xcolumn, ycolumn) +// ln := NewLine(txy) +// if ln == nil { +// return nil +// } +// plt.Add(ln) +// return ln +// } diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 82dcdc4836..b5c22e6c07 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -12,20 +12,27 @@ package plots //go:generate core generate import ( - "cogentcore.org/core/base/errors" "cogentcore.org/core/math32" + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" - "cogentcore.org/core/tensor" ) -// XY draws lines between and / or points for XY data values, -// based on Style properties. +// XYType is be used for specifying the type name. +const XYType = "XY" + +func init() { + plot.RegisterPlotter(XYType, "draws lines between and / or points for X,Y data values, using optional Size and Color data for the points, for a bubble plot.", []plot.Roles{plot.X, plot.Y}, []plot.Roles{plot.Size, plot.Color}, func(data map[plot.Roles]plot.Data) plot.Plotter { + return NewXY(data) + }) +} + +// XY draws lines between and / or points for XY data values. type XY struct { - // XYs is a copy of the points for this line. - plot.XYs + // copies of data for this line + X, Y, Color, Size plot.Values - // PXYs is the actual pixel plotting coordinates for each XY value. - PXYs plot.XYs + // PX, PY are the actual pixel plotting coordinates for each XY value. + PX, PY []float32 // Style is the style for plotting. Style plot.Style @@ -33,44 +40,42 @@ type XY struct { stylers plot.Stylers } -// NewLine returns an XY plot drawing Lines by default. -func NewLine(xys plot.XYer) *XY { - data, err := plot.CopyXYs(xys) - if errors.Log(err) != nil { +// NewXY returns an XY plot. +func NewXY(data map[plot.Roles]plot.Data) *XY { + ln := &XY{} + ln.X = plot.MustCopyRole(data, plot.X) + ln.Y = plot.MustCopyRole(data, plot.Y) + if ln.X == nil || ln.Y == nil { return nil } - ln := &XY{XYs: data} + ln.Color = plot.CopyRole(data, plot.Color) + ln.Size = plot.CopyRole(data, plot.Size) ln.Defaults() - ln.Style.Line.On = plot.On - ln.Style.Point.On = plot.Off return ln } -// NewLineTensor returns an XY plot drawing Lines, -// using two tensors for X, Y values. -func NewLineTensor(x, y tensor.Tensor) *XY { - return NewLine(plot.TensorXYs{X: x, Y: y}) +// NewLine returns an XY plot drawing Lines by default. +func NewLine(data map[plot.Roles]plot.Data) *XY { + ln := NewXY(data) + if ln == nil { + return ln + } + ln.Style.Line.On = plot.On + ln.Style.Point.On = plot.Off + return ln } // NewScatter returns an XY scatter plot drawing Points by default. -func NewScatter(xys plot.XYer) *XY { - data, err := plot.CopyXYs(xys) - if errors.Log(err) != nil { - return nil +func NewScatter(data map[plot.Roles]plot.Data) *XY { + ln := NewXY(data) + if ln == nil { + return ln } - ln := &XY{XYs: data} - ln.Defaults() ln.Style.Line.On = plot.Off ln.Style.Point.On = plot.On return ln } -// NewScatterTensor returns an XY scatter plot drawing Points by default, -// using two tensors for X, Y values. -func NewScatterTensor(x, y tensor.Tensor) *XY { - return NewScatter(plot.TensorXYs{X: x, Y: y}) -} - func (ln *XY) Defaults() { ln.Style.Defaults() } @@ -88,117 +93,127 @@ func (ln *XY) ApplyStyle(ps *plot.PlotStyle) { ln.stylers.Run(&ln.Style) } -func (ln *XY) XYData() (data plot.XYer, pixels plot.XYer) { - data = ln.XYs - pixels = ln.PXYs +func (ln *XY) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { + pixX = ln.PX + pixY = ln.PY + data = map[plot.Roles]plot.Data{} + data[plot.X] = ln.X + data[plot.Y] = ln.Y + if ln.Size != nil { + data[plot.Size] = ln.Size + } + if ln.Color != nil { + data[plot.Color] = ln.Color + } return } // Plot does the drawing, implementing the plot.Plotter interface. func (ln *XY) Plot(plt *plot.Plot) { pc := plt.Paint - - ps := plot.PlotXYs(plt, ln.XYs) - np := len(ps) - ln.PXYs = ps + ln.PX = plot.PlotX(plt, ln.X) + ln.PY = plot.PlotX(plt, ln.Y) + np := len(ln.PX) if ln.Style.Line.Fill != nil { pc.FillStyle.Color = ln.Style.Line.Fill - minY := plt.PY(plt.Y.Min) - prev := math32.Vec2(ps[0].X, minY) - pc.MoveTo(prev.X, prev.Y) - for i := range ps { - pt := ps[i] + minY := plt.PY(plt.Y.Range.Min) + prevX := ln.PX[0] + prevY := minY + pc.MoveTo(prevX, prevY) + for i, ptx := range ln.PX { + pty := ln.PY[i] switch ln.Style.Line.Step { case plot.NoStep: - if pt.X < prev.X { - pc.LineTo(prev.X, minY) + if ptx < prevX { + pc.LineTo(prevX, minY) pc.ClosePath() - pc.MoveTo(pt.X, minY) + pc.MoveTo(ptx, minY) } - pc.LineTo(pt.X, pt.Y) + pc.LineTo(ptx, pty) case plot.PreStep: if i == 0 { continue } - if pt.X < prev.X { - pc.LineTo(prev.X, minY) + if ptx < prevX { + pc.LineTo(prevX, minY) pc.ClosePath() - pc.MoveTo(pt.X, minY) + pc.MoveTo(ptx, minY) } else { - pc.LineTo(prev.X, pt.Y) + pc.LineTo(prevX, pty) } - pc.LineTo(pt.X, pt.Y) + pc.LineTo(ptx, pty) case plot.MidStep: - if pt.X < prev.X { - pc.LineTo(prev.X, minY) + if ptx < prevX { + pc.LineTo(prevX, minY) pc.ClosePath() - pc.MoveTo(pt.X, minY) + pc.MoveTo(ptx, minY) } else { - pc.LineTo(0.5*(prev.X+pt.X), prev.Y) - pc.LineTo(0.5*(prev.X+pt.X), pt.Y) + pc.LineTo(0.5*(prevX+ptx), prevY) + pc.LineTo(0.5*(prevX+ptx), pty) } - pc.LineTo(pt.X, pt.Y) + pc.LineTo(ptx, pty) case plot.PostStep: - if pt.X < prev.X { - pc.LineTo(prev.X, minY) + if ptx < prevX { + pc.LineTo(prevX, minY) pc.ClosePath() - pc.MoveTo(pt.X, minY) + pc.MoveTo(ptx, minY) } else { - pc.LineTo(pt.X, prev.Y) + pc.LineTo(ptx, prevY) } - pc.LineTo(pt.X, pt.Y) + pc.LineTo(ptx, pty) } - prev = pt + prevX, prevY = ptx, pty } - pc.LineTo(prev.X, minY) + pc.LineTo(prevX, minY) pc.ClosePath() pc.Fill() } pc.FillStyle.Color = nil if ln.Style.Line.SetStroke(plt) { - prev := ps[0] - pc.MoveTo(prev.X, prev.Y) + prevX, prevY := ln.PX[0], ln.PY[0] + pc.MoveTo(prevX, prevY) for i := 1; i < np; i++ { - pt := ps[i] + ptx, pty := ln.PX[i], ln.PY[i] if ln.Style.Line.Step != plot.NoStep { - if pt.X >= prev.X { + if ptx >= prevX { switch ln.Style.Line.Step { case plot.PreStep: - pc.LineTo(prev.X, pt.Y) + pc.LineTo(prevX, pty) case plot.MidStep: - pc.LineTo(0.5*(prev.X+pt.X), prev.Y) - pc.LineTo(0.5*(prev.X+pt.X), pt.Y) + pc.LineTo(0.5*(prevX+ptx), prevY) + pc.LineTo(0.5*(prevX+ptx), pty) case plot.PostStep: - pc.LineTo(pt.X, prev.Y) + pc.LineTo(ptx, prevY) } } else { - pc.MoveTo(pt.X, pt.Y) + pc.MoveTo(ptx, pty) } } - if !ln.Style.Line.NegativeX && pt.X < prev.X { - pc.MoveTo(pt.X, pt.Y) + if !ln.Style.Line.NegativeX && ptx < prevX { + pc.MoveTo(ptx, pty) } else { - pc.LineTo(pt.X, pt.Y) + pc.LineTo(ptx, pty) } - prev = pt + prevX, prevY = ptx, pty } pc.Stroke() } if ln.Style.Point.SetStroke(plt) { - for i := range ps { - pt := ps[i] - ln.Style.Point.DrawShape(pc, math32.Vec2(pt.X, pt.Y)) + for i, ptx := range ln.PX { + pty := ln.PY[i] + ln.Style.Point.DrawShape(pc, math32.Vec2(ptx, pty)) } } pc.FillStyle.Color = nil } -// DataRange returns the minimum and maximum -// x and y values, implementing the plot.DataRanger interface. -func (ln *XY) DataRange(plt *plot.Plot) (xmin, xmax, ymin, ymax float32) { - return plot.XYRangeClamp(ln, &ln.Style.Range) +// UpdateRange updates the given ranges. +func (ln *XY) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { + // todo: include point sizes! + plot.Range(ln.X, x) + plot.RangeClamp(ln.Y, y, &ln.Style.Range) } // Thumbnail returns the thumbnail, implementing the plot.Thumbnailer interface. diff --git a/plot/plotter.go b/plot/plotter.go index 9d5f91812e..4e15fc9001 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -4,16 +4,25 @@ package plot +import ( + "log/slog" + + "cogentcore.org/core/math32/minmax" +) + // Plotter is an interface that wraps the Plot method. // Standard implementations of Plotter are in the [plots] package. type Plotter interface { - // Plot draws the data to the Plot Paint + // Plot draws the data to the Plot Paint. Plot(pt *Plot) - // returns the data for this plot as X,Y points, - // including corresponding pixel data. - // This allows gui interface to inspect data etc. - XYData() (data XYer, pixels XYer) + // UpdateRange updates the given ranges. + UpdateRange(plt *Plot, x, y, z *minmax.F64) + + // Data returns the data by roles for this plot, for both the original + // data and the pixel-transformed X,Y coordinates for that data. + // This allows a GUI interface to inspect data etc. + Data() (data map[Roles]Data, pixX, pixY []float32) // Stylers returns the styler functions for this element. Stylers() *Stylers @@ -24,8 +33,43 @@ type Plotter interface { ApplyStyle(plotStyle *PlotStyle) } -// DataRanger wraps the DataRange method. -type DataRanger interface { - // DataRange returns the range of X and Y values. - DataRange(pt *Plot) (xmin, xmax, ymin, ymax float32) +// PlotterType registers a Plotter so that it can be created with appropriate data. +type PlotterType struct { + // Name of the plot type. + Name string + + // Doc is the documentation for this Plotter. + Doc string + + // Required Data roles for this plot. Data for these Roles must be provided. + Required []Roles + + // Optional Data roles for this plot. + Optional []Roles + + // New returns a new plotter of this type with given data in given roles. + New func(data map[Roles]Data) Plotter +} + +// Plotters is the registry of [Plotter] types. +var Plotters map[string]PlotterType + +// RegisterPlotter registers a plotter type. +func RegisterPlotter(name, doc string, required, optional []Roles, newFun func(data map[Roles]Data) Plotter) { + if Plotters == nil { + Plotters = make(map[string]PlotterType) + } + Plotters[name] = PlotterType{Name: name, Doc: doc, Required: required, Optional: optional, New: newFun} +} + +// NewPlotter returns a new plotter of given type, e.g., "XY", "Bar" etc, +// for given data roles (which must include Required roles, and may include Optional ones). +// Logs an error and returns nil if type name is not recognized as a registered type. +func NewPlotter(typeName string, data map[Roles]Data) Plotter { + pt, ok := Plotters[typeName] + if !ok { + slog.Error("plot.NewPlotter type name is not registered", "typeName", typeName) + return nil + } + return pt.New(data) } diff --git a/plot/point.go b/plot/point.go index e78c80e2cb..739aa439f2 100644 --- a/plot/point.go +++ b/plot/point.go @@ -160,6 +160,7 @@ func DrawCross(pc *paint.Context, pos math32.Vector2, size float32) { pc.Stroke() } +// Shapes has the options for how to draw points in the plot. type Shapes int32 //enums:enum const ( diff --git a/plot/style.go b/plot/style.go index 2560d8456e..f09ac265a6 100644 --- a/plot/style.go +++ b/plot/style.go @@ -21,8 +21,16 @@ type Style struct { //types:add -setters // On specifies whether to plot this item, for cases where it can be turned off. On DefaultOffOn + // Role specifies a role for this item, used for table-based plots to indicate + // how a particular column of data should be used. + Role Roles + + // Group specifies a group of related data items, used for table-based plots + // where different columns of data within the same Group play different Roles + Group string + // Range is the effective range of data to plot, where either end can be fixed. - Range minmax.Range32 `display:"inline"` + Range minmax.Range64 `display:"inline"` // Label provides an alternative label to use for axis, if set. Label string @@ -67,21 +75,21 @@ type WidthStyle struct { //types:add -setters // Offset for Bar plot is the offset added to each X axis value // relative to the Stride computed value (X = offset + index * Stride) // Defaults to 1. - Offset float32 + Offset float64 // Stride for Bar plot is distance between bars. Defaults to 1. - Stride float32 + Stride float64 // Width for Bar plot is the width of the bars, which should be less than // the Stride (1 typically) to prevent bar overlap. Defaults to .8. - Width float32 `min:"0.01" max:"1" default:"0.8"` + Width float64 `min:"0.01" max:"1" default:"0.8"` // Pad for Bar plot is additional space at start / end of data range, // to keep bars from overflowing ends. This amount is subtracted from Offset // and added to (len(Values)-1)*Stride -- no other accommodation for bar // width is provided, so that should be built into this value as well. // Defaults to 1. - Pad float32 + Pad float64 } func (ws *WidthStyle) Defaults() { diff --git a/plot/tick.go b/plot/tick.go index 8045c2ac49..450d4ade50 100644 --- a/plot/tick.go +++ b/plot/tick.go @@ -5,16 +5,15 @@ package plot import ( + "math" "strconv" "time" - - "cogentcore.org/core/math32" ) // A Tick is a single tick mark on an axis. type Tick struct { // Value is the data value marked by this Tick. - Value float32 + Value float64 // Label is the text to display at the tick mark. // If Label is an empty string then this is a minor tick mark. @@ -29,7 +28,7 @@ func (tk *Tick) IsMinor() bool { // Ticker creates Ticks in a specified range type Ticker interface { // Ticks returns Ticks in a specified range - Ticks(min, max float32) []Tick + Ticks(min, max float64) []Tick } // DefaultTicks is suitable for the Ticker field of an Axis, @@ -39,7 +38,7 @@ type DefaultTicks struct{} var _ Ticker = DefaultTicks{} // Ticks returns Ticks in the specified range. -func (DefaultTicks) Ticks(min, max float32) []Tick { +func (DefaultTicks) Ticks(min, max float64) []Tick { if max <= min { panic("illegal range") } @@ -47,7 +46,7 @@ func (DefaultTicks) Ticks(min, max float32) []Tick { const suggestedTicks = 3 labels, step, q, mag := talbotLinHanrahan(min, max, suggestedTicks, withinData, nil, nil, nil) - majorDelta := step * math32.Pow10(mag) + majorDelta := step * math.Pow10(mag) if q == 0 { // Simple fall back was chosen, so // majorDelta is the label distance. @@ -62,16 +61,16 @@ func (DefaultTicks) Ticks(min, max float32) []Tick { off = 1 fc = 'g' } - if math32.Trunc(q) != q { + if math.Trunc(q) != q { off += 2 } prec := minInt(6, maxInt(off, -mag)) ticks := make([]Tick, len(labels)) for i, v := range labels { - ticks[i] = Tick{Value: v, Label: strconv.FormatFloat(float64(v), fc, prec, 32)} + ticks[i] = Tick{Value: v, Label: strconv.FormatFloat(float64(v), fc, prec, 64)} } - var minorDelta float32 + var minorDelta float64 // See talbotLinHanrahan for the values used here. switch step { case 1, 2.5: @@ -87,7 +86,7 @@ func (DefaultTicks) Ticks(min, max float32) []Tick { // Find the first minor tick not greater // than the lowest data value. - var i float32 + var i float64 for labels[0]+(i-1)*minorDelta > min { i-- } @@ -101,7 +100,7 @@ func (DefaultTicks) Ticks(min, max float32) []Tick { } found := false for _, t := range ticks { - if math32.Abs(t.Value-val) < minorDelta/2 { + if math.Abs(t.Value-val) < minorDelta/2 { found = true } } @@ -139,20 +138,20 @@ type LogTicks struct { var _ Ticker = LogTicks{} // Ticks returns Ticks in a specified range -func (t LogTicks) Ticks(min, max float32) []Tick { +func (t LogTicks) Ticks(min, max float64) []Tick { if min <= 0 || max <= 0 { panic("Values must be greater than 0 for a log scale.") } - val := math32.Pow10(int(math32.Log10(min))) - max = math32.Pow10(int(math32.Ceil(math32.Log10(max)))) + val := math.Pow10(int(math.Log10(min))) + max = math.Pow10(int(math.Ceil(math.Log10(max)))) var ticks []Tick for val < max { for i := 1; i < 10; i++ { if i == 1 { ticks = append(ticks, Tick{Value: val, Label: formatFloatTick(val, t.Prec)}) } - ticks = append(ticks, Tick{Value: val * float32(i)}) + ticks = append(ticks, Tick{Value: val * float64(i)}) } val *= 10 } @@ -168,13 +167,13 @@ type ConstantTicks []Tick var _ Ticker = ConstantTicks{} // Ticks returns Ticks in a specified range -func (ts ConstantTicks) Ticks(float32, float32) []Tick { +func (ts ConstantTicks) Ticks(float64, float64) []Tick { return ts } // UnixTimeIn returns a time conversion function for the given location. -func UnixTimeIn(loc *time.Location) func(t float32) time.Time { - return func(t float32) time.Time { +func UnixTimeIn(loc *time.Location) func(t float64) time.Time { + return func(t float64) time.Time { return time.Unix(int64(t), 0).In(loc) } } @@ -194,13 +193,13 @@ type TimeTicks struct { // Time takes a float32 value and converts it into a time.Time. // If nil, UTCUnixTime is used. - Time func(t float32) time.Time + Time func(t float64) time.Time } var _ Ticker = TimeTicks{} // Ticks implements plot.Ticker. -func (t TimeTicks) Ticks(min, max float32) []Tick { +func (t TimeTicks) Ticks(min, max float64) []Tick { if t.Ticker == nil { t.Ticker = DefaultTicks{} } @@ -269,17 +268,17 @@ func tickLabelWidth(sty text.Style, ticks []Tick) vg.Length { // formatFloatTick returns a g-formated string representation of v // to the specified precision. -func formatFloatTick(v float32, prec int) string { - return strconv.FormatFloat(float64(v), 'g', prec, 32) +func formatFloatTick(v float64, prec int) string { + return strconv.FormatFloat(float64(v), 'g', prec, 64) } // TickerFunc is suitable for the Ticker field of an Axis. // It is an adapter which allows to quickly setup a Ticker using a function with an appropriate signature. -type TickerFunc func(min, max float32) []Tick +type TickerFunc func(min, max float64) []Tick var _ Ticker = TickerFunc(nil) // Ticks implements plot.Ticker. -func (f TickerFunc) Ticks(min, max float32) []Tick { +func (f TickerFunc) Ticks(min, max float64) []Tick { return f(min, max) } diff --git a/plot/typegen.go b/plot/typegen.go index 0e086d62ea..f462ffb0f0 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -40,9 +40,9 @@ func (t *AxisStyle) SetTickLine(v LineStyle) *AxisStyle { t.TickLine = v; return // TickLength is the length of tick lines. func (t *AxisStyle) SetTickLength(v units.Value) *AxisStyle { t.TickLength = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Axis", IDName: "axis", Doc: "Axis represents either a horizontal or vertical\naxis of a plot.", Fields: []types.Field{{Name: "Min", Doc: "Min and Max are the minimum and maximum data\nvalues represented by the axis."}, {Name: "Max", Doc: "Min and Max are the minimum and maximum data\nvalues represented by the axis."}, {Name: "Axis", Doc: "specifies which axis this is: X or Y"}, {Name: "Label", Doc: "Label for the axis"}, {Name: "Style", Doc: "Style has the style parameters for the Axis."}, {Name: "TickText", Doc: "TickText is used for rendering the tick text labels."}, {Name: "Ticker", Doc: "Ticker generates the tick marks. Any tick marks\nreturned by the Marker function that are not in\nrange of the axis are not drawn."}, {Name: "Scale", Doc: "Scale transforms a value given in the data coordinate system\nto the normalized coordinate system of the axis—its distance\nalong the axis as a fraction of the axis range."}, {Name: "AutoRescale", Doc: "AutoRescale enables an axis to automatically adapt its minimum\nand maximum boundaries, according to its underlying Ticker."}, {Name: "ticks", Doc: "cached list of ticks, set in size"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Axis", IDName: "axis", Doc: "Axis represents either a horizontal or vertical\naxis of a plot.", Fields: []types.Field{{Name: "Range", Doc: "Range has the Min, Max range of values for the axis (in raw data units.)"}, {Name: "Axis", Doc: "specifies which axis this is: X, Y or Z."}, {Name: "Label", Doc: "Label for the axis."}, {Name: "Style", Doc: "Style has the style parameters for the Axis."}, {Name: "TickText", Doc: "TickText is used for rendering the tick text labels."}, {Name: "Ticker", Doc: "Ticker generates the tick marks. Any tick marks\nreturned by the Marker function that are not in\nrange of the axis are not drawn."}, {Name: "Scale", Doc: "Scale transforms a value given in the data coordinate system\nto the normalized coordinate system of the axis—its distance\nalong the axis as a fraction of the axis range."}, {Name: "AutoRescale", Doc: "AutoRescale enables an axis to automatically adapt its minimum\nand maximum boundaries, according to its underlying Ticker."}, {Name: "ticks", Doc: "cached list of ticks, set in size"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Normalizer", IDName: "normalizer", Doc: "Normalizer rescales values from the data coordinate system to the\nnormalized coordinate system.", Methods: []types.Method{{Name: "Normalize", Doc: "Normalize transforms a value x in the data coordinate system to\nthe normalized coordinate system.", Args: []string{"min", "max", "x"}, Returns: []string{"float32"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Normalizer", IDName: "normalizer", Doc: "Normalizer rescales values from the data coordinate system to the\nnormalized coordinate system.", Methods: []types.Method{{Name: "Normalize", Doc: "Normalize transforms a value x in the data coordinate system to\nthe normalized coordinate system.", Args: []string{"min", "max", "x"}, Returns: []string{"float64"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LinearScale", IDName: "linear-scale", Doc: "LinearScale an be used as the value of an Axis.Scale function to\nset the axis to a standard linear scale."}) @@ -50,31 +50,13 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LogScale", IDN var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.InvertedScale", IDName: "inverted-scale", Doc: "InvertedScale can be used as the value of an Axis.Scale function to\ninvert the axis using any Normalizer.", Embeds: []types.Field{{Name: "Normalizer"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Valuer", IDName: "valuer", Doc: "Valuer provides an interface for a list of scalar values", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of values.", Returns: []string{"int"}}, {Name: "Value", Doc: "Value returns a value.", Args: []string{"i"}, Returns: []string{"float32"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Data", IDName: "data", Doc: "Data is the data interface for plotting, supporting either\nfloat64 or string representations. It is satisfied by the tensor.Tensor\ninterface, so a tensor can be used directly for plot Data.", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of values.", Returns: []string{"int"}}, {Name: "Float1D", Doc: "Float1D(i int) returns float64 value at given index.", Args: []string{"i"}, Returns: []string{"float64"}}, {Name: "String1D", Doc: "String1D(i int) returns string value at given index.", Args: []string{"i"}, Returns: []string{"string"}}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Values", IDName: "values", Doc: "Values implements the Valuer interface."}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Roles", IDName: "roles", Doc: "Roles are the roles that a given set of data values can play,\ndesigned to be sufficiently generalizable across all different\ntypes of plots, even if sometimes it is a bit of a stretch."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TensorValues", IDName: "tensor-values", Doc: "TensorValues provides a Valuer interface wrapper for a tensor.", Embeds: []types.Field{{Name: "Tensor"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Values", IDName: "values", Doc: "Values provides a minimal implementation of the Data interface\nusing a slice of float64."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYer", IDName: "x-yer", Doc: "XYer provides an interface for a list of X,Y data pairs", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of x, y pairs.", Returns: []string{"int"}}, {Name: "XY", Doc: "XY returns an x, y pair.", Args: []string{"i"}, Returns: []string{"x", "y"}}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYs", IDName: "x-ys", Doc: "XYs implements the XYer interface."}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TensorXYs", IDName: "tensor-x-ys", Doc: "TensorXYs provides a XYer interface wrapper for a tensor.", Fields: []types.Field{{Name: "X"}, {Name: "Y"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XValues", IDName: "x-values", Doc: "XValues implements the Valuer interface,\nreturning the x value from an XYer.", Embeds: []types.Field{{Name: "XYer"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.YValues", IDName: "y-values", Doc: "YValues implements the Valuer interface,\nreturning the y value from an XYer.", Embeds: []types.Field{{Name: "XYer"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYZer", IDName: "xy-zer", Doc: "XYZer provides an interface for a list of X,Y,Z data triples.\nIt also satisfies the XYer interface for the X,Y pairs.", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of x, y, z triples.", Returns: []string{"int"}}, {Name: "XYZ", Doc: "XYZ returns an x, y, z triple.", Args: []string{"i"}, Returns: []string{"float32", "float32", "float32"}}, {Name: "XY", Doc: "XY returns an x, y pair.", Args: []string{"i"}, Returns: []string{"float32", "float32"}}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYZs", IDName: "xy-zs", Doc: "XYZs implements the XYZer interface using a slice."}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYZ", IDName: "xyz", Doc: "XYZ is an x, y and z value.", Fields: []types.Field{{Name: "X"}, {Name: "Y"}, {Name: "Z"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XYValues", IDName: "xy-values", Doc: "XYValues implements the XYer interface, returning\nthe x and y values from an XYZer.", Embeds: []types.Field{{Name: "XYZer"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Labeler", IDName: "labeler", Doc: "Labeler provides an interface for a list of string labels", Methods: []types.Method{{Name: "Label", Doc: "Label returns a label.", Args: []string{"i"}, Returns: []string{"string"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Labels", IDName: "labels", Doc: "Labels provides a minimal implementation of the Data interface\nusing a slice of string. It always returns 0 for Float1D."}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.selection", IDName: "selection", Fields: []types.Field{{Name: "n", Doc: "n is the number of labels selected."}, {Name: "lMin", Doc: "lMin and lMax are the selected min\nand max label values. lq is the q\nchosen."}, {Name: "lMax", Doc: "lMin and lMax are the selected min\nand max label values. lq is the q\nchosen."}, {Name: "lStep", Doc: "lMin and lMax are the selected min\nand max label values. lq is the q\nchosen."}, {Name: "lq", Doc: "lMin and lMax are the selected min\nand max label values. lq is the q\nchosen."}, {Name: "score", Doc: "score is the score for the selection."}, {Name: "magnitude", Doc: "magnitude is the magnitude of the\nlabel step distance."}}}) @@ -236,13 +218,13 @@ func (t *PlotStyle) SetLabelSize(v units.Value) *PlotStyle { t.LabelSize = v; re // BarWidth for Bar plot sets the default width of the bars, // which should be less than the Stride (1 typically) to prevent // bar overlap. Defaults to .8. -func (t *PlotStyle) SetBarWidth(v float32) *PlotStyle { t.BarWidth = v; return t } +func (t *PlotStyle) SetBarWidth(v float64) *PlotStyle { t.BarWidth = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Y", Doc: "X and Y are the horizontal and vertical axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Y", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Z", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint", Args: []string{"pt"}}, {Name: "XYData", Doc: "returns the data for this plot as X,Y points,\nincluding corresponding pixel data.\nThis allows gui interface to inspect data etc.", Returns: []string{"data", "pixels"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint.", Args: []string{"pt"}}, {Name: "UpdateRange", Doc: "UpdateRange updates the given ranges.", Args: []string{"x", "y", "z"}}, {Name: "Data", Doc: "Data returns the data by roles for this plot, for both the original\ndata and the pixel-transformed X,Y coordinates for that data.\nThis allows a GUI interface to inspect data etc.", Returns: []string{"data", "pixX", "pixY"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DataRanger", IDName: "data-ranger", Doc: "DataRanger wraps the DataRange method.", Methods: []types.Method{{Name: "DataRange", Doc: "DataRange returns the range of X and Y values.", Args: []string{"pt"}, Returns: []string{"xmin", "xmax", "ymin", "ymax"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotterType", IDName: "plotter-type", Doc: "PlotterType registers a Plotter so that it can be created with appropriate data.", Fields: []types.Field{{Name: "Name", Doc: "Name of the plot type."}, {Name: "Doc", Doc: "Doc is the documentation for this Plotter."}, {Name: "Required", Doc: "Required Data roles for this plot. Data for these Roles must be provided."}, {Name: "Optional", Doc: "Optional Data roles for this plot."}, {Name: "New", Doc: "New returns a new plotter of this type with given data in given roles."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PointStyle", IDName: "point-style", Doc: "PointStyle has style properties for drawing points as different shapes.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On indicates whether to plot points."}, {Name: "Shape", Doc: "Shape to draw."}, {Name: "Color", Doc: "Color is the stroke color image specification.\nSetting to nil turns line off."}, {Name: "Fill", Doc: "Fill is the color to fill solid regions, in a plot-specific\nway (e.g., the area below a Line plot, the bar color).\nUse nil to disable filling."}, {Name: "Width", Doc: "Width is the line width for point glyphs, with a default of 1 Pt (point).\nSetting to 0 turns line off."}, {Name: "Size", Doc: "Size of shape to draw for each point.\nDefaults to 4 Pt (point)."}}}) @@ -275,9 +257,9 @@ func (t *PointStyle) SetWidth(v units.Value) *PointStyle { t.Width = v; return t // Defaults to 4 Pt (point). func (t *PointStyle) SetSize(v units.Value) *PointStyle { t.Size = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Shapes", IDName: "shapes"}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Shapes", IDName: "shapes", Doc: "Shapes has the options for how to draw points in the plot."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for cases where it can be turned off."}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for cases where it can be turned off."}, {Name: "Role", Doc: "Role specifies a role for this item, used for table-based plots to indicate\nhow a particular column of data should be used."}, {Name: "Group", Doc: "Group specifies a group of related data items, used for table-based plots\nwhere different columns of data within the same Group play different Roles"}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) // SetPlot sets the [Style.Plot]: // @@ -290,9 +272,19 @@ func (t *Style) SetPlot(v PlotStyle) *Style { t.Plot = v; return t } // On specifies whether to plot this item, for cases where it can be turned off. func (t *Style) SetOn(v DefaultOffOn) *Style { t.On = v; return t } +// SetRole sets the [Style.Role]: +// Role specifies a role for this item, used for table-based plots to indicate +// how a particular column of data should be used. +func (t *Style) SetRole(v Roles) *Style { t.Role = v; return t } + +// SetGroup sets the [Style.Group]: +// Group specifies a group of related data items, used for table-based plots +// where different columns of data within the same Group play different Roles +func (t *Style) SetGroup(v string) *Style { t.Group = v; return t } + // SetRange sets the [Style.Range]: // Range is the effective range of data to plot, where either end can be fixed. -func (t *Style) SetRange(v minmax.Range32) *Style { t.Range = v; return t } +func (t *Style) SetRange(v minmax.Range64) *Style { t.Range = v; return t } // SetLabel sets the [Style.Label]: // Label provides an alternative label to use for axis, if set. @@ -329,16 +321,16 @@ func (t *WidthStyle) SetCap(v units.Value) *WidthStyle { t.Cap = v; return t } // Offset for Bar plot is the offset added to each X axis value // relative to the Stride computed value (X = offset + index * Stride) // Defaults to 1. -func (t *WidthStyle) SetOffset(v float32) *WidthStyle { t.Offset = v; return t } +func (t *WidthStyle) SetOffset(v float64) *WidthStyle { t.Offset = v; return t } // SetStride sets the [WidthStyle.Stride]: // Stride for Bar plot is distance between bars. Defaults to 1. -func (t *WidthStyle) SetStride(v float32) *WidthStyle { t.Stride = v; return t } +func (t *WidthStyle) SetStride(v float64) *WidthStyle { t.Stride = v; return t } // SetWidth sets the [WidthStyle.Width]: // Width for Bar plot is the width of the bars, which should be less than // the Stride (1 typically) to prevent bar overlap. Defaults to .8. -func (t *WidthStyle) SetWidth(v float32) *WidthStyle { t.Width = v; return t } +func (t *WidthStyle) SetWidth(v float64) *WidthStyle { t.Width = v; return t } // SetPad sets the [WidthStyle.Pad]: // Pad for Bar plot is additional space at start / end of data range, @@ -346,7 +338,7 @@ func (t *WidthStyle) SetWidth(v float32) *WidthStyle { t.Width = v; return t } // and added to (len(Values)-1)*Stride -- no other accommodation for bar // width is provided, so that should be built into this value as well. // Defaults to 1. -func (t *WidthStyle) SetPad(v float32) *WidthStyle { t.Pad = v; return t } +func (t *WidthStyle) SetPad(v float64) *WidthStyle { t.Pad = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Stylers", IDName: "stylers", Doc: "Stylers is a list of styling functions that set Style properties.\nThese are called in the order added."}) From c00873f3961492ad34fa695f7608c8c13b1db46b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 10:09:15 -0800 Subject: [PATCH 271/311] use Data type for map, much simpler call signature --- plot/README.md | 6 ++++++ plot/data.go | 22 +++++++++++++--------- plot/plot.go | 2 +- plot/plots/bar.go | 8 ++++---- plot/plots/errbars.go | 12 ++++++------ plot/plots/labels.go | 6 +++--- plot/plots/plot_test.go | 24 ++++++++++-------------- plot/plots/xy.go | 12 ++++++------ plot/plotter.go | 8 ++++---- 9 files changed, 53 insertions(+), 47 deletions(-) diff --git a/plot/README.md b/plot/README.md index bf7ba363cb..5b72c9f519 100644 --- a/plot/README.md +++ b/plot/README.md @@ -13,6 +13,12 @@ The GUI constraint requires a more systematic, factorial organization of the spa * Plot content is driven by `Plotter` elements that each consume one or more sets of data, which is provided by a `Data` interface that maps onto a minimal subset of the `tensor.Tensor` interface, so a tensor directly satisfies the interface. * Each `Plotter` element can generally handle multiple different data elements, that are index-aligned. For example, the basic `XY` plotter requires `X` and `Y` Valuers, and optionally `Size` or `Color` Valuers that apply to the Point elements, while `Bar` gets at least a `Y` but also optionally a `High` Valuer for an error bar. +Here is a example for how a plotter element is created with the `plot.Data` map of roles to data: + +```Go +plt.Add(plots.NewLine(plot.Data{plot.X: xd, plot.Y: yd, plot.Low: low, plot.High: high})) +``` + The table-driven plotting case uses a `Group` name along with the `Roles` type (`X`, `Y` etc) and Plotter type names to organize different plots based on `Style` settings. Columns with the same Group name all provide data to the same plotter using their different Roles, making it easy to configure various statistical plots of multiple series of grouped data. Different plotter types (including custom ones) are registered along with their accepted input roles, to allow any type of plot to be generated. diff --git a/plot/data.go b/plot/data.go index 720ac89bbc..0072fb3e98 100644 --- a/plot/data.go +++ b/plot/data.go @@ -26,10 +26,14 @@ var ( ErrNoData = errors.New("plotter: no data points") ) -// Data is the data interface for plotting, supporting either +// Data is a map of Roles and Data for that Role, providing the +// primary way of passing data to a Plotter +type Data map[Roles]Valuer + +// Valuer is the data interface for plotting, supporting either // float64 or string representations. It is satisfied by the tensor.Tensor // interface, so a tensor can be used directly for plot Data. -type Data interface { +type Valuer interface { // Len returns the number of values. Len() int @@ -113,7 +117,7 @@ func CheckNaNs(fs ...float64) bool { } // Range updates given Range with values from data. -func Range(data Data, rng *minmax.F64) { +func Range(data Valuer, rng *minmax.F64) { for i := 0; i < data.Len(); i++ { v := data.Float1D(i) if math.IsNaN(v) { @@ -125,7 +129,7 @@ func Range(data Data, rng *minmax.F64) { // RangeClamp updates the given axis Min, Max range values based // on the range of values in the given [Data], and the given style range. -func RangeClamp(data Data, axisRng *minmax.F64, styleRng *minmax.Range64) { +func RangeClamp(data Valuer, axisRng *minmax.F64, styleRng *minmax.Range64) { Range(data, axisRng) axisRng.Min, axisRng.Max = styleRng.Clamp(axisRng.Min, axisRng.Max) } @@ -150,7 +154,7 @@ func (vs Values) String1D(i int) string { // from Data, or an error if there are no values, or if one of // the copied values is a Infinity. // NaN values are skipped in the copying process. -func CopyValues(data Data) (Values, error) { +func CopyValues(data Valuer) (Values, error) { if data == nil { return nil, ErrNoData } @@ -173,7 +177,7 @@ func CopyValues(data Data) (Values, error) { // MustCopyRole returns Values copy of given role from given data map, // logging an error and returning nil if not present. -func MustCopyRole(data map[Roles]Data, role Roles) Values { +func MustCopyRole(data Data, role Roles) Values { d, ok := data[role] if !ok { slog.Error("plot Data role not present, but is required", "role:", role) @@ -185,7 +189,7 @@ func MustCopyRole(data map[Roles]Data, role Roles) Values { // CopyRole returns Values copy of given role from given data map, // returning nil if role not present. -func CopyRole(data map[Roles]Data, role Roles) Values { +func CopyRole(data Data, role Roles) Values { d, ok := data[role] if !ok { return nil @@ -195,7 +199,7 @@ func CopyRole(data map[Roles]Data, role Roles) Values { } // PlotX returns plot pixel X coordinate values for given data. -func PlotX(plt *Plot, data Data) []float32 { +func PlotX(plt *Plot, data Valuer) []float32 { px := make([]float32, data.Len()) for i := range px { px[i] = plt.PX(data.Float1D(i)) @@ -204,7 +208,7 @@ func PlotX(plt *Plot, data Data) []float32 { } // PlotY returns plot pixel Y coordinate values for given data. -func PlotY(plt *Plot, data Data) []float32 { +func PlotY(plt *Plot, data Valuer) []float32 { py := make([]float32, data.Len()) for i := range py { py[i] = plt.PY(data.Float1D(i)) diff --git a/plot/plot.go b/plot/plot.go index 3e4eeb9581..3f3574472d 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -335,7 +335,7 @@ func (pt *Plot) PY(v float64) float32 { // ClosestDataToPixel returns the Plotter data point closest to given pixel point, // in the Pixels image. -func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float32, pixel math32.Vector2, data map[Roles]Data, legend string) { +func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float32, pixel math32.Vector2, data Data, legend string) { tp := math32.Vec2(float32(px), float32(py)) dist = float32(math32.MaxFloat32) for _, p := range pt.Plotters { diff --git a/plot/plots/bar.go b/plot/plots/bar.go index d8e09c8ae7..61f5b3523b 100644 --- a/plot/plots/bar.go +++ b/plot/plots/bar.go @@ -23,7 +23,7 @@ import ( const BarType = "Bar" func init() { - plot.RegisterPlotter(BarType, "A Bar presents ordinally-organized data with rectangular bars with lengths proportional to the data values, and an optional error bar at the top of the bar using the High data role.", []plot.Roles{plot.Y}, []plot.Roles{plot.High}, func(data map[plot.Roles]plot.Data) plot.Plotter { + plot.RegisterPlotter(BarType, "A Bar presents ordinally-organized data with rectangular bars with lengths proportional to the data values, and an optional error bar at the top of the bar using the High data role.", []plot.Roles{plot.Y}, []plot.Roles{plot.High}, func(data plot.Data) plot.Plotter { return NewBar(data) }) } @@ -65,7 +65,7 @@ type Bar struct { // The bars heights correspond to the values and their x locations correspond // to the index of their value in the Valuer. // Optional error-bar values can be provided using the High data role. -func NewBar(data map[plot.Roles]plot.Data) *Bar { +func NewBar(data plot.Data) *Bar { bc := &Bar{} bc.Y = plot.MustCopyRole(data, plot.Y) if bc.Y == nil { @@ -92,10 +92,10 @@ func (bc *Bar) ApplyStyle(ps *plot.PlotStyle) { func (bc *Bar) Stylers() *plot.Stylers { return &bc.stylers } -func (bc *Bar) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { +func (bc *Bar) Data() (data plot.Data, pixX, pixY []float32) { pixX = bc.PX pixY = bc.PY - data = map[plot.Roles]plot.Data{} + data = plot.Data{} data[plot.X] = bc.X data[plot.Y] = bc.Y if bc.Err != nil { diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 2db6586df2..1dd063510e 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -32,7 +32,7 @@ func (eb *YErrorBars) Defaults() { // NewYErrorBars returns a new YErrorBars plotter, // using Low, High data roles for error deviations around X, Y coordinates. -func NewYErrorBars(data map[plot.Roles]plot.Data) *YErrorBars { +func NewYErrorBars(data plot.Data) *YErrorBars { eb := &YErrorBars{} eb.X = plot.MustCopyRole(data, plot.X) eb.Y = plot.MustCopyRole(data, plot.Y) @@ -58,10 +58,10 @@ func (eb *YErrorBars) ApplyStyle(ps *plot.PlotStyle) { func (eb *YErrorBars) Stylers() *plot.Stylers { return &eb.stylers } -func (eb *YErrorBars) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { +func (eb *YErrorBars) Data() (data plot.Data, pixX, pixY []float32) { pixX = eb.PX pixY = eb.PY - data = map[plot.Roles]plot.Data{} + data = plot.Data{} data[plot.X] = eb.X data[plot.Y] = eb.Y data[plot.Low] = eb.Low @@ -133,7 +133,7 @@ func (eb *XErrorBars) Defaults() { // NewXErrorBars returns a new XErrorBars plotter, // using Low, High data roles for error deviations around X, Y coordinates. -func NewXErrorBars(data map[plot.Roles]plot.Data) *XErrorBars { +func NewXErrorBars(data plot.Data) *XErrorBars { eb := &XErrorBars{} eb.X = plot.MustCopyRole(data, plot.X) eb.Y = plot.MustCopyRole(data, plot.Y) @@ -159,10 +159,10 @@ func (eb *XErrorBars) ApplyStyle(ps *plot.PlotStyle) { func (eb *XErrorBars) Stylers() *plot.Stylers { return &eb.stylers } -func (eb *XErrorBars) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { +func (eb *XErrorBars) Data() (data plot.Data, pixX, pixY []float32) { pixX = eb.PX pixY = eb.PY - data = map[plot.Roles]plot.Data{} + data = plot.Data{} data[plot.X] = eb.X data[plot.Y] = eb.Y data[plot.Low] = eb.Low diff --git a/plot/plots/labels.go b/plot/plots/labels.go index fa047b8ca6..ff7c5de951 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -31,7 +31,7 @@ type Labels struct { } // NewLabels returns a new Labels using defaults -func NewLabels(data map[plot.Roles]plot.Data) *Labels { +func NewLabels(data plot.Data) *Labels { lb := &Labels{} lb.X = plot.MustCopyRole(data, plot.X) lb.Y = plot.MustCopyRole(data, plot.Y) @@ -74,10 +74,10 @@ func (lb *Labels) ApplyStyle(ps *plot.PlotStyle) { func (lb *Labels) Stylers() *plot.Stylers { return &lb.stylers } -func (lb *Labels) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { +func (lb *Labels) Data() (data plot.Data, pixX, pixY []float32) { pixX = lb.PX pixY = lb.PY - data = map[plot.Roles]plot.Data{} + data = plot.Data{} data[plot.X] = lb.X data[plot.Y] = lb.Y data[plot.Label] = lb.Labels diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 9dfbb10651..b0e568a35c 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -45,7 +45,7 @@ func TestMain(m *testing.M) { } // sinCosWrapData returns overlapping sin / cos curves in one sequence. -func sinCosWrapData() map[plot.Roles]plot.Data { +func sinCosWrapData() plot.Data { xd, yd := make(plot.Values, 42), make(plot.Values, 42) for i := range xd { x := float64(i % 21) @@ -56,42 +56,42 @@ func sinCosWrapData() map[plot.Roles]plot.Data { yd[i] = float64(50) + 40*math.Cos((x/8)*math.Pi) } } - data := map[plot.Roles]plot.Data{} + data := plot.Data{} data[plot.X] = xd data[plot.Y] = yd return data } -func sinDataXY() map[plot.Roles]plot.Data { +func sinDataXY() plot.Data { xd, yd := make(plot.Values, 21), make(plot.Values, 21) for i := range xd { xd[i] = float64(i * 5) xd[i] = float64(50) + 40*math.Sin((float64(i)/8)*math.Pi) } - data := map[plot.Roles]plot.Data{} + data := plot.Data{} data[plot.X] = xd data[plot.Y] = yd return data } -func sinData() map[plot.Roles]plot.Data { +func sinData() plot.Data { yd := make(plot.Values, 21) for i := range yd { x := float64(i % 21) yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) } - data := map[plot.Roles]plot.Data{} + data := plot.Data{} data[plot.Y] = yd return data } -func cosData() map[plot.Roles]plot.Data { +func cosData() plot.Data { yd := make(plot.Values, 21) for i := range yd { x := float64(i % 21) yd[i] = float64(50) + 40*math.Cos((x/8)*math.Pi) } - data := map[plot.Roles]plot.Data{} + data := plot.Data{} data[plot.Y] = yd return data } @@ -186,7 +186,7 @@ func TestLabels(t *testing.T) { yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) labels[i] = fmt.Sprintf("%7.4g", yd[i]) } - data := map[plot.Roles]plot.Data{} + data := plot.Data{} data[plot.X] = xd data[plot.Y] = yd data[plot.Label] = labels @@ -338,11 +338,7 @@ func TestErrBar(t *testing.T) { low[i] = -high[i] } - data := map[plot.Roles]plot.Data{} - data[plot.X] = xd - data[plot.Y] = yd - data[plot.Low] = low - data[plot.High] = high + data := plot.Data{plot.X: xd, plot.Y: yd, plot.Low: low, plot.High: high} l1 := NewLine(data) if l1 == nil { diff --git a/plot/plots/xy.go b/plot/plots/xy.go index b5c22e6c07..788b205d54 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -21,7 +21,7 @@ import ( const XYType = "XY" func init() { - plot.RegisterPlotter(XYType, "draws lines between and / or points for X,Y data values, using optional Size and Color data for the points, for a bubble plot.", []plot.Roles{plot.X, plot.Y}, []plot.Roles{plot.Size, plot.Color}, func(data map[plot.Roles]plot.Data) plot.Plotter { + plot.RegisterPlotter(XYType, "draws lines between and / or points for X,Y data values, using optional Size and Color data for the points, for a bubble plot.", []plot.Roles{plot.X, plot.Y}, []plot.Roles{plot.Size, plot.Color}, func(data plot.Data) plot.Plotter { return NewXY(data) }) } @@ -41,7 +41,7 @@ type XY struct { } // NewXY returns an XY plot. -func NewXY(data map[plot.Roles]plot.Data) *XY { +func NewXY(data plot.Data) *XY { ln := &XY{} ln.X = plot.MustCopyRole(data, plot.X) ln.Y = plot.MustCopyRole(data, plot.Y) @@ -55,7 +55,7 @@ func NewXY(data map[plot.Roles]plot.Data) *XY { } // NewLine returns an XY plot drawing Lines by default. -func NewLine(data map[plot.Roles]plot.Data) *XY { +func NewLine(data plot.Data) *XY { ln := NewXY(data) if ln == nil { return ln @@ -66,7 +66,7 @@ func NewLine(data map[plot.Roles]plot.Data) *XY { } // NewScatter returns an XY scatter plot drawing Points by default. -func NewScatter(data map[plot.Roles]plot.Data) *XY { +func NewScatter(data plot.Data) *XY { ln := NewXY(data) if ln == nil { return ln @@ -93,10 +93,10 @@ func (ln *XY) ApplyStyle(ps *plot.PlotStyle) { ln.stylers.Run(&ln.Style) } -func (ln *XY) Data() (data map[plot.Roles]plot.Data, pixX, pixY []float32) { +func (ln *XY) Data() (data plot.Data, pixX, pixY []float32) { pixX = ln.PX pixY = ln.PY - data = map[plot.Roles]plot.Data{} + data = plot.Data{} data[plot.X] = ln.X data[plot.Y] = ln.Y if ln.Size != nil { diff --git a/plot/plotter.go b/plot/plotter.go index 4e15fc9001..6cc764b7c6 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -22,7 +22,7 @@ type Plotter interface { // Data returns the data by roles for this plot, for both the original // data and the pixel-transformed X,Y coordinates for that data. // This allows a GUI interface to inspect data etc. - Data() (data map[Roles]Data, pixX, pixY []float32) + Data() (data Data, pixX, pixY []float32) // Stylers returns the styler functions for this element. Stylers() *Stylers @@ -48,14 +48,14 @@ type PlotterType struct { Optional []Roles // New returns a new plotter of this type with given data in given roles. - New func(data map[Roles]Data) Plotter + New func(data Data) Plotter } // Plotters is the registry of [Plotter] types. var Plotters map[string]PlotterType // RegisterPlotter registers a plotter type. -func RegisterPlotter(name, doc string, required, optional []Roles, newFun func(data map[Roles]Data) Plotter) { +func RegisterPlotter(name, doc string, required, optional []Roles, newFun func(data Data) Plotter) { if Plotters == nil { Plotters = make(map[string]PlotterType) } @@ -65,7 +65,7 @@ func RegisterPlotter(name, doc string, required, optional []Roles, newFun func(d // NewPlotter returns a new plotter of given type, e.g., "XY", "Bar" etc, // for given data roles (which must include Required roles, and may include Optional ones). // Logs an error and returns nil if type name is not recognized as a registered type. -func NewPlotter(typeName string, data map[Roles]Data) Plotter { +func NewPlotter(typeName string, data Data) Plotter { pt, ok := Plotters[typeName] if !ok { slog.Error("plot.NewPlotter type name is not registered", "typeName", typeName) From 8c171b0b074ff3aa6d653d23e1b9db8b367caab6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 10:57:09 -0800 Subject: [PATCH 272/311] plot.Data is the map, Valuer is the value interface -- use plot.Data{plot.X: xd..} to pass data to plot -- works well. all tests fixed except styling range not working. --- plot/plot.go | 8 ++++++-- plot/plots/bar.go | 2 ++ plot/plots/plot_test.go | 22 ++++++---------------- plot/plots/xy.go | 2 +- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/plot/plot.go b/plot/plot.go index 3f3574472d..e6d61d5d1f 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -213,8 +213,12 @@ func (pt *Plot) applyStyle() { pt.Legend.Style.Text.openFont(pt) pt.X.Style = pt.Style.Axis pt.Y.Style = pt.Style.Axis - pt.X.Label.Text = pt.Style.XAxis.Label - pt.Y.Label.Text = pt.Style.YAxisLabel + if pt.Style.XAxis.Label != "" { + pt.X.Label.Text = pt.Style.XAxis.Label + } + if pt.Style.YAxisLabel != "" { + pt.Y.Label.Text = pt.Style.YAxisLabel + } pt.X.Label.Style = pt.Style.Axis.Text pt.Y.Label.Style = pt.Style.Axis.Text pt.X.TickText.Style = pt.Style.Axis.TickText diff --git a/plot/plots/bar.go b/plot/plots/bar.go index 61f5b3523b..fdc01f8f04 100644 --- a/plot/plots/bar.go +++ b/plot/plots/bar.go @@ -139,6 +139,8 @@ func (bc *Bar) Plot(plt *plot.Plot) { nv := len(bc.Y) bc.X = make(plot.Values, nv) bc.Yp = make(plot.Values, nv) + bc.PX = make([]float32, nv) + bc.PY = make([]float32, nv) hw := 0.5 * bw.Width ew := bw.Width / 3 diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index b0e568a35c..06aee06e96 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -56,22 +56,16 @@ func sinCosWrapData() plot.Data { yd[i] = float64(50) + 40*math.Cos((x/8)*math.Pi) } } - data := plot.Data{} - data[plot.X] = xd - data[plot.Y] = yd - return data + return plot.Data{plot.X: xd, plot.Y: yd} } func sinDataXY() plot.Data { xd, yd := make(plot.Values, 21), make(plot.Values, 21) for i := range xd { xd[i] = float64(i * 5) - xd[i] = float64(50) + 40*math.Sin((float64(i)/8)*math.Pi) + yd[i] = float64(50) + 40*math.Sin((float64(i)/8)*math.Pi) } - data := plot.Data{} - data[plot.X] = xd - data[plot.Y] = yd - return data + return plot.Data{plot.X: xd, plot.Y: yd} } func sinData() plot.Data { @@ -80,9 +74,7 @@ func sinData() plot.Data { x := float64(i % 21) yd[i] = float64(50) + 40*math.Sin((x/8)*math.Pi) } - data := plot.Data{} - data[plot.Y] = yd - return data + return plot.Data{plot.Y: yd} } func cosData() plot.Data { @@ -91,9 +83,7 @@ func cosData() plot.Data { x := float64(i % 21) yd[i] = float64(50) + 40*math.Cos((x/8)*math.Pi) } - data := plot.Data{} - data[plot.Y] = yd - return data + return plot.Data{plot.Y: yd} } func TestLine(t *testing.T) { @@ -246,7 +236,7 @@ func TestBar(t *testing.T) { l2.Style.Width.Stride = 2 l2.Style.Width.Offset = 2 - plt.Add(l2) // note: range updated when added! + plt.Add(l2) plt.Draw() imagex.Assert(t, plt.Pixels, "bar-cos.png") } diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 788b205d54..9d4563ae34 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -112,7 +112,7 @@ func (ln *XY) Data() (data plot.Data, pixX, pixY []float32) { func (ln *XY) Plot(plt *plot.Plot) { pc := plt.Paint ln.PX = plot.PlotX(plt, ln.X) - ln.PY = plot.PlotX(plt, ln.Y) + ln.PY = plot.PlotY(plt, ln.Y) np := len(ln.PX) if ln.Style.Line.Fill != nil { From 1bdcd3df5d08a340eb7015bf07239ecb045b99b3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 14:42:54 -0800 Subject: [PATCH 273/311] fix Xaxis style scaling --- plot/plot.go | 8 ++++++- plot/plots/plot_test.go | 46 ++++++++++++++++++++++++----------------- plot/typegen.go | 8 ++++--- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/plot/plot.go b/plot/plot.go index e6d61d5d1f..aaada176be 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -38,7 +38,7 @@ type XAxisStyle struct { //types:add -setters Label string // Range is the effective range of XAxis data to plot, where either end can be fixed. - Range minmax.Range32 `display:"inline"` + Range minmax.Range64 `display:"inline"` } // PlotStyle has overall plot level styling properties. @@ -321,6 +321,12 @@ func (pt *Plot) UpdateRange() { pt.X.Range.SetInfinity() pt.Y.Range.SetInfinity() pt.Z.Range.SetInfinity() + if pt.Style.XAxis.Range.FixMin { + pt.X.Range.Min = pt.Style.XAxis.Range.Min + } + if pt.Style.XAxis.Range.FixMax { + pt.X.Range.Max = pt.Style.XAxis.Range.Max + } for _, pl := range pt.Plotters { pl.UpdateRange(pt, &pt.X.Range, &pt.Y.Range, &pt.Z.Range) } diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 06aee06e96..6b588292e1 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -18,25 +18,33 @@ import ( ) func ExampleLine() { - // data := make(plot.XYs, 42) - // for i := range data { - // x := float32(i % 21) - // data[i].X = x * 5 - // if i < 21 { - // data[i].Y = float32(50) + 40*math32.Sin((x/8)*math32.Pi) - // } else { - // data[i].Y = float32(50) + 40*math32.Cos((x/8)*math32.Pi) - // } - // } - // - // plt := plot.New() - // plt.Add(NewLine(data).Styler(func(s *plot.Style) { - // s.Line.Color = colors.Uniform(colors.Red) - // s.Line.Width.Pt(2) - // })) - // plt.Draw() - // imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") - // // Output: + xd, yd := make(plot.Values, 21), make(plot.Values, 21) + for i := range xd { + xd[i] = float64(i * 5) + yd[i] = float64(50) + 40*math.Sin((float64(i)/8)*math.Pi) + } + data := plot.Data{plot.X: xd, plot.Y: yd} + plt := plot.New() + plt.Add(NewLine(data).Styler(func(s *plot.Style) { + s.Plot.Title = "Test Line" + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plot.XAxis.Range.SetMax(105) + s.Plot.LineWidth.Pt(2) + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) + s.Plot.TitleStyle.Size.Dp(48) + s.Plot.Legend.Position.Left = true + s.Plot.Legend.Text.Size.Dp(24) + s.Plot.Axis.Text.Size.Dp(32) + s.Plot.Axis.TickText.Size.Dp(24) + s.Plot.XAxis.Rotation = -45 + s.Line.Color = colors.Uniform(colors.Red) + s.Point.Color = colors.Uniform(colors.Blue) + s.Range.SetMax(100) + })) + plt.Draw() + imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") + // Output: } func TestMain(m *testing.M) { diff --git a/plot/typegen.go b/plot/typegen.go index f462ffb0f0..c309400383 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -50,7 +50,9 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LogScale", IDN var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.InvertedScale", IDName: "inverted-scale", Doc: "InvertedScale can be used as the value of an Axis.Scale function to\ninvert the axis using any Normalizer.", Embeds: []types.Field{{Name: "Normalizer"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Data", IDName: "data", Doc: "Data is the data interface for plotting, supporting either\nfloat64 or string representations. It is satisfied by the tensor.Tensor\ninterface, so a tensor can be used directly for plot Data.", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of values.", Returns: []string{"int"}}, {Name: "Float1D", Doc: "Float1D(i int) returns float64 value at given index.", Args: []string{"i"}, Returns: []string{"float64"}}, {Name: "String1D", Doc: "String1D(i int) returns string value at given index.", Args: []string{"i"}, Returns: []string{"string"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Data", IDName: "data", Doc: "Data is a map of Roles and Data for that Role, providing the\nprimary way of passing data to a Plotter"}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Valuer", IDName: "valuer", Doc: "Valuer is the data interface for plotting, supporting either\nfloat64 or string representations. It is satisfied by the tensor.Tensor\ninterface, so a tensor can be used directly for plot Data.", Methods: []types.Method{{Name: "Len", Doc: "Len returns the number of values.", Returns: []string{"int"}}, {Name: "Float1D", Doc: "Float1D(i int) returns float64 value at given index.", Args: []string{"i"}, Returns: []string{"float64"}}, {Name: "String1D", Doc: "String1D(i int) returns string value at given index.", Args: []string{"i"}, Returns: []string{"string"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Roles", IDName: "roles", Doc: "Roles are the roles that a given set of data values can play,\ndesigned to be sufficiently generalizable across all different\ntypes of plots, even if sometimes it is a bit of a stretch."}) @@ -152,7 +154,7 @@ func (t *XAxisStyle) SetLabel(v string) *XAxisStyle { t.Label = v; return t } // SetRange sets the [XAxisStyle.Range]: // Range is the effective range of XAxis data to plot, where either end can be fixed. -func (t *XAxisStyle) SetRange(v minmax.Range32) *XAxisStyle { t.Range = v; return t } +func (t *XAxisStyle) SetRange(v minmax.Range64) *XAxisStyle { t.Range = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotStyle", IDName: "plot-style", Doc: "PlotStyle has overall plot level styling properties.\nSome properties provide defaults for individual elements, which can\nthen be overwritten by element-level properties.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Title", Doc: "Title is the overall title of the plot."}, {Name: "TitleStyle", Doc: "TitleStyle is the text styling parameters for the title."}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Legend", Doc: "Legend has the styling properties for the Legend."}, {Name: "Axis", Doc: "Axis has the styling properties for the Axes."}, {Name: "XAxis", Doc: "XAxis has plot-level XAxis style properties."}, {Name: "YAxisLabel", Doc: "YAxisLabel is the optional label to use for the YAxis instead of the default."}, {Name: "LinesOn", Doc: "LinesOn determines whether lines are plotted by default,\nfor elements that plot lines (e.g., plots.XY)."}, {Name: "LineWidth", Doc: "LineWidth sets the default line width for data plotting lines."}, {Name: "PointsOn", Doc: "PointsOn determines whether points are plotted by default,\nfor elements that plot points (e.g., plots.XY)."}, {Name: "PointSize", Doc: "PointSize sets the default point size."}, {Name: "LabelSize", Doc: "LabelSize sets the default label text size."}, {Name: "BarWidth", Doc: "BarWidth for Bar plot sets the default width of the bars,\nwhich should be less than the Stride (1 typically) to prevent\nbar overlap. Defaults to .8."}}}) @@ -222,7 +224,7 @@ func (t *PlotStyle) SetBarWidth(v float64) *PlotStyle { t.BarWidth = v; return t var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Y", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Z", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint.", Args: []string{"pt"}}, {Name: "UpdateRange", Doc: "UpdateRange updates the given ranges.", Args: []string{"x", "y", "z"}}, {Name: "Data", Doc: "Data returns the data by roles for this plot, for both the original\ndata and the pixel-transformed X,Y coordinates for that data.\nThis allows a GUI interface to inspect data etc.", Returns: []string{"data", "pixX", "pixY"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint.", Args: []string{"pt"}}, {Name: "UpdateRange", Doc: "UpdateRange updates the given ranges.", Args: []string{"plt", "x", "y", "z"}}, {Name: "Data", Doc: "Data returns the data by roles for this plot, for both the original\ndata and the pixel-transformed X,Y coordinates for that data.\nThis allows a GUI interface to inspect data etc.", Returns: []string{"data", "pixX", "pixY"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotterType", IDName: "plotter-type", Doc: "PlotterType registers a Plotter so that it can be created with appropriate data.", Fields: []types.Field{{Name: "Name", Doc: "Name of the plot type."}, {Name: "Doc", Doc: "Doc is the documentation for this Plotter."}, {Name: "Required", Doc: "Required Data roles for this plot. Data for these Roles must be provided."}, {Name: "Optional", Doc: "Optional Data roles for this plot."}, {Name: "New", Doc: "New returns a new plotter of this type with given data in given roles."}}}) From d028ce76a796a5a84ae7122b92704799f83f1a18 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 15:01:42 -0800 Subject: [PATCH 274/311] checklengths --- plot/data.go | 17 +++++++++++++++++ plot/plots/bar.go | 21 ++++++++++++--------- plot/plots/errbars.go | 18 ++++++++++++------ plot/plots/labels.go | 33 ++++++++++++++------------------- plot/plots/xy.go | 9 ++++++--- plot/plotter.go | 2 +- 6 files changed, 62 insertions(+), 38 deletions(-) diff --git a/plot/data.go b/plot/data.go index 0072fb3e98..177ae1146e 100644 --- a/plot/data.go +++ b/plot/data.go @@ -134,6 +134,23 @@ func RangeClamp(data Valuer, axisRng *minmax.F64, styleRng *minmax.Range64) { axisRng.Min, axisRng.Max = styleRng.Clamp(axisRng.Min, axisRng.Max) } +// CheckLengths checks that all the data elements have the same length. +// Logs and returns an error if not. +func (dt Data) CheckLengths() error { + n := 0 + for _, v := range dt { + if n == 0 { + n = v.Len() + } else { + if v.Len() != n { + err := errors.New("plot.Data has inconsistent lengths -- all data elements must have the same length -- plotting aborted") + return errors.Log(err) + } + } + } + return nil +} + // Values provides a minimal implementation of the Data interface // using a slice of float64. type Values []float64 diff --git a/plot/plots/bar.go b/plot/plots/bar.go index fdc01f8f04..f1d669263a 100644 --- a/plot/plots/bar.go +++ b/plot/plots/bar.go @@ -66,6 +66,9 @@ type Bar struct { // to the index of their value in the Valuer. // Optional error-bar values can be provided using the High data role. func NewBar(data plot.Data) *Bar { + if data.CheckLengths() != nil { + return nil + } bc := &Bar{} bc.Y = plot.MustCopyRole(data, plot.Y) if bc.Y == nil { @@ -202,7 +205,7 @@ func (bc *Bar) Plot(plt *plot.Plot) { } // UpdateRange updates the given ranges. -func (bc *Bar) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { +func (bc *Bar) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { bw := bc.Style.Width catMin := bw.Offset - bw.Pad catMax := bw.Offset + float64(len(bc.Y)-1)*bw.Stride + bw.Pad @@ -214,19 +217,19 @@ func (bc *Bar) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { valTop += math.Abs(bc.Err[i]) } if bc.Horizontal { - x.FitValInRange(valBot) - x.FitValInRange(valTop) + xr.FitValInRange(valBot) + xr.FitValInRange(valTop) } else { - y.FitValInRange(valBot) - y.FitValInRange(valTop) + yr.FitValInRange(valBot) + yr.FitValInRange(valTop) } } if bc.Horizontal { - x.Min, x.Max = bc.Style.Range.Clamp(x.Min, x.Max) - y.FitInRange(minmax.F64{catMin, catMax}) + xr.Min, xr.Max = bc.Style.Range.Clamp(xr.Min, xr.Max) + yr.FitInRange(minmax.F64{catMin, catMax}) } else { - y.Min, y.Max = bc.Style.Range.Clamp(y.Min, y.Max) - x.FitInRange(minmax.F64{catMin, catMax}) + yr.Min, yr.Max = bc.Style.Range.Clamp(yr.Min, yr.Max) + xr.FitInRange(minmax.F64{catMin, catMax}) } } diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 1dd063510e..e6531886e9 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -33,6 +33,9 @@ func (eb *YErrorBars) Defaults() { // NewYErrorBars returns a new YErrorBars plotter, // using Low, High data roles for error deviations around X, Y coordinates. func NewYErrorBars(data plot.Data) *YErrorBars { + if data.CheckLengths() != nil { + return nil + } eb := &YErrorBars{} eb.X = plot.MustCopyRole(data, plot.X) eb.Y = plot.MustCopyRole(data, plot.Y) @@ -100,12 +103,12 @@ func (eb *YErrorBars) Plot(plt *plot.Plot) { } // UpdateRange updates the given ranges. -func (eb *YErrorBars) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { - plot.Range(eb.X, x) - for i, yv := range eb.Y { - ylow := yv - math.Abs(eb.Low[i]) - yhigh := yv + math.Abs(eb.High[i]) - y.FitInRange(minmax.F64{ylow, yhigh}) +func (eb *YErrorBars) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { + plot.Range(eb.X, xr) + for i, y := range eb.Y { + ylow := y - math.Abs(eb.Low[i]) + yhigh := y + math.Abs(eb.High[i]) + yr.FitInRange(minmax.F64{ylow, yhigh}) } return } @@ -134,6 +137,9 @@ func (eb *XErrorBars) Defaults() { // NewXErrorBars returns a new XErrorBars plotter, // using Low, High data roles for error deviations around X, Y coordinates. func NewXErrorBars(data plot.Data) *XErrorBars { + if data.CheckLengths() != nil { + return nil + } eb := &XErrorBars{} eb.X = plot.MustCopyRole(data, plot.X) eb.Y = plot.MustCopyRole(data, plot.Y) diff --git a/plot/plots/labels.go b/plot/plots/labels.go index ff7c5de951..e2c7570acc 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -32,6 +32,9 @@ type Labels struct { // NewLabels returns a new Labels using defaults func NewLabels(data plot.Data) *Labels { + if data.CheckLengths() != nil { + return nil + } lb := &Labels{} lb.X = plot.MustCopyRole(data, plot.X) lb.Y = plot.MustCopyRole(data, plot.Y) @@ -42,12 +45,6 @@ func NewLabels(data plot.Data) *Labels { if ld == nil { return nil } - - // todo: in general need length checking on all data maps! - // if d.Len() != len(xys) { - // errors.Log(errors.New("plotter: number of points does not match the number of labels")) - // return nil - // } lb.Labels = make(plot.Labels, lb.X.Len()) for i := range ld.Len() { lb.Labels[i] = ld.String1D(i) @@ -106,13 +103,13 @@ func (lb *Labels) Plot(plt *plot.Plot) { } // UpdateRange updates the given ranges. -func (lb *Labels) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { +func (lb *Labels) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { // todo: include point sizes! - plot.Range(lb.X, x) - plot.Range(lb.Y, y) + plot.Range(lb.X, xr) + plot.Range(lb.Y, yr) pxToData := math32.FromPoint(plt.Size) - pxToData.X = float32(x.Range()) / pxToData.X - pxToData.Y = float32(y.Range()) / pxToData.Y + pxToData.X = float32(xr.Range()) / pxToData.X + pxToData.Y = float32(yr.Range()) / pxToData.Y var ltxt plot.Text ltxt.Style = lb.Style for i, label := range lb.Labels { @@ -123,13 +120,11 @@ func (lb *Labels) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { ltxt.Config(plt) tht := pxToData.Y * ltxt.PaintText.BBox.Size().Y twd := 1.1 * pxToData.X * ltxt.PaintText.BBox.Size().X - xv := lb.X[i] - yv := lb.Y[i] - minx := xv - maxx := xv + float64(pxToData.X*lb.Style.Offset.X.Dots+twd) - miny := yv - maxy := yv + float64(pxToData.Y*lb.Style.Offset.Y.Dots+tht) // y is up here - x.FitInRange(minmax.F64{minx, maxx}) - y.FitInRange(minmax.F64{miny, maxy}) + x := lb.X[i] + y := lb.Y[i] + maxx := x + float64(pxToData.X*lb.Style.Offset.X.Dots+twd) + maxy := y + float64(pxToData.Y*lb.Style.Offset.Y.Dots+tht) // y is up here + xr.FitInRange(minmax.F64{x, maxx}) + yr.FitInRange(minmax.F64{y, maxy}) } } diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 9d4563ae34..31b6c4d717 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -42,6 +42,9 @@ type XY struct { // NewXY returns an XY plot. func NewXY(data plot.Data) *XY { + if data.CheckLengths() != nil { + return nil + } ln := &XY{} ln.X = plot.MustCopyRole(data, plot.X) ln.Y = plot.MustCopyRole(data, plot.Y) @@ -210,10 +213,10 @@ func (ln *XY) Plot(plt *plot.Plot) { } // UpdateRange updates the given ranges. -func (ln *XY) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { +func (ln *XY) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { // todo: include point sizes! - plot.Range(ln.X, x) - plot.RangeClamp(ln.Y, y, &ln.Style.Range) + plot.Range(ln.X, xr) + plot.RangeClamp(ln.Y, yr, &ln.Style.Range) } // Thumbnail returns the thumbnail, implementing the plot.Thumbnailer interface. diff --git a/plot/plotter.go b/plot/plotter.go index 6cc764b7c6..68c41cc231 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -17,7 +17,7 @@ type Plotter interface { Plot(pt *Plot) // UpdateRange updates the given ranges. - UpdateRange(plt *Plot, x, y, z *minmax.F64) + UpdateRange(plt *Plot, xr, yr, zr *minmax.F64) // Data returns the data by roles for this plot, for both the original // data and the pixel-transformed X,Y coordinates for that data. From 71150b9da0a71c1304ba311d77bef76d7462012b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 15:13:02 -0800 Subject: [PATCH 275/311] rest of register functions --- plot/plots/errbars.go | 14 +++++++++++++- plot/plots/labels.go | 12 ++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index e6531886e9..264cfbe850 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -11,7 +11,19 @@ import ( "cogentcore.org/core/plot" ) -// YErrorBars draws vertical error bars, denoting error in Y values, +// YErrorBarsType is be used for specifying the type name. +const YErrorBarsType = "YErrorBars" + +func init() { + plot.RegisterPlotter(YErrorBarsType, "draws draws vertical error bars, denoting error in Y values, using Low, High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.Low, plot.High}, []plot.Roles{}, func(data plot.Data) plot.Plotter { + return NewYErrorBars(data) + }) + plot.RegisterPlotter(XErrorBarsType, "draws draws horizontal error bars, denoting error in X values, using Low, High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.Low, plot.High}, []plot.Roles{}, func(data plot.Data) plot.Plotter { + return NewXErrorBars(data) + }) +} + +// XErrorBars draws vertical error bars, denoting error in Y values, // using Low, High data roles for error deviations around X, Y coordinates. type YErrorBars struct { // copies of data for this line diff --git a/plot/plots/labels.go b/plot/plots/labels.go index e2c7570acc..a16b9c7a7b 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -12,8 +12,16 @@ import ( "cogentcore.org/core/plot" ) -// Labels implements the Plotter interface, -// drawing a set of labels at specified points. +// LabelsType is be used for specifying the type name. +const LabelsType = "Labels" + +func init() { + plot.RegisterPlotter(LabelsType, "draws text labels at specified X, Y points.", []plot.Roles{plot.X, plot.Y, plot.Label}, []plot.Roles{}, func(data plot.Data) plot.Plotter { + return NewLabels(data) + }) +} + +// Labels draws text labels at specified X, Y points. type Labels struct { // copies of data for this line X, Y plot.Values From 90536d62d7554a81ca2cedc416716543f7e63ebb Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 9 Nov 2024 15:14:16 -0800 Subject: [PATCH 276/311] fix xerr --- plot/plots/errbars.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 264cfbe850..ae806de37b 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -11,8 +11,13 @@ import ( "cogentcore.org/core/plot" ) -// YErrorBarsType is be used for specifying the type name. -const YErrorBarsType = "YErrorBars" +const ( + // YErrorBarsType is be used for specifying the type name. + YErrorBarsType = "YErrorBars" + + // XErrorBarsType is be used for specifying the type name. + XErrorBarsType = "XErrorBars" +) func init() { plot.RegisterPlotter(YErrorBarsType, "draws draws vertical error bars, denoting error in Y values, using Low, High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.Low, plot.High}, []plot.Roles{}, func(data plot.Data) plot.Plotter { From 43c31571a899ca158dab2ab03e466025d5a16eda Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sun, 10 Nov 2024 19:14:50 -0800 Subject: [PATCH 277/311] update README --- plot/README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/plot/README.md b/plot/README.md index 5b72c9f519..c43b0ce0e4 100644 --- a/plot/README.md +++ b/plot/README.md @@ -9,13 +9,17 @@ The `plot` package generates 2D plots of data using the Cogent Core `paint` rend * GUI-based configuration of plots based on a `tensor.Table` of data columns (via `PlotEditor`). The GUI constraint requires a more systematic, factorial organization of the space of possible plot data and how it is organized to create a plot, so that it can be configured with a relatively simple set of GUI settings. The overall logic is as follows: -* The overall plot has a single shared range of X, Y and optionally Z coordinate ranges, that defines where a data value in any plot type is plotted. These ranges are set based on the DataRanger interface. -* Plot content is driven by `Plotter` elements that each consume one or more sets of data, which is provided by a `Data` interface that maps onto a minimal subset of the `tensor.Tensor` interface, so a tensor directly satisfies the interface. -* Each `Plotter` element can generally handle multiple different data elements, that are index-aligned. For example, the basic `XY` plotter requires `X` and `Y` Valuers, and optionally `Size` or `Color` Valuers that apply to the Point elements, while `Bar` gets at least a `Y` but also optionally a `High` Valuer for an error bar. + +* The overall plot has a single shared range of X, Y and optionally Z coordinate ranges (under the corresponding `Axis` field), that defines where a data value in any plot type is plotted. These ranges are set based on the DataRanger interface. + +* Plot content is driven by `Plotter` elements that each consume one or more sets of data, which is provided by a `Valuer` interface that maps onto a minimal subset of the `tensor.Tensor` interface, so a tensor directly satisfies the interface. + +* Each `Plotter` element can generally handle multiple different data elements, that are index-aligned. For example, the basic `XY` plotter requires `X` and `Y` Valuers, and optionally `Size` or `Color` Valuers that apply to the Point elements, while `Bar` gets at least a `Y` but also optionally a `High` Valuer for an error bar. The `plot.Data` = `map[Roles]Valuer` is used to create new Plotter elements, allowing an unordered and explicit way of specifying the `Roles` of each `Valuer` item. Here is a example for how a plotter element is created with the `plot.Data` map of roles to data: ```Go +plt := plot.NewPlot() plt.Add(plots.NewLine(plot.Data{plot.X: xd, plot.Y: yd, plot.Low: low, plot.High: high})) ``` @@ -31,7 +35,7 @@ Each such plot element defines a `Styler` method, e.g.,: ```Go plt := plot.NewPlot() -ln := plots.AddLine.Styler(func(s *plot.Style) { +ln := plots.NewLine(data).Styler(func(s *plot.Style) { s.Plot.Title = "My Plot" // overall Plot styles s.Line.Color = colors.Uniform(colors.Red) // line-specific styles }) @@ -76,7 +80,7 @@ To create a plot with multiple error bars, multiple Bar Plotters are created, wi ### XFill, YFill -`XFill` and `YFill` are used to draw filled regions between pairs of X or Y points, using the `X`, `Y`, and `BoxLow`, `BoxHigh` values to specify the center point (X, Y) and the region below / left and above / right to fill around that central point. +`XFill` and `YFill` are used to draw filled regions between pairs of X or Y points, using the `X`, `Y`, and `Low`, `High` values to specify the center point (X, Y) and the region below / left and above / right to fill around that central point. XFill along with an XY line can be used to draw the equivalent of the [matplotlib fill_between](https://matplotlib.org/stable/plot_types/basic/fill_between.html#sphx-glr-plot-types-basic-fill-between-py) plot. @@ -84,7 +88,9 @@ YFill can be used to draw the equivalent of the [matplotlib violin plot](https:/ ### Pie -TODO +`Pie` takes a list of `Y` values that are plotted as the size of segments of a circular pie plot. Y values are automatically normalized for plotting. + +TODO: implement, details on mapping, ## 2D Grid-based @@ -137,5 +143,5 @@ Here is the copyright notice for that package: # TODO -* Grid? +* Grid? in styling. From 780dd78b8b527313198623caa5434cec6a929e9b Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 11 Nov 2024 00:58:15 -0800 Subject: [PATCH 278/311] styler metadata; metadata has Metadataer interface for grabing from any obj. --- base/metadata/metadata.go | 33 +++++++++++++++++++-- plot/axis.go | 37 ++++++++++++++++++++++- plot/draw.go | 7 +++-- plot/plot.go | 5 ++++ plot/plots/bar.go | 4 ++- plot/plots/errbars.go | 2 ++ plot/plots/labels.go | 2 ++ plot/plots/table.go | 62 --------------------------------------- plot/plots/xy.go | 5 +++- plot/style.go | 25 ++++++++++++++++ plot/table.go | 14 +++++++++ plot/tick.go | 39 ++++++++++++------------ plot/typegen.go | 15 ++++++++-- 13 files changed, 159 insertions(+), 91 deletions(-) delete mode 100644 plot/plots/table.go create mode 100644 plot/table.go diff --git a/base/metadata/metadata.go b/base/metadata/metadata.go index 4517452659..d239783831 100644 --- a/base/metadata/metadata.go +++ b/base/metadata/metadata.go @@ -38,8 +38,8 @@ func (md *Data) Set(key string, value any) { (*md)[key] = value } -// Get gets metadata value of given type. -// returns error if not present or item is a different type. +// Get gets metadata value of given type from given Data. +// Returns error if not present or item is a different type. func Get[T any](md Data, key string) (T, error) { var z T x, ok := md[key] @@ -84,3 +84,32 @@ func (md *Data) SetDoc(doc string) { func (md *Data) Doc() string { return errors.Ignore1(Get[string](*md, "Doc")) } + +//////// Metadataer + +// Metadataer is an interface for a type that returns associated +// metadata.Data using a Metadata() method. +type Metadataer interface { + Metadata() *Data +} + +// GetData gets the Data from given object, if it implements the +// Metadata() method. Returns nil if it does not. +func GetData(obj any) *Data { + if md, ok := obj.(Metadataer); ok { + return md.Metadata() + } + return nil +} + +// GetFrom gets metadata value of given type from given object, +// if it implements the Metadata() method. +// Returns error if not present or item is a different type. +func GetFrom[T any](obj any, key string) (T, error) { + md := GetData(obj) + if md == nil { + var zv T + return zv, errors.New("metadata not available for given object type") + } + return Get[T](*md, key) +} diff --git a/plot/axis.go b/plot/axis.go index 56207427f4..91e02e1d4e 100644 --- a/plot/axis.go +++ b/plot/axis.go @@ -18,6 +18,21 @@ import ( "cogentcore.org/core/styles/units" ) +// AxisScales are the scaling options for how values are distributed +// along an axis: Linear, Log, etc. +type AxisScales int32 //enums:enum + +const ( + // Linear is a linear axis scale. + Linear AxisScales = iota + + // Log is a Logarithmic axis scale. + Log + + // Inverted is an inverted axis scale. + Inverted +) + // AxisStyle has style properties for the axis. type AxisStyle struct { //types:add -setters @@ -32,6 +47,13 @@ type AxisStyle struct { //types:add -setters // on the axis, thus making it easier to see. Padding units.Value + // NTicks is the desired number of ticks. + NTicks int + + // Scale specifies how values are scaled along the axis: + // Linear, Log, Inverted + Scale AxisScales + // TickText has the text style for rendering tick labels, // and is shared for actual rendering. TickText TextStyle @@ -48,6 +70,7 @@ func (ax *AxisStyle) Defaults() { ax.Text.Defaults() ax.Text.Size.Dp(20) ax.Padding.Pt(5) + ax.NTicks = 3 ax.TickText.Defaults() ax.TickText.Size.Dp(16) ax.TickText.Padding.Dp(2) @@ -104,11 +127,23 @@ func (ax *Axis) Defaults(dim math32.Dims) { ax.Ticker = DefaultTicks{} } +// drawConfig configures for drawing. +func (ax *Axis) drawConfig() { + switch ax.Style.Scale { + case Linear: + ax.Scale = LinearScale{} + case Log: + ax.Scale = LogScale{} + case Inverted: + ax.Scale = InvertedScale{} + } +} + // SanitizeRange ensures that the range of the axis makes sense. func (ax *Axis) SanitizeRange() { ax.Range.Sanitize() if ax.AutoRescale { - marks := ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max) + marks := ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max, ax.Style.NTicks) for _, t := range marks { ax.Range.FitValInRange(t.Value) } diff --git a/plot/draw.go b/plot/draw.go index 52dd9b836e..e3c69332df 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -55,6 +55,9 @@ func (pt *Plot) SVGToFile(filename string) error { func (pt *Plot) drawConfig() { pt.Resize(pt.Size) // ensure pt.applyStyle() + pt.X.drawConfig() + pt.Y.drawConfig() + pt.Z.drawConfig() pt.Paint.ToDots() } @@ -139,7 +142,7 @@ func (ax *Axis) sizeX(pt *Plot, axw float32) (ht, lpad, rpad int) { pc := pt.Paint uc := &pc.UnitContext ax.Style.TickLength.ToDots(uc) - ax.ticks = ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max) + ax.ticks = ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max, ax.Style.NTicks) h := float32(0) if ax.Label.Text != "" { // We assume that the label isn't rotated. ax.Label.Config(pt) @@ -231,7 +234,7 @@ func (ax *Axis) longestTickLabel() string { func (ax *Axis) sizeY(pt *Plot) (ywidth, tickWidth, tpad, bpad int) { pc := pt.Paint uc := &pc.UnitContext - ax.ticks = ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max) + ax.ticks = ax.Ticker.Ticks(ax.Range.Min, ax.Range.Max, ax.Style.NTicks) ax.Style.TickLength.ToDots(uc) w := float32(0) diff --git a/plot/plot.go b/plot/plot.go index aaada176be..d2069b469d 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -39,6 +39,10 @@ type XAxisStyle struct { //types:add -setters // Range is the effective range of XAxis data to plot, where either end can be fixed. Range minmax.Range64 `display:"inline"` + + // Scale specifies how values are scaled along the X axis: + // Linear, Log, Inverted + Scale AxisScales } // PlotStyle has overall plot level styling properties. @@ -212,6 +216,7 @@ func (pt *Plot) applyStyle() { pt.Legend.Style = pt.Style.Legend pt.Legend.Style.Text.openFont(pt) pt.X.Style = pt.Style.Axis + pt.X.Style.Scale = pt.Style.XAxis.Scale pt.Y.Style = pt.Style.Axis if pt.Style.XAxis.Label != "" { pt.X.Label.Text = pt.Style.XAxis.Label diff --git a/plot/plots/bar.go b/plot/plots/bar.go index f1d669263a..806d8d31ec 100644 --- a/plot/plots/bar.go +++ b/plot/plots/bar.go @@ -61,10 +61,11 @@ type Bar struct { stylers plot.Stylers } -// NewBar returns a new bar chart with a single bar for each value. +// NewBar returns a new bar plotter with a single bar for each value. // The bars heights correspond to the values and their x locations correspond // to the index of their value in the Valuer. // Optional error-bar values can be provided using the High data role. +// Styler functions are obtained from the Y metadata if present. func NewBar(data plot.Data) *Bar { if data.CheckLengths() != nil { return nil @@ -74,6 +75,7 @@ func NewBar(data plot.Data) *Bar { if bc.Y == nil { return nil } + bc.stylers = plot.GetStylersFromData(data, plot.Y) bc.Err = plot.CopyRole(data, plot.High) bc.Defaults() return bc diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index ae806de37b..21266e445d 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -49,6 +49,7 @@ func (eb *YErrorBars) Defaults() { // NewYErrorBars returns a new YErrorBars plotter, // using Low, High data roles for error deviations around X, Y coordinates. +// Styler functions are obtained from the High data if present. func NewYErrorBars(data plot.Data) *YErrorBars { if data.CheckLengths() != nil { return nil @@ -61,6 +62,7 @@ func NewYErrorBars(data plot.Data) *YErrorBars { if eb.X == nil || eb.Y == nil || eb.Low == nil || eb.High == nil { return nil } + eb.stylers = plot.GetStylersFromData(data, plot.High) eb.Defaults() return eb } diff --git a/plot/plots/labels.go b/plot/plots/labels.go index a16b9c7a7b..e1fd7078ed 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -39,6 +39,7 @@ type Labels struct { } // NewLabels returns a new Labels using defaults +// Styler functions are obtained from the Label metadata if present. func NewLabels(data plot.Data) *Labels { if data.CheckLengths() != nil { return nil @@ -53,6 +54,7 @@ func NewLabels(data plot.Data) *Labels { if ld == nil { return nil } + lb.stylers = plot.GetStylersFromData(data, plot.Label) lb.Labels = make(plot.Labels, lb.X.Len()) for i := range ld.Len() { lb.Labels[i] = ld.String1D(i) diff --git a/plot/plots/table.go b/plot/plots/table.go deleted file mode 100644 index 6403edcbb0..0000000000 --- a/plot/plots/table.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package plots - -// Table is an interface for tabular data for plotting, -// with columns of values. -type Table interface { - // number of columns of data - NumColumns() int - - // name of given column - ColumnName(i int) string - - // number of rows of data - NumRows() int - - // PlotData returns the data value at given column and row - PlotData(column, row int) float32 -} - -func TableColumnByIndex(tab Table, name string) int { - for i := range tab.NumColumns() { - if tab.ColumnName(i) == name { - return i - } - } - return -1 -} - -// // TableXYer is an interface for providing XY access to Table data -// type TableXYer struct { -// Table Table -// -// // the indexes of the tensor columns to use for the X and Y data, respectively -// XColumn, YColumn int -// } -// -// func NewTableXYer(tab Table, xcolumn, ycolumn int) *TableXYer { -// txy := &TableXYer{Table: tab, XColumn: xcolumn, YColumn: ycolumn} -// return txy -// } -// -// func (dt *TableXYer) Len() int { -// return dt.Table.NumRows() -// } -// -// func (dt *TableXYer) XY(i int) (x, y float32) { -// return dt.Table.PlotData(dt.XColumn, i), dt.Table.PlotData(dt.YColumn, i) -// } -// -// // AddTableLine adds XY Line with given x, y columns from given tabular data. -// func AddTableLine(plt *plot.Plot, tab Table, xcolumn, ycolumn int) *XY { -// txy := NewTableXYer(tab, xcolumn, ycolumn) -// ln := NewLine(txy) -// if ln == nil { -// return nil -// } -// plt.Add(ln) -// return ln -// } diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 31b6c4d717..b018af27fc 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -40,7 +40,9 @@ type XY struct { stylers plot.Stylers } -// NewXY returns an XY plot. +// NewXY returns an XY plotter for given X, Y data. +// data can also include Color and / or Size for the points. +// Styler functions are obtained from the Y metadata if present. func NewXY(data plot.Data) *XY { if data.CheckLengths() != nil { return nil @@ -51,6 +53,7 @@ func NewXY(data plot.Data) *XY { if ln.X == nil || ln.Y == nil { return nil } + ln.stylers = plot.GetStylersFromData(data, plot.Y) ln.Color = plot.CopyRole(data, plot.Color) ln.Size = plot.CopyRole(data, plot.Size) ln.Defaults() diff --git a/plot/style.go b/plot/style.go index f09ac265a6..3b3f0019ad 100644 --- a/plot/style.go +++ b/plot/style.go @@ -5,6 +5,7 @@ package plot import ( + "cogentcore.org/core/base/metadata" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/styles/units" ) @@ -125,6 +126,30 @@ func (st *Stylers) NewStyle(ps *PlotStyle) *Style { return s } +// SetStylers sets the [Stylers] into given [metadata.Data]. +func SetStylers(md *metadata.Data, st Stylers) { + md.Set("PlotStylers", st) +} + +// GetStylers gets the [Stylers] from given [metadata.Data] (nil if none). +func GetStylers(md *metadata.Data) Stylers { + st, _ := metadata.Get[Stylers](*md, "PlotStylers") + return st +} + +// GetStylersFromData returns [Stylers] from given role +// in given [Data]. nil if not present. +func GetStylersFromData(data Data, role Roles) Stylers { + vr, ok := data[role] + if !ok { + return nil + } + st, _ := metadata.GetFrom[Stylers](vr, "PlotStylers") + return st +} + +//////// + // DefaultOffOn specifies whether to use the default value for a bool option, // or to override the default and set Off or On. type DefaultOffOn int32 //enums:enum diff --git a/plot/table.go b/plot/table.go new file mode 100644 index 0000000000..1e4ae008ec --- /dev/null +++ b/plot/table.go @@ -0,0 +1,14 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package plot + +import "cogentcore.org/core/tensor/table" + +// NewTablePlot returns a new Plot with all configuration based on given +// [table.Table] set of columns and associated metadata. +func NewTablePlot(dt *table.Table) *Plot { + // todo: go through columns, find what is styled as On, etc + return nil +} diff --git a/plot/tick.go b/plot/tick.go index 450d4ade50..3be73dfc27 100644 --- a/plot/tick.go +++ b/plot/tick.go @@ -27,8 +27,9 @@ func (tk *Tick) IsMinor() bool { // Ticker creates Ticks in a specified range type Ticker interface { - // Ticks returns Ticks in a specified range - Ticks(min, max float64) []Tick + // Ticks returns Ticks in a specified range, with desired number of ticks, + // which can be ignored depending on the ticker type. + Ticks(min, max float64, nticks int) []Tick } // DefaultTicks is suitable for the Ticker field of an Axis, @@ -38,14 +39,12 @@ type DefaultTicks struct{} var _ Ticker = DefaultTicks{} // Ticks returns Ticks in the specified range. -func (DefaultTicks) Ticks(min, max float64) []Tick { +func (DefaultTicks) Ticks(min, max float64, nticks int) []Tick { if max <= min { panic("illegal range") } - const suggestedTicks = 3 - - labels, step, q, mag := talbotLinHanrahan(min, max, suggestedTicks, withinData, nil, nil, nil) + labels, step, q, mag := talbotLinHanrahan(min, max, nticks, withinData, nil, nil, nil) majorDelta := step * math.Pow10(mag) if q == 0 { // Simple fall back was chosen, so @@ -138,7 +137,7 @@ type LogTicks struct { var _ Ticker = LogTicks{} // Ticks returns Ticks in a specified range -func (t LogTicks) Ticks(min, max float64) []Tick { +func (t LogTicks) Ticks(min, max float64, nticks int) []Tick { if min <= 0 || max <= 0 { panic("Values must be greater than 0 for a log scale.") } @@ -167,7 +166,7 @@ type ConstantTicks []Tick var _ Ticker = ConstantTicks{} // Ticks returns Ticks in a specified range -func (ts ConstantTicks) Ticks(float64, float64) []Tick { +func (ts ConstantTicks) Ticks(float64, float64, int) []Tick { return ts } @@ -199,7 +198,7 @@ type TimeTicks struct { var _ Ticker = TimeTicks{} // Ticks implements plot.Ticker. -func (t TimeTicks) Ticks(min, max float64) []Tick { +func (t TimeTicks) Ticks(min, max float64, nticks int) []Tick { if t.Ticker == nil { t.Ticker = DefaultTicks{} } @@ -210,7 +209,7 @@ func (t TimeTicks) Ticks(min, max float64) []Tick { t.Time = UTCUnixTime } - ticks := t.Ticker.Ticks(min, max) + ticks := t.Ticker.Ticks(min, max, nticks) for i := range ticks { tick := &ticks[i] if tick.Label == "" { @@ -272,13 +271,13 @@ func formatFloatTick(v float64, prec int) string { return strconv.FormatFloat(float64(v), 'g', prec, 64) } -// TickerFunc is suitable for the Ticker field of an Axis. -// It is an adapter which allows to quickly setup a Ticker using a function with an appropriate signature. -type TickerFunc func(min, max float64) []Tick - -var _ Ticker = TickerFunc(nil) - -// Ticks implements plot.Ticker. -func (f TickerFunc) Ticks(min, max float64) []Tick { - return f(min, max) -} +// // TickerFunc is suitable for the Ticker field of an Axis. +// // It is an adapter which allows to quickly setup a Ticker using a function with an appropriate signature. +// type TickerFunc func(min, max float64) []Tick +// +// var _ Ticker = TickerFunc(nil) +// +// // Ticks implements plot.Ticker. +// func (f TickerFunc) Ticks(min, max float64) []Tick { +// return f(min, max) +// } diff --git a/plot/typegen.go b/plot/typegen.go index c309400383..eca23e7358 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -11,7 +11,9 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisScales", IDName: "axis-scales", Doc: "AxisScales are the scaling options for how values are distributed\nalong an axis: Linear, Log, etc."}) + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "NTicks", Doc: "NTicks is the desired number of ticks"}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the axis:\nLinear, Log, Inverted"}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) // SetText sets the [AxisStyle.Text]: // Text has the text style parameters for the text label. @@ -27,6 +29,15 @@ func (t *AxisStyle) SetLine(v LineStyle) *AxisStyle { t.Line = v; return t } // on the axis, thus making it easier to see. func (t *AxisStyle) SetPadding(v units.Value) *AxisStyle { t.Padding = v; return t } +// SetNTicks sets the [AxisStyle.NTicks]: +// NTicks is the desired number of ticks +func (t *AxisStyle) SetNTicks(v int) *AxisStyle { t.NTicks = v; return t } + +// SetScale sets the [AxisStyle.Scale]: +// Scale specifies how values are scaled along the axis: +// Linear, Log, Inverted +func (t *AxisStyle) SetScale(v AxisScales) *AxisStyle { t.Scale = v; return t } + // SetTickText sets the [AxisStyle.TickText]: // TickText has the text style for rendering tick labels, // and is shared for actual rendering. @@ -224,7 +235,7 @@ func (t *PlotStyle) SetBarWidth(v float64) *PlotStyle { t.BarWidth = v; return t var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plot", IDName: "plot", Doc: "Plot is the basic type representing a plot.\nIt renders into its own image.RGBA Pixels image,\nand can also save a corresponding SVG version.", Fields: []types.Field{{Name: "Title", Doc: "Title of the plot"}, {Name: "Style", Doc: "Style has the styling properties for the plot."}, {Name: "StandardTextStyle", Doc: "standard text style with default options"}, {Name: "X", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Y", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Z", Doc: "X, Y, and Z are the horizontal, vertical, and depth axes\nof the plot respectively."}, {Name: "Legend", Doc: "Legend is the plot's legend."}, {Name: "Plotters", Doc: "plotters are drawn by calling their Plot method\nafter the axes are drawn."}, {Name: "Size", Doc: "size is the target size of the image to render to"}, {Name: "DPI", Doc: "DPI is the dots per inch for rendering the image.\nLarger numbers result in larger scaling of the plot contents\nwhich is strongly recommended for print (e.g., use 300 for print)"}, {Name: "Paint", Doc: "painter for rendering"}, {Name: "Pixels", Doc: "pixels that we render into"}, {Name: "PlotBox", Doc: "Current plot bounding box in image coordinates, for plotting coordinates"}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint.", Args: []string{"pt"}}, {Name: "UpdateRange", Doc: "UpdateRange updates the given ranges.", Args: []string{"plt", "x", "y", "z"}}, {Name: "Data", Doc: "Data returns the data by roles for this plot, for both the original\ndata and the pixel-transformed X,Y coordinates for that data.\nThis allows a GUI interface to inspect data etc.", Returns: []string{"data", "pixX", "pixY"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDName: "plotter", Doc: "Plotter is an interface that wraps the Plot method.\nStandard implementations of Plotter are in the [plots] package.", Methods: []types.Method{{Name: "Plot", Doc: "Plot draws the data to the Plot Paint.", Args: []string{"pt"}}, {Name: "UpdateRange", Doc: "UpdateRange updates the given ranges.", Args: []string{"plt", "xr", "yr", "zr"}}, {Name: "Data", Doc: "Data returns the data by roles for this plot, for both the original\ndata and the pixel-transformed X,Y coordinates for that data.\nThis allows a GUI interface to inspect data etc.", Returns: []string{"data", "pixX", "pixY"}}, {Name: "Stylers", Doc: "Stylers returns the styler functions for this element.", Returns: []string{"Stylers"}}, {Name: "ApplyStyle", Doc: "ApplyStyle applies any stylers to this element,\nfirst initializing from the given global plot style, which has\nalready been styled with defaults and all the plot element stylers.", Args: []string{"plotStyle"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotterType", IDName: "plotter-type", Doc: "PlotterType registers a Plotter so that it can be created with appropriate data.", Fields: []types.Field{{Name: "Name", Doc: "Name of the plot type."}, {Name: "Doc", Doc: "Doc is the documentation for this Plotter."}, {Name: "Required", Doc: "Required Data roles for this plot. Data for these Roles must be provided."}, {Name: "Optional", Doc: "Optional Data roles for this plot."}, {Name: "New", Doc: "New returns a new plotter of this type with given data in given roles."}}}) From 58000e08740bdf5ffdc5636c2fea4e5812d381e3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 11 Nov 2024 11:58:13 -0800 Subject: [PATCH 279/311] tensor-attached plot stylers example, tests etc. --- base/metadata/metadata.go | 11 ++++++++ math32/minmax/range.go | 12 ++++++--- plot/README.md | 33 +++++++++++++++++++++++ plot/draw.go | 2 +- plot/plot.go | 1 - plot/plots/plot_test.go | 57 ++++++++++++++++++++++++++++++++++----- plot/style.go | 5 ++++ 7 files changed, 109 insertions(+), 12 deletions(-) diff --git a/base/metadata/metadata.go b/base/metadata/metadata.go index d239783831..9eeea554fc 100644 --- a/base/metadata/metadata.go +++ b/base/metadata/metadata.go @@ -113,3 +113,14 @@ func GetFrom[T any](obj any, key string) (T, error) { } return Get[T](*md, key) } + +// SetTo sets metadata value on given object, if it implements +// the Metadata() method. Returns error if no Metadata on object. +func SetTo(obj any, key string, value any) error { + md := GetData(obj) + if md == nil { + return errors.New("metadata not available for given object type") + } + md.Set(key, value) + return nil +} diff --git a/math32/minmax/range.go b/math32/minmax/range.go index 83b92a2850..84e7c159e4 100644 --- a/math32/minmax/range.go +++ b/math32/minmax/range.go @@ -16,15 +16,17 @@ type Range32 struct { } // SetMin sets a fixed min value -func (rr *Range32) SetMin(mn float32) { +func (rr *Range32) SetMin(mn float32) *Range32 { rr.FixMin = true rr.Min = mn + return rr } // SetMax sets a fixed max value -func (rr *Range32) SetMax(mx float32) { +func (rr *Range32) SetMax(mx float32) *Range32 { rr.FixMax = true rr.Max = mx + return rr } // Range returns Max - Min @@ -59,15 +61,17 @@ type Range64 struct { } // SetMin sets a fixed min value -func (rr *Range64) SetMin(mn float64) { +func (rr *Range64) SetMin(mn float64) *Range64 { rr.FixMin = true rr.Min = mn + return rr } // SetMax sets a fixed max value -func (rr *Range64) SetMax(mx float64) { +func (rr *Range64) SetMax(mx float64) *Range64 { rr.FixMax = true rr.Max = mx + return rr } // Range returns Max - Min diff --git a/plot/README.md b/plot/README.md index c43b0ce0e4..080dacb35e 100644 --- a/plot/README.md +++ b/plot/README.md @@ -46,6 +46,39 @@ The `Plot` field (of type `PlotStyle`) contains all the properties that apply to The rest of the style properties (e.g., `Line`, `Point`) apply to the element in question. There are also some default plot-level settings in `Plot` that apply to all elements, and the plot-level styles are updated first, so in this way it is possible to have plot-wide settings applied from one styler, that affect all plots (e.g., the line width, and whether lines and / or points are plotted or not). +## Tensor metadata + +Styler functions can be attached directly to a `tensor.Tensor` via its metadata, and the `Plotter` elements will automatically grab these functions from any data source that has such metadata set. This allows the data generator to directly set default styling parameters, which can always be overridden later by adding more styler functions. Tying the plot styling directly to the source data allows all of the relevant logic to be put in one place, instead of spreading this logic across different places in the code. + +Here is an example of how this works: + +```Go + tx, ty := tensor.NewFloat64(21), tensor.NewFloat64(21) + for i := range tx.DimSize(0) { + tx.SetFloat1D(float64(i*5), i) + ty.SetFloat1D(50.0+40*math.Sin((float64(i)/8)*math.Pi), i) + } + // attach stylers to the Y axis data: that is where plotter looks for it + plot.SetStylersTo(ty, plot.Stylers{func(s *plot.Style) { + s.Plot.Title = "Test Line" + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plot.Scale = 2 + s.Plot.XAxis.Range.SetMax(105) + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) + s.Line.Color = colors.Uniform(colors.Red) + s.Point.Color = colors.Uniform(colors.Blue) + s.Range.SetMin(0).SetMax(100) + }}) + + // somewhere else in the code: + + plt := plot.New() + // NewLine automatically gets stylers from ty tensor metadata + plt.Add(plots.NewLine(plot.Data{plot.X: tx, plot.Y: ty})) + plt.Draw() +``` + # Plot Types The following are the builtin standard plot types, in the `plots` package: diff --git a/plot/draw.go b/plot/draw.go index e3c69332df..2a015bd9ea 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -53,8 +53,8 @@ func (pt *Plot) SVGToFile(filename string) error { // drawConfig configures everything for drawing, applying styles etc. func (pt *Plot) drawConfig() { - pt.Resize(pt.Size) // ensure pt.applyStyle() + pt.Resize(pt.Size) // ensure pt.X.drawConfig() pt.Y.drawConfig() pt.Z.drawConfig() diff --git a/plot/plot.go b/plot/plot.go index d2069b469d..d6d06b3cad 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -214,7 +214,6 @@ func (pt *Plot) applyStyle() { pt.Title.Text = pt.Style.Title } pt.Legend.Style = pt.Style.Legend - pt.Legend.Style.Text.openFont(pt) pt.X.Style = pt.Style.Axis pt.X.Style.Scale = pt.Style.XAxis.Scale pt.Y.Style = pt.Style.Axis diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 6b588292e1..d335c2656a 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -15,13 +15,14 @@ import ( "cogentcore.org/core/colors" "cogentcore.org/core/paint" "cogentcore.org/core/plot" + "cogentcore.org/core/tensor" ) func ExampleLine() { xd, yd := make(plot.Values, 21), make(plot.Values, 21) for i := range xd { xd[i] = float64(i * 5) - yd[i] = float64(50) + 40*math.Sin((float64(i)/8)*math.Pi) + yd[i] = 50.0 + 40*math.Sin((float64(i)/8)*math.Pi) } data := plot.Data{plot.X: xd, plot.Y: yd} plt := plot.New() @@ -40,13 +41,42 @@ func ExampleLine() { s.Plot.XAxis.Rotation = -45 s.Line.Color = colors.Uniform(colors.Red) s.Point.Color = colors.Uniform(colors.Blue) - s.Range.SetMax(100) + s.Range.SetMin(0).SetMax(100) })) plt.Draw() imagex.Save(plt.Pixels, "testdata/ex_line_plot.png") // Output: } +func ExampleStylerMetadata() { + tx, ty := tensor.NewFloat64(21), tensor.NewFloat64(21) + for i := range tx.DimSize(0) { + tx.SetFloat1D(float64(i*5), i) + ty.SetFloat1D(50.0+40*math.Sin((float64(i)/8)*math.Pi), i) + } + // attach stylers to the Y axis data: that is where plotter looks for it + plot.SetStylersTo(ty, plot.Stylers{func(s *plot.Style) { + s.Plot.Title = "Test Line" + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plot.Scale = 2 + s.Plot.XAxis.Range.SetMax(105) + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) + s.Line.Color = colors.Uniform(colors.Red) + s.Point.Color = colors.Uniform(colors.Blue) + s.Range.SetMin(0).SetMax(100) + }}) + + // somewhere else in the code: + + plt := plot.New() + // NewLine automatically gets stylers from ty tensor metadata + plt.Add(NewLine(plot.Data{plot.X: tx, plot.Y: ty})) + plt.Draw() + imagex.Save(plt.Pixels, "testdata/ex_styler_metadata.png") + // Output: +} + func TestMain(m *testing.M) { paint.FontLibrary.InitFontPaths(paint.FontPaths...) os.Exit(m.Run()) @@ -359,8 +389,7 @@ func TestErrBar(t *testing.T) { func TestStyle(t *testing.T) { data := sinCosWrapData() - plt := plot.New() - l1 := NewLine(data).Styler(func(s *plot.Style) { + stf := func(s *plot.Style) { s.Plot.Title = "Test Line" s.Plot.XAxis.Label = "X Axis" s.Plot.YAxisLabel = "Y Axis" @@ -377,12 +406,28 @@ func TestStyle(t *testing.T) { s.Line.Color = colors.Uniform(colors.Red) s.Point.Color = colors.Uniform(colors.Blue) s.Range.SetMax(100) - }) + } + + plt := plot.New() + l1 := NewLine(data).Styler(stf) plt.Add(l1) - plt.Legend.Add("Sine", l1) + plt.Legend.Add("Sine", l1) // todo: auto-add! plt.Legend.Add("Cos", l1) plt.Resize(image.Point{640, 480}) plt.Draw() imagex.Assert(t, plt.Pixels, "style_line_point.png") + + plt = plot.New() + tdy := tensor.NewFloat64FromValues(data[plot.Y].(plot.Values)...) + plot.SetStylersTo(tdy, plot.Stylers{stf}) // set metadata for tensor + tdx := tensor.NewFloat64FromValues(data[plot.X].(plot.Values)...) + // NewLine auto-grabs from Y metadata + l1 = NewLine(plot.Data{plot.X: tdx, plot.Y: tdy}) + plt.Add(l1) + plt.Legend.Add("Sine", l1) // todo: auto-add! + plt.Legend.Add("Cos", l1) + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Assert(t, plt.Pixels, "style_line_point_auto.png") } diff --git a/plot/style.go b/plot/style.go index 3b3f0019ad..062bdb2e4d 100644 --- a/plot/style.go +++ b/plot/style.go @@ -137,6 +137,11 @@ func GetStylers(md *metadata.Data) Stylers { return st } +// SetStylersTo sets the [Stylers] into given object that has metadata. +func SetStylersTo(obj any, st Stylers) { + metadata.SetTo(obj, "PlotStylers", st) +} + // GetStylersFromData returns [Stylers] from given role // in given [Data]. nil if not present. func GetStylersFromData(data Data, role Roles) Stylers { From 3c88bf99238e9de1059b67427b8da0b56332d0bd Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 11 Nov 2024 12:50:33 -0800 Subject: [PATCH 280/311] plot: start on table based plotting -- not working --- plot/plots/plot_test.go | 41 +++++++++++++++++++++++++++ plot/plots/xy.go | 7 ++++- plot/plotter.go | 21 ++++++++++---- plot/style.go | 18 +++++++++--- plot/table.go | 62 ++++++++++++++++++++++++++++++++++++++--- 5 files changed, 135 insertions(+), 14 deletions(-) diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index d335c2656a..1ea4105aef 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -16,6 +16,8 @@ import ( "cogentcore.org/core/paint" "cogentcore.org/core/plot" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" + "github.com/stretchr/testify/assert" ) func ExampleLine() { @@ -431,3 +433,42 @@ func TestStyle(t *testing.T) { plt.Draw() imagex.Assert(t, plt.Pixels, "style_line_point_auto.png") } + +// todo: move into statplot and test everything + +func TestTable(t *testing.T) { + tx, ty := tensor.NewFloat64(21), tensor.NewFloat64(21) + for i := range tx.DimSize(0) { + tx.SetFloat1D(float64(i*5), i) + ty.SetFloat1D(50.0+40*math.Sin((float64(i)/8)*math.Pi), i) + } + // attach stylers to the Y axis data: that is where plotter looks for it + plot.SetStylersTo(ty, plot.Stylers{func(s *plot.Style) { + s.Plot.Title = "Test Line" + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plot.Scale = 2 + s.Plot.XAxis.Range.SetMax(105) + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) + s.On = plot.On + s.Role = plot.Y + s.Line.Color = colors.Uniform(colors.Red) + s.Point.Color = colors.Uniform(colors.Blue) + s.Range.SetMin(0).SetMax(100) + }}) + + plot.SetStylersTo(tx, plot.Stylers{func(s *plot.Style) { + s.Role = plot.X + }}) + + dt := table.New("Test Table") // todo: use Name by default for plot. + dt.AddColumn("X", tx) + dt.AddColumn("Y", ty) + + plt, err := plot.NewTablePlot(dt) + assert.NoError(t, err) + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Save(plt.Pixels, "testdata/table_xy.png") + // Output: +} diff --git a/plot/plots/xy.go b/plot/plots/xy.go index b018af27fc..9e77c794fc 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -12,6 +12,8 @@ package plots //go:generate core generate import ( + "fmt" + "cogentcore.org/core/math32" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" @@ -92,7 +94,10 @@ func (ln *XY) Styler(f func(s *plot.Style)) *XY { return ln } -func (ln *XY) Stylers() *plot.Stylers { return &ln.stylers } +func (ln *XY) Stylers() *plot.Stylers { + fmt.Println("current ln", ln) + return &ln.stylers +} func (ln *XY) ApplyStyle(ps *plot.PlotStyle) { ps.SetElementStyle(&ln.Style) diff --git a/plot/plotter.go b/plot/plotter.go index 68c41cc231..e0e61d09fd 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -5,8 +5,9 @@ package plot import ( - "log/slog" + "fmt" + "cogentcore.org/core/base/errors" "cogentcore.org/core/math32/minmax" ) @@ -62,13 +63,23 @@ func RegisterPlotter(name, doc string, required, optional []Roles, newFun func(d Plotters[name] = PlotterType{Name: name, Doc: doc, Required: required, Optional: optional, New: newFun} } +// PlotterByType returns [PlotterType] info for a registered [Plotter] +// of given type name, e.g., "XY", "Bar" etc, +// Returns an error and nil if type name is not a registered type. +func PlotterByType(typeName string) (*PlotterType, error) { + pt, ok := Plotters[typeName] + if !ok { + return nil, fmt.Errorf("plot.PlotterByType type name is not registered: %s", typeName) + } + return &pt, nil +} + // NewPlotter returns a new plotter of given type, e.g., "XY", "Bar" etc, // for given data roles (which must include Required roles, and may include Optional ones). -// Logs an error and returns nil if type name is not recognized as a registered type. +// Logs an error and returns nil if type name is not a registered type. func NewPlotter(typeName string, data Data) Plotter { - pt, ok := Plotters[typeName] - if !ok { - slog.Error("plot.NewPlotter type name is not registered", "typeName", typeName) + pt, err := PlotterByType(typeName) + if errors.Log(err) != nil { return nil } return pt.New(data) diff --git a/plot/style.go b/plot/style.go index 062bdb2e4d..529586ff1a 100644 --- a/plot/style.go +++ b/plot/style.go @@ -19,9 +19,13 @@ type Style struct { //types:add -setters // plot element, and are updated first, before applying element-wise styles. Plot PlotStyle - // On specifies whether to plot this item, for cases where it can be turned off. + // On specifies whether to plot this item, for table-based plots. On DefaultOffOn + // Plotter is the type of plotter to use in plotting this data, + // for table-based plots. Blank means use default ([plots.XY] is overall default). + Plotter string + // Role specifies a role for this item, used for table-based plots to indicate // how a particular column of data should be used. Role Roles @@ -137,11 +141,18 @@ func GetStylers(md *metadata.Data) Stylers { return st } -// SetStylersTo sets the [Stylers] into given object that has metadata. +// SetStylersTo sets the [Stylers] into given object's [metadata]. func SetStylersTo(obj any, st Stylers) { metadata.SetTo(obj, "PlotStylers", st) } +// GetStylersFrom returns [Stylers] from given object's [metadata]. +// Returns nil if none or no metadata. +func GetStylersFrom(obj any) Stylers { + st, _ := metadata.GetFrom[Stylers](obj, "PlotStylers") + return st +} + // GetStylersFromData returns [Stylers] from given role // in given [Data]. nil if not present. func GetStylersFromData(data Data, role Roles) Stylers { @@ -149,8 +160,7 @@ func GetStylersFromData(data Data, role Roles) Stylers { if !ok { return nil } - st, _ := metadata.GetFrom[Stylers](vr, "PlotStylers") - return st + return GetStylersFrom(vr) } //////// diff --git a/plot/table.go b/plot/table.go index 1e4ae008ec..9869b2200b 100644 --- a/plot/table.go +++ b/plot/table.go @@ -4,11 +4,65 @@ package plot -import "cogentcore.org/core/tensor/table" +import ( + "fmt" + "log/slog" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" +) // NewTablePlot returns a new Plot with all configuration based on given // [table.Table] set of columns and associated metadata. -func NewTablePlot(dt *table.Table) *Plot { - // todo: go through columns, find what is styled as On, etc - return nil +// Only columns marked as On are plotted. +// Must have a column with a Role = X (first one found is used). +func NewTablePlot(dt *table.Table) (*Plot, error) { + var xt tensor.Values + // var xst *Style + for _, cl := range dt.Columns.Values { + st := &Style{} + stl := GetStylersFrom(cl) + if stl == nil { + continue + } + stl.Run(st) + if st.Role == X { + xt = cl + // xst = st + break + } + } + if xt == nil { + return nil, errors.New("plot.NewTablePlot: X axis (Style.Role = X) not found") + } + plt := New() + for _, cl := range dt.Columns.Values { + st := &Style{} + stl := GetStylersFrom(cl) + if stl == nil { + continue + } + stl.Run(st) + if st.On != On || st.Role == X { + continue + } + ptyp := "XY" + if st.Plotter != "" { + ptyp = st.Plotter + } + pt, err := PlotterByType(ptyp) + if errors.Log(err) != nil { + continue + } + // todo: collect all roles from group + pl := pt.New(Data{X: xt, st.Role: cl}) + if pl != nil { + fmt.Println("adding pl", pl) + plt.Add(pl) + } else { + slog.Error("plot.NewTablePlot: error in creating plotter of type:", ptyp) + } + } + return plt, nil } From c293888e8682828f1fb63ef587b2df4e6c7dd65a Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 11 Nov 2024 16:06:07 -0800 Subject: [PATCH 281/311] table plot working for each plot type; fix label plotting; fix minmax to use +/-inf instead of MaxFloat; --- math32/minmax/avgmax.go | 7 ++- math32/minmax/minmax32.go | 6 +- math32/minmax/minmax64.go | 13 ++--- plot/data.go | 3 +- plot/plot_test.go | 6 +- plot/plots/bar.go | 2 +- plot/plots/errbars.go | 1 + plot/plots/labels.go | 28 +++++----- plot/plots/plot_test.go | 115 +++++++++++++++++++++++++++----------- plot/plots/xy.go | 7 +-- plot/plotter.go | 5 +- plot/table.go | 102 +++++++++++++++++++++++++-------- tensor/table/columns.go | 4 ++ 13 files changed, 198 insertions(+), 101 deletions(-) diff --git a/math32/minmax/avgmax.go b/math32/minmax/avgmax.go index a89e914908..72fb1dbe7a 100644 --- a/math32/minmax/avgmax.go +++ b/math32/minmax/avgmax.go @@ -4,7 +4,10 @@ package minmax -import "fmt" +import ( + "fmt" + "math" +) //gosl:start @@ -114,7 +117,7 @@ func (am *AvgMax64) Init() { am.Avg = 0 am.Sum = 0 am.N = 0 - am.Max = -MaxFloat64 + am.Max = math.Inf(-1) am.MaxIndex = -1 } diff --git a/math32/minmax/minmax32.go b/math32/minmax/minmax32.go index d7c96b742f..be1d27b4a8 100644 --- a/math32/minmax/minmax32.go +++ b/math32/minmax/minmax32.go @@ -28,10 +28,10 @@ func (mr *F32) Set(mn, mx float32) { } // SetInfinity sets the Min to +MaxFloat, Max to -MaxFloat -- suitable for -// iteratively calling Fit*InRange +// iteratively calling Fit*InRange. See also Sanitize when done. func (mr *F32) SetInfinity() { - mr.Min = MaxFloat32 - mr.Max = -MaxFloat32 + mr.Min = math32.Inf(1) + mr.Max = math32.Inf(-1) } // IsValid returns true if Min <= Max diff --git a/math32/minmax/minmax64.go b/math32/minmax/minmax64.go index 011db38328..d34c1c0750 100644 --- a/math32/minmax/minmax64.go +++ b/math32/minmax/minmax64.go @@ -9,11 +9,6 @@ import "math" //go:generate core generate -const ( - MaxFloat64 float64 = 1.7976931348623158e+308 - MinFloat64 float64 = 2.2250738585072014e-308 -) - // F64 represents a min / max range for float64 values. // Supports clipping, renormalizing, etc type F64 struct { @@ -27,11 +22,11 @@ func (mr *F64) Set(mn, mx float64) { mr.Max = mx } -// SetInfinity sets the Min to +MaxFloat, Max to -MaxFloat -- suitable for -// iteratively calling Fit*InRange. +// SetInfinity sets the Min to +Inf, Max to -Inf, suitable for +// iteratively calling Fit*InRange. See also Sanitize when done. func (mr *F64) SetInfinity() { - mr.Min = MaxFloat64 - mr.Max = -MaxFloat64 + mr.Min = math.Inf(1) + mr.Max = math.Inf(-1) } // IsValid returns true if Min <= Max. diff --git a/plot/data.go b/plot/data.go index 177ae1146e..8e3a8718c4 100644 --- a/plot/data.go +++ b/plot/data.go @@ -200,8 +200,7 @@ func MustCopyRole(data Data, role Roles) Values { slog.Error("plot Data role not present, but is required", "role:", role) return nil } - v, _ := CopyValues(d) - return v + return errors.Log1(CopyValues(d)) } // CopyRole returns Values copy of given role from given data map, diff --git a/plot/plot_test.go b/plot/plot_test.go index 8332dee986..d3c0eff9bb 100644 --- a/plot/plot_test.go +++ b/plot/plot_test.go @@ -21,11 +21,9 @@ func TestMain(m *testing.M) { func TestPlot(t *testing.T) { pt := New() pt.Title.Text = "Test Plot" - pt.X.Min = 0 - pt.X.Max = 100 + pt.X.Range.Max = 100 pt.X.Label.Text = "X Axis" - pt.Y.Min = 0 - pt.Y.Max = 100 + pt.Y.Range.Max = 100 pt.Y.Label.Text = "Y Axis" pt.Resize(image.Point{640, 480}) diff --git a/plot/plots/bar.go b/plot/plots/bar.go index 806d8d31ec..4fe1fe8238 100644 --- a/plot/plots/bar.go +++ b/plot/plots/bar.go @@ -186,7 +186,7 @@ func (bc *Bar) Plot(plt *plot.Plot) { pc.FillStrokeClear() if i < len(bc.Err) { - errval := bc.Err[i] + errval := math.Abs(bc.Err[i]) if bc.Horizontal { eVal := plt.PX(bottom + ht + math.Abs(errval)) pc.MoveTo(valMax, catVal) diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index 21266e445d..e317fd8859 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -167,6 +167,7 @@ func NewXErrorBars(data plot.Data) *XErrorBars { if eb.X == nil || eb.Y == nil || eb.Low == nil || eb.High == nil { return nil } + eb.stylers = plot.GetStylersFromData(data, plot.High) eb.Defaults() return eb } diff --git a/plot/plots/labels.go b/plot/plots/labels.go index e1fd7078ed..f9ffe84b76 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -31,7 +31,7 @@ type Labels struct { PX, PY []float32 // Style is the style of the label text. - Style plot.TextStyle + Style plot.Style // plot size and number of TextStyle when styles last generated -- don't regen styleSize image.Point @@ -54,12 +54,12 @@ func NewLabels(data plot.Data) *Labels { if ld == nil { return nil } - lb.stylers = plot.GetStylersFromData(data, plot.Label) lb.Labels = make(plot.Labels, lb.X.Len()) for i := range ld.Len() { lb.Labels[i] = ld.String1D(i) } + lb.stylers = plot.GetStylersFromData(data, plot.Label) lb.Defaults() return lb } @@ -75,8 +75,8 @@ func (lb *Labels) Styler(f func(s *plot.Style)) *Labels { } func (lb *Labels) ApplyStyle(ps *plot.PlotStyle) { - st := lb.stylers.NewStyle(ps) - lb.Style = st.Text + ps.SetElementStyle(&lb.Style) + lb.stylers.Run(&lb.Style) } func (lb *Labels) Stylers() *plot.Stylers { return &lb.stylers } @@ -96,11 +96,12 @@ func (lb *Labels) Plot(plt *plot.Plot) { pc := plt.Paint uc := &pc.UnitContext lb.PX = plot.PlotX(plt, lb.X) - lb.PY = plot.PlotX(plt, lb.Y) - lb.Style.Offset.ToDots(uc) - lb.Style.ToDots(uc) + lb.PY = plot.PlotY(plt, lb.Y) + st := &lb.Style.Text + st.Offset.ToDots(uc) + st.ToDots(uc) var ltxt plot.Text - ltxt.Style = lb.Style + ltxt.Style = *st for i, label := range lb.Labels { if label == "" { continue @@ -108,7 +109,7 @@ func (lb *Labels) Plot(plt *plot.Plot) { ltxt.Text = label ltxt.Config(plt) tht := ltxt.PaintText.BBox.Size().Y - ltxt.Draw(plt, math32.Vec2(lb.PX[i]+lb.Style.Offset.X.Dots, lb.PY[i]+lb.Style.Offset.Y.Dots-tht)) + ltxt.Draw(plt, math32.Vec2(lb.PX[i]+st.Offset.X.Dots, lb.PY[i]+st.Offset.Y.Dots-tht)) } } @@ -116,12 +117,13 @@ func (lb *Labels) Plot(plt *plot.Plot) { func (lb *Labels) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { // todo: include point sizes! plot.Range(lb.X, xr) - plot.Range(lb.Y, yr) + plot.RangeClamp(lb.Y, yr, &lb.Style.Range) pxToData := math32.FromPoint(plt.Size) pxToData.X = float32(xr.Range()) / pxToData.X pxToData.Y = float32(yr.Range()) / pxToData.Y + st := &lb.Style.Text var ltxt plot.Text - ltxt.Style = lb.Style + ltxt.Style = *st for i, label := range lb.Labels { if label == "" { continue @@ -132,8 +134,8 @@ func (lb *Labels) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { twd := 1.1 * pxToData.X * ltxt.PaintText.BBox.Size().X x := lb.X[i] y := lb.Y[i] - maxx := x + float64(pxToData.X*lb.Style.Offset.X.Dots+twd) - maxy := y + float64(pxToData.Y*lb.Style.Offset.Y.Dots+tht) // y is up here + maxx := x + float64(pxToData.X*st.Offset.X.Dots+twd) + maxy := y + float64(pxToData.Y*st.Offset.Y.Dots+tht) // y is up here xr.FitInRange(minmax.F64{x, maxx}) yr.FitInRange(minmax.F64{y, maxy}) } diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 1ea4105aef..569e3689a7 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -8,7 +8,10 @@ import ( "fmt" "image" "math" + "math/rand" "os" + "slices" + "strconv" "testing" "cogentcore.org/core/base/iox/imagex" @@ -18,6 +21,7 @@ import ( "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" "github.com/stretchr/testify/assert" + "golang.org/x/exp/maps" ) func ExampleLine() { @@ -207,7 +211,6 @@ func TestLabels(t *testing.T) { plt.X.Label.Text = "X Axis" plt.Y.Label.Text = "Y Axis" - // note: making two overlapping series xd, yd := make(plot.Values, 12), make(plot.Values, 12) labels := make(plot.Labels, 12) for i := range xd { @@ -233,8 +236,8 @@ func TestLabels(t *testing.T) { if l2 == nil { t.Error("bad data") } - l2.Style.Offset.X.Dp(6) - l2.Style.Offset.Y.Dp(-6) + l2.Style.Text.Offset.X.Dp(6) + l2.Style.Text.Offset.Y.Dp(-6) plt.Add(l2) plt.Resize(image.Point{640, 480}) @@ -437,38 +440,82 @@ func TestStyle(t *testing.T) { // todo: move into statplot and test everything func TestTable(t *testing.T) { - tx, ty := tensor.NewFloat64(21), tensor.NewFloat64(21) - for i := range tx.DimSize(0) { + rand.Seed(1) + n := 21 + tx, ty := tensor.NewFloat64(n), tensor.NewFloat64(n) + tl, th := tensor.NewFloat64(n), tensor.NewFloat64(n) + ts, tc := tensor.NewFloat64(n), tensor.NewFloat64(n) + lbls := tensor.NewString(n) + for i := range n { tx.SetFloat1D(float64(i*5), i) ty.SetFloat1D(50.0+40*math.Sin((float64(i)/8)*math.Pi), i) + tl.SetFloat1D(5*rand.Float64(), i) + th.SetFloat1D(5*rand.Float64(), i) + ts.SetFloat1D(1+5*rand.Float64(), i) + tc.SetFloat1D(float64(i), i) + lbls.SetString1D(strconv.Itoa(i), i) + } + ptyps := maps.Keys(plot.Plotters) + slices.Sort(ptyps) + for _, ttyp := range ptyps { + // attach stylers to the Y axis data: that is where plotter looks for it + genst := func(s *plot.Style) { + s.Plot.Title = "Test " + ttyp + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plotter = ttyp + s.Plot.Scale = 2 + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) + s.Line.Color = colors.Uniform(colors.Red) + s.Point.Color = colors.Uniform(colors.Blue) + s.Range.SetMin(0).SetMax(100) + } + plot.SetStylersTo(ty, plot.Stylers{genst, func(s *plot.Style) { + s.On = plot.On + s.Role = plot.Y + s.Group = "Y" + }}) + // others get basic styling + plot.SetStylersTo(tx, plot.Stylers{func(s *plot.Style) { + s.Role = plot.X + s.Group = "Y" + }}) + plot.SetStylersTo(tl, plot.Stylers{func(s *plot.Style) { + s.Role = plot.Low + s.Group = "Y" + }}) + plot.SetStylersTo(th, plot.Stylers{genst, func(s *plot.Style) { + s.On = plot.On + s.Role = plot.High + s.Group = "Y" + }}) + plot.SetStylersTo(ts, plot.Stylers{func(s *plot.Style) { + s.Role = plot.Size + s.Group = "Y" + }}) + plot.SetStylersTo(tc, plot.Stylers{func(s *plot.Style) { + s.Role = plot.Color + s.Group = "Y" + }}) + plot.SetStylersTo(lbls, plot.Stylers{genst, func(s *plot.Style) { + s.On = plot.On + s.Role = plot.Label + s.Group = "Y" + }}) + dt := table.New("Test Table") // todo: use Name by default for plot. + dt.AddColumn("X", tx) + dt.AddColumn("Y", ty) + dt.AddColumn("Low", tl) + dt.AddColumn("High", th) + dt.AddColumn("Size", ts) + dt.AddColumn("Color", tc) + dt.AddColumn("Labels", lbls) + + plt, err := plot.NewTablePlot(dt) + assert.NoError(t, err) + plt.Resize(image.Point{640, 480}) + plt.Draw() + fnm := "table_" + ttyp + ".png" + imagex.Assert(t, plt.Pixels, fnm) } - // attach stylers to the Y axis data: that is where plotter looks for it - plot.SetStylersTo(ty, plot.Stylers{func(s *plot.Style) { - s.Plot.Title = "Test Line" - s.Plot.XAxis.Label = "X Axis" - s.Plot.YAxisLabel = "Y Axis" - s.Plot.Scale = 2 - s.Plot.XAxis.Range.SetMax(105) - s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) - s.On = plot.On - s.Role = plot.Y - s.Line.Color = colors.Uniform(colors.Red) - s.Point.Color = colors.Uniform(colors.Blue) - s.Range.SetMin(0).SetMax(100) - }}) - - plot.SetStylersTo(tx, plot.Stylers{func(s *plot.Style) { - s.Role = plot.X - }}) - - dt := table.New("Test Table") // todo: use Name by default for plot. - dt.AddColumn("X", tx) - dt.AddColumn("Y", ty) - - plt, err := plot.NewTablePlot(dt) - assert.NoError(t, err) - plt.Resize(image.Point{640, 480}) - plt.Draw() - imagex.Save(plt.Pixels, "testdata/table_xy.png") - // Output: } diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 9e77c794fc..b018af27fc 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -12,8 +12,6 @@ package plots //go:generate core generate import ( - "fmt" - "cogentcore.org/core/math32" "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" @@ -94,10 +92,7 @@ func (ln *XY) Styler(f func(s *plot.Style)) *XY { return ln } -func (ln *XY) Stylers() *plot.Stylers { - fmt.Println("current ln", ln) - return &ln.stylers -} +func (ln *XY) Stylers() *plot.Stylers { return &ln.stylers } func (ln *XY) ApplyStyle(ps *plot.PlotStyle) { ps.SetElementStyle(&ln.Style) diff --git a/plot/plotter.go b/plot/plotter.go index e0e61d09fd..3d59874068 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -53,13 +53,10 @@ type PlotterType struct { } // Plotters is the registry of [Plotter] types. -var Plotters map[string]PlotterType +var Plotters = map[string]PlotterType{} // RegisterPlotter registers a plotter type. func RegisterPlotter(name, doc string, required, optional []Roles, newFun func(data Data) Plotter) { - if Plotters == nil { - Plotters = make(map[string]PlotterType) - } Plotters[name] = PlotterType{Name: name, Doc: doc, Required: required, Optional: optional, New: newFun} } diff --git a/plot/table.go b/plot/table.go index 9869b2200b..156f3e179e 100644 --- a/plot/table.go +++ b/plot/table.go @@ -6,7 +6,6 @@ package plot import ( "fmt" - "log/slog" "cogentcore.org/core/base/errors" "cogentcore.org/core/tensor" @@ -14,55 +13,112 @@ import ( ) // NewTablePlot returns a new Plot with all configuration based on given -// [table.Table] set of columns and associated metadata. -// Only columns marked as On are plotted. -// Must have a column with a Role = X (first one found is used). +// [table.Table] set of columns and associated metadata, which must have +// [Stylers] functions set (e.g., [SetStylersTo]) that at least set basic +// table parameters, including: +// - On: Set the main (typically Role = Y) column On to include in plot. +// - Role: Set the appropriate [Roles] role for this column (Y, X, etc). +// - Group: Multiple columns used for a given Plotter type must be grouped +// together with a common name (typically the name of the main Y axis), +// e.g., for Low, High error bars, Size, Color, etc. If only one On column, +// then Group can be empty and all other such columns will be grouped. +// - Plotter: Determines the type of Plotter element to use, which in turn +// determines the additional Roles that can be used within a Group. func NewTablePlot(dt *table.Table) (*Plot, error) { + nc := len(dt.Columns.Values) + if nc == 0 { + return nil, errors.New("plot.NewTablePlot: no columns in data table") + } + csty := make(map[tensor.Values]*Style, nc) + gps := make(map[string][]tensor.Values, nc) var xt tensor.Values - // var xst *Style + var errs []error for _, cl := range dt.Columns.Values { st := &Style{} + st.Defaults() stl := GetStylersFrom(cl) if stl == nil { continue } + csty[cl] = st stl.Run(st) - if st.Role == X { + gps[st.Group] = append(gps[st.Group], cl) + if xt == nil && st.Role == X { xt = cl - // xst = st - break } } - if xt == nil { - return nil, errors.New("plot.NewTablePlot: X axis (Style.Role = X) not found") - } + doneGps := map[string]bool{} plt := New() - for _, cl := range dt.Columns.Values { - st := &Style{} - stl := GetStylersFrom(cl) - if stl == nil { + for ci, cl := range dt.Columns.Values { + cnm := dt.Columns.Keys[ci] + st := csty[cl] + if st.On != On || st.Role == X { continue } - stl.Run(st) - if st.On != On || st.Role == X { + gp := st.Group + if doneGps[gp] { continue } + if gp != "" { + doneGps[gp] = true + } ptyp := "XY" if st.Plotter != "" { ptyp = st.Plotter } pt, err := PlotterByType(ptyp) - if errors.Log(err) != nil { + if err != nil { + errs = append(errs, err) continue } - // todo: collect all roles from group - pl := pt.New(Data{X: xt, st.Role: cl}) + data := Data{st.Role: cl} + gcols := gps[gp] + gotReq := true + for _, rl := range pt.Required { + if rl == st.Role { + continue + } + got := false + for _, gc := range gcols { + gst := csty[gc] + if gst.Role == rl { + data[rl] = gc + got = true + break + } + } + if !got { + if rl == X && xt != nil { + data[rl] = xt + } else { + err = fmt.Errorf("plot.NewTablePlot: Required Role %q not found in Group %q, Plotter %q not added for Column: %q", rl.String(), gp, ptyp, cnm) + errs = append(errs, err) + gotReq = false + } + } + } + if !gotReq { + continue + } + for _, rl := range pt.Optional { + if rl == st.Role { // should not happen + continue + } + for _, gc := range gcols { + gst := csty[gc] + if gst.Role == rl { + data[rl] = gc + break + } + } + } + pl := pt.New(data) if pl != nil { - fmt.Println("adding pl", pl) plt.Add(pl) } else { - slog.Error("plot.NewTablePlot: error in creating plotter of type:", ptyp) + err = fmt.Errorf("plot.NewTablePlot: error in creating plotter type: %q", ptyp) + errs = append(errs, err) } } - return plt, nil + return plt, errors.Join(errs...) } diff --git a/tensor/table/columns.go b/tensor/table/columns.go index 75a14ca5d2..8f31070df0 100644 --- a/tensor/table/columns.go +++ b/tensor/table/columns.go @@ -40,8 +40,12 @@ func (cl *Columns) SetNumRows(rows int) *Columns { //types:add // AddColumn adds the given tensor (as a [tensor.Values]) as a column, // returning an error and not adding if the name is not unique. // Automatically adjusts the shape to fit the current number of rows, +// (setting Rows if this is the first column added) // and calls the metadata SetName with column name. func (cl *Columns) AddColumn(name string, tsr tensor.Values) error { + if cl.Len() == 0 { + cl.Rows = tsr.DimSize(0) + } err := cl.Add(name, tsr) if err != nil { return err From f75d485c3d8f07d6c863b4e88f870ac71633e903 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 11 Nov 2024 17:13:17 -0800 Subject: [PATCH 282/311] error bars use Y range properly, and range more generally; Low is optional -- uses symmetric High if only that is present. --- plot/plots/errbars.go | 47 ++++++++++++++++++++++++++--------- plot/plots/labels.go | 7 +++++- plot/plots/plot_test.go | 55 +++++++++++++++++++++++++++++++++++++++++ plot/table.go | 3 ++- 4 files changed, 98 insertions(+), 14 deletions(-) diff --git a/plot/plots/errbars.go b/plot/plots/errbars.go index e317fd8859..4d37a6279e 100644 --- a/plot/plots/errbars.go +++ b/plot/plots/errbars.go @@ -20,16 +20,17 @@ const ( ) func init() { - plot.RegisterPlotter(YErrorBarsType, "draws draws vertical error bars, denoting error in Y values, using Low, High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.Low, plot.High}, []plot.Roles{}, func(data plot.Data) plot.Plotter { + plot.RegisterPlotter(YErrorBarsType, "draws draws vertical error bars, denoting error in Y values, using either High or Low & High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.High}, []plot.Roles{plot.Low}, func(data plot.Data) plot.Plotter { return NewYErrorBars(data) }) - plot.RegisterPlotter(XErrorBarsType, "draws draws horizontal error bars, denoting error in X values, using Low, High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.Low, plot.High}, []plot.Roles{}, func(data plot.Data) plot.Plotter { + plot.RegisterPlotter(XErrorBarsType, "draws draws horizontal error bars, denoting error in X values, using either High or Low & High data roles for error deviations around X, Y coordinates.", []plot.Roles{plot.X, plot.Y, plot.High}, []plot.Roles{plot.Low}, func(data plot.Data) plot.Plotter { return NewXErrorBars(data) }) } -// XErrorBars draws vertical error bars, denoting error in Y values, -// using Low, High data roles for error deviations around X, Y coordinates. +// YErrorBars draws vertical error bars, denoting error in Y values, +// using ether High or Low, High data roles for error deviations +// around X, Y coordinates. type YErrorBars struct { // copies of data for this line X, Y, Low, High plot.Values @@ -40,7 +41,8 @@ type YErrorBars struct { // Style is the style for plotting. Style plot.Style - stylers plot.Stylers + stylers plot.Stylers + ystylers plot.Stylers } func (eb *YErrorBars) Defaults() { @@ -57,12 +59,16 @@ func NewYErrorBars(data plot.Data) *YErrorBars { eb := &YErrorBars{} eb.X = plot.MustCopyRole(data, plot.X) eb.Y = plot.MustCopyRole(data, plot.Y) - eb.Low = plot.MustCopyRole(data, plot.Low) - eb.High = plot.MustCopyRole(data, plot.High) + eb.Low = plot.CopyRole(data, plot.Low) + eb.High = plot.CopyRole(data, plot.High) + if eb.Low == nil && eb.High != nil { + eb.Low = eb.High + } if eb.X == nil || eb.Y == nil || eb.Low == nil || eb.High == nil { return nil } eb.stylers = plot.GetStylersFromData(data, plot.High) + eb.ystylers = plot.GetStylersFromData(data, plot.Y) eb.Defaults() return eb } @@ -75,6 +81,9 @@ func (eb *YErrorBars) Styler(f func(s *plot.Style)) *YErrorBars { func (eb *YErrorBars) ApplyStyle(ps *plot.PlotStyle) { ps.SetElementStyle(&eb.Style) + yst := &plot.Style{} + eb.ystylers.Run(yst) + eb.Style.Range = yst.Range // get range from y eb.stylers.Run(&eb.Style) } @@ -124,6 +133,7 @@ func (eb *YErrorBars) Plot(plt *plot.Plot) { // UpdateRange updates the given ranges. func (eb *YErrorBars) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { plot.Range(eb.X, xr) + plot.RangeClamp(eb.Y, yr, &eb.Style.Range) for i, y := range eb.Y { ylow := y - math.Abs(eb.Low[i]) yhigh := y + math.Abs(eb.High[i]) @@ -135,7 +145,8 @@ func (eb *YErrorBars) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { //////// XErrorBars // XErrorBars draws horizontal error bars, denoting error in X values, -// using Low, High data roles for error deviations around X, Y coordinates. +// using ether High or Low, High data roles for error deviations +// around X, Y coordinates. type XErrorBars struct { // copies of data for this line X, Y, Low, High plot.Values @@ -146,7 +157,9 @@ type XErrorBars struct { // Style is the style for plotting. Style plot.Style - stylers plot.Stylers + stylers plot.Stylers + ystylers plot.Stylers + yrange minmax.Range64 } func (eb *XErrorBars) Defaults() { @@ -164,10 +177,16 @@ func NewXErrorBars(data plot.Data) *XErrorBars { eb.Y = plot.MustCopyRole(data, plot.Y) eb.Low = plot.MustCopyRole(data, plot.Low) eb.High = plot.MustCopyRole(data, plot.High) + eb.Low = plot.CopyRole(data, plot.Low) + eb.High = plot.CopyRole(data, plot.High) + if eb.Low == nil && eb.High != nil { + eb.Low = eb.High + } if eb.X == nil || eb.Y == nil || eb.Low == nil || eb.High == nil { return nil } eb.stylers = plot.GetStylersFromData(data, plot.High) + eb.ystylers = plot.GetStylersFromData(data, plot.Y) eb.Defaults() return eb } @@ -180,6 +199,9 @@ func (eb *XErrorBars) Styler(f func(s *plot.Style)) *XErrorBars { func (eb *XErrorBars) ApplyStyle(ps *plot.PlotStyle) { ps.SetElementStyle(&eb.Style) + yst := &plot.Style{} + eb.ystylers.Run(yst) + eb.yrange = yst.Range // get range from y eb.stylers.Run(&eb.Style) } @@ -227,12 +249,13 @@ func (eb *XErrorBars) Plot(plt *plot.Plot) { } // UpdateRange updates the given ranges. -func (eb *XErrorBars) UpdateRange(plt *plot.Plot, x, y, z *minmax.F64) { - plot.Range(eb.Y, y) +func (eb *XErrorBars) UpdateRange(plt *plot.Plot, xr, yr, zr *minmax.F64) { + plot.RangeClamp(eb.X, xr, &eb.Style.Range) + plot.RangeClamp(eb.Y, yr, &eb.yrange) for i, xv := range eb.X { xlow := xv - math.Abs(eb.Low[i]) xhigh := xv + math.Abs(eb.High[i]) - x.FitInRange(minmax.F64{xlow, xhigh}) + xr.FitInRange(minmax.F64{xlow, xhigh}) } return } diff --git a/plot/plots/labels.go b/plot/plots/labels.go index f9ffe84b76..1ecde00e18 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -36,6 +36,7 @@ type Labels struct { // plot size and number of TextStyle when styles last generated -- don't regen styleSize image.Point stylers plot.Stylers + ystylers plot.Stylers } // NewLabels returns a new Labels using defaults @@ -60,6 +61,7 @@ func NewLabels(data plot.Data) *Labels { } lb.stylers = plot.GetStylersFromData(data, plot.Label) + lb.ystylers = plot.GetStylersFromData(data, plot.Y) lb.Defaults() return lb } @@ -76,7 +78,10 @@ func (lb *Labels) Styler(f func(s *plot.Style)) *Labels { func (lb *Labels) ApplyStyle(ps *plot.PlotStyle) { ps.SetElementStyle(&lb.Style) - lb.stylers.Run(&lb.Style) + yst := &plot.Style{} + lb.ystylers.Run(yst) + lb.Style.Range = yst.Range // get range from y + lb.stylers.Run(&lb.Style) // can still override here } func (lb *Labels) Stylers() *plot.Stylers { return &lb.stylers } diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 569e3689a7..917d6eb574 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -14,6 +14,7 @@ import ( "strconv" "testing" + "cogentcore.org/core/base/errors" "cogentcore.org/core/base/iox/imagex" "cogentcore.org/core/colors" "cogentcore.org/core/paint" @@ -83,6 +84,60 @@ func ExampleStylerMetadata() { // Output: } +func ExampleTable() { + rand.Seed(1) + n := 21 + tx, ty, th := tensor.NewFloat64(n), tensor.NewFloat64(n), tensor.NewFloat64(n) + lbls := tensor.NewString(n) + for i := range n { + tx.SetFloat1D(float64(i*5), i) + ty.SetFloat1D(50.0+40*math.Sin((float64(i)/8)*math.Pi), i) + th.SetFloat1D(5*rand.Float64(), i) + lbls.SetString1D(strconv.Itoa(i), i) + } + genst := func(s *plot.Style) { + s.Plot.Title = "Test Table" + s.Plot.XAxis.Label = "X Axis" + s.Plot.YAxisLabel = "Y Axis" + s.Plot.Scale = 2 + s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.Off) + s.Line.Color = colors.Uniform(colors.Red) + s.Range.SetMin(0).SetMax(100) + } + plot.SetStylersTo(ty, plot.Stylers{genst, func(s *plot.Style) { + s.On = plot.On + s.Plotter = "XY" + s.Role = plot.Y + }}) + // others get basic styling + plot.SetStylersTo(tx, plot.Stylers{func(s *plot.Style) { + s.Role = plot.X + }}) + plot.SetStylersTo(th, plot.Stylers{func(s *plot.Style) { + s.On = plot.On + s.Plotter = "YErrorBars" + s.Role = plot.High + }}) + plot.SetStylersTo(lbls, plot.Stylers{func(s *plot.Style) { + s.On = plot.On + s.Plotter = "Labels" + s.Role = plot.Label + s.Text.Offset.X.Dp(6) + s.Text.Offset.Y.Dp(-6) + }}) + dt := table.New("Test Table") // todo: use Name by default for plot. + dt.AddColumn("X", tx) + dt.AddColumn("Y", ty) + dt.AddColumn("High", th) + dt.AddColumn("Labels", lbls) + + plt := errors.Log1(plot.NewTablePlot(dt)) + plt.Resize(image.Point{640, 480}) + plt.Draw() + imagex.Save(plt.Pixels, "testdata/ex_table.png") + // Output: +} + func TestMain(m *testing.M) { paint.FontLibrary.InitFontPaths(paint.FontPaths...) os.Exit(m.Run()) diff --git a/plot/table.go b/plot/table.go index 156f3e179e..2158fe8ecd 100644 --- a/plot/table.go +++ b/plot/table.go @@ -52,7 +52,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { for ci, cl := range dt.Columns.Values { cnm := dt.Columns.Keys[ci] st := csty[cl] - if st.On != On || st.Role == X { + if st == nil || st.On != On || st.Role == X { continue } gp := st.Group @@ -94,6 +94,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { err = fmt.Errorf("plot.NewTablePlot: Required Role %q not found in Group %q, Plotter %q not added for Column: %q", rl.String(), gp, ptyp, cnm) errs = append(errs, err) gotReq = false + fmt.Println(err) } } } From a725107d1c4c99ae26ab02f0119460bd8fcff7b4 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Mon, 11 Nov 2024 17:19:03 -0800 Subject: [PATCH 283/311] fix inverted scale -> needs underlying scale --- plot/axis.go | 13 +++++++++---- plot/plots/plot_test.go | 4 ++-- plot/typegen.go | 24 ++++++++++++++++-------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/plot/axis.go b/plot/axis.go index 91e02e1d4e..43cdb36711 100644 --- a/plot/axis.go +++ b/plot/axis.go @@ -29,8 +29,11 @@ const ( // Log is a Logarithmic axis scale. Log - // Inverted is an inverted axis scale. - Inverted + // InverseLinear is an inverted linear axis scale. + InverseLinear + + // InverseLog is an inverted log axis scale. + InverseLog ) // AxisStyle has style properties for the axis. @@ -134,8 +137,10 @@ func (ax *Axis) drawConfig() { ax.Scale = LinearScale{} case Log: ax.Scale = LogScale{} - case Inverted: - ax.Scale = InvertedScale{} + case InverseLinear: + ax.Scale = InvertedScale{LinearScale{}} + case InverseLog: + ax.Scale = InvertedScale{LogScale{}} } } diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 917d6eb574..6895ba7f1f 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -101,13 +101,13 @@ func ExampleTable() { s.Plot.YAxisLabel = "Y Axis" s.Plot.Scale = 2 s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.Off) - s.Line.Color = colors.Uniform(colors.Red) - s.Range.SetMin(0).SetMax(100) } plot.SetStylersTo(ty, plot.Stylers{genst, func(s *plot.Style) { s.On = plot.On s.Plotter = "XY" s.Role = plot.Y + s.Line.Color = colors.Uniform(colors.Red) + s.Range.SetMin(0).SetMax(100) }}) // others get basic styling plot.SetStylersTo(tx, plot.Stylers{func(s *plot.Style) { diff --git a/plot/typegen.go b/plot/typegen.go index eca23e7358..2025794cbb 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -13,7 +13,7 @@ import ( var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisScales", IDName: "axis-scales", Doc: "AxisScales are the scaling options for how values are distributed\nalong an axis: Linear, Log, etc."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "NTicks", Doc: "NTicks is the desired number of ticks"}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the axis:\nLinear, Log, Inverted"}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "NTicks", Doc: "NTicks is the desired number of ticks."}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the axis:\nLinear, Log, Inverted"}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) // SetText sets the [AxisStyle.Text]: // Text has the text style parameters for the text label. @@ -30,7 +30,7 @@ func (t *AxisStyle) SetLine(v LineStyle) *AxisStyle { t.Line = v; return t } func (t *AxisStyle) SetPadding(v units.Value) *AxisStyle { t.Padding = v; return t } // SetNTicks sets the [AxisStyle.NTicks]: -// NTicks is the desired number of ticks +// NTicks is the desired number of ticks. func (t *AxisStyle) SetNTicks(v int) *AxisStyle { t.NTicks = v; return t } // SetScale sets the [AxisStyle.Scale]: @@ -146,7 +146,7 @@ func (t *LineStyle) SetStep(v StepKind) *LineStyle { t.Step = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.StepKind", IDName: "step-kind", Doc: "StepKind specifies a form of a connection of two consecutive points."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XAxisStyle", IDName: "x-axis-style", Doc: "XAxisStyle has overall plot level styling properties for the XAxis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Column", Doc: "Column specifies the column to use for the common X axis in a table based plot.\nif empty or not found, the row number is used.\nThis optional for Bar plots, if present and Legend is also present,\nthen an extra space will be put between X values."}, {Name: "Rotation", Doc: "Rotation is the rotation of the X Axis labels, in degrees."}, {Name: "Label", Doc: "Label is the optional label to use for the XAxis instead of the default."}, {Name: "Range", Doc: "Range is the effective range of XAxis data to plot, where either end can be fixed."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XAxisStyle", IDName: "x-axis-style", Doc: "XAxisStyle has overall plot level styling properties for the XAxis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Column", Doc: "Column specifies the column to use for the common X axis in a table based plot.\nif empty or not found, the row number is used.\nThis optional for Bar plots, if present and Legend is also present,\nthen an extra space will be put between X values."}, {Name: "Rotation", Doc: "Rotation is the rotation of the X Axis labels, in degrees."}, {Name: "Label", Doc: "Label is the optional label to use for the XAxis instead of the default."}, {Name: "Range", Doc: "Range is the effective range of XAxis data to plot, where either end can be fixed."}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the X axis:\nLinear, Log, Inverted"}}}) // SetColumn sets the [XAxisStyle.Column]: // Column specifies the column to use for the common X axis in a table based plot. @@ -167,6 +167,11 @@ func (t *XAxisStyle) SetLabel(v string) *XAxisStyle { t.Label = v; return t } // Range is the effective range of XAxis data to plot, where either end can be fixed. func (t *XAxisStyle) SetRange(v minmax.Range64) *XAxisStyle { t.Range = v; return t } +// SetScale sets the [XAxisStyle.Scale]: +// Scale specifies how values are scaled along the X axis: +// Linear, Log, Inverted +func (t *XAxisStyle) SetScale(v AxisScales) *XAxisStyle { t.Scale = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotStyle", IDName: "plot-style", Doc: "PlotStyle has overall plot level styling properties.\nSome properties provide defaults for individual elements, which can\nthen be overwritten by element-level properties.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Title", Doc: "Title is the overall title of the plot."}, {Name: "TitleStyle", Doc: "TitleStyle is the text styling parameters for the title."}, {Name: "Background", Doc: "Background is the background of the plot.\nThe default is [colors.Scheme.Surface]."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Legend", Doc: "Legend has the styling properties for the Legend."}, {Name: "Axis", Doc: "Axis has the styling properties for the Axes."}, {Name: "XAxis", Doc: "XAxis has plot-level XAxis style properties."}, {Name: "YAxisLabel", Doc: "YAxisLabel is the optional label to use for the YAxis instead of the default."}, {Name: "LinesOn", Doc: "LinesOn determines whether lines are plotted by default,\nfor elements that plot lines (e.g., plots.XY)."}, {Name: "LineWidth", Doc: "LineWidth sets the default line width for data plotting lines."}, {Name: "PointsOn", Doc: "PointsOn determines whether points are plotted by default,\nfor elements that plot points (e.g., plots.XY)."}, {Name: "PointSize", Doc: "PointSize sets the default point size."}, {Name: "LabelSize", Doc: "LabelSize sets the default label text size."}, {Name: "BarWidth", Doc: "BarWidth for Bar plot sets the default width of the bars,\nwhich should be less than the Stride (1 typically) to prevent\nbar overlap. Defaults to .8."}}}) // SetTitle sets the [PlotStyle.Title]: @@ -272,7 +277,7 @@ func (t *PointStyle) SetSize(v units.Value) *PointStyle { t.Size = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Shapes", IDName: "shapes", Doc: "Shapes has the options for how to draw points in the plot."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for cases where it can be turned off."}, {Name: "Role", Doc: "Role specifies a role for this item, used for table-based plots to indicate\nhow a particular column of data should be used."}, {Name: "Group", Doc: "Group specifies a group of related data items, used for table-based plots\nwhere different columns of data within the same Group play different Roles"}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for table-based plots."}, {Name: "Plotter", Doc: "Plotter is the type of plotter to use in plotting this data,\nfor table-based plots. Blank means use default ([plots.XY] is overall default)."}, {Name: "Role", Doc: "Role specifies a role for this item, used for table-based plots to indicate\nhow a particular column of data should be used."}, {Name: "Group", Doc: "Group specifies a group of related data items, used for table-based plots\nwhere different columns of data within the same Group play different Roles"}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) // SetPlot sets the [Style.Plot]: // @@ -282,9 +287,14 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName func (t *Style) SetPlot(v PlotStyle) *Style { t.Plot = v; return t } // SetOn sets the [Style.On]: -// On specifies whether to plot this item, for cases where it can be turned off. +// On specifies whether to plot this item, for table-based plots. func (t *Style) SetOn(v DefaultOffOn) *Style { t.On = v; return t } +// SetPlotter sets the [Style.Plotter]: +// Plotter is the type of plotter to use in plotting this data, +// for table-based plots. Blank means use default ([plots.XY] is overall default). +func (t *Style) SetPlotter(v string) *Style { t.Plotter = v; return t } + // SetRole sets the [Style.Role]: // Role specifies a role for this item, used for table-based plots to indicate // how a particular column of data should be used. @@ -379,7 +389,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Text", IDName: var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Tick", IDName: "tick", Doc: "A Tick is a single tick mark on an axis.", Fields: []types.Field{{Name: "Value", Doc: "Value is the data value marked by this Tick."}, {Name: "Label", Doc: "Label is the text to display at the tick mark.\nIf Label is an empty string then this is a minor tick mark."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Ticker", IDName: "ticker", Doc: "Ticker creates Ticks in a specified range", Methods: []types.Method{{Name: "Ticks", Doc: "Ticks returns Ticks in a specified range", Args: []string{"min", "max"}, Returns: []string{"Tick"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Ticker", IDName: "ticker", Doc: "Ticker creates Ticks in a specified range", Methods: []types.Method{{Name: "Ticks", Doc: "Ticks returns Ticks in a specified range, with desired number of ticks,\nwhich can be ignored depending on the ticker type.", Args: []string{"min", "max", "nticks"}, Returns: []string{"Tick"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DefaultTicks", IDName: "default-ticks", Doc: "DefaultTicks is suitable for the Ticker field of an Axis,\nit returns a reasonable default set of tick marks."}) @@ -388,5 +398,3 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LogTicks", IDN var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.ConstantTicks", IDName: "constant-ticks", Doc: "ConstantTicks is suitable for the Ticker field of an Axis.\nThis function returns the given set of ticks."}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TimeTicks", IDName: "time-ticks", Doc: "TimeTicks is suitable for axes representing time values.", Fields: []types.Field{{Name: "Ticker", Doc: "Ticker is used to generate a set of ticks.\nIf nil, DefaultTicks will be used."}, {Name: "Format", Doc: "Format is the textual representation of the time value.\nIf empty, time.RFC3339 will be used"}, {Name: "Time", Doc: "Time takes a float32 value and converts it into a time.Time.\nIf nil, UTCUnixTime is used."}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TickerFunc", IDName: "ticker-func", Doc: "TickerFunc is suitable for the Ticker field of an Axis.\nIt is an adapter which allows to quickly setup a Ticker using a function with an appropriate signature."}) From 45890e0bd902189e567e72d51a0099a4c73968e3 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 02:28:54 -0800 Subject: [PATCH 284/311] ploteditor taking shape -- just need a reflectx copy named fields function for the styler, to copy from current style object to target in styler function. --- core/form.go | 34 ++- examples/plot/plot.go | 19 +- plot/draw.go | 10 +- plot/legend.go | 6 +- plot/plot.go | 6 +- plot/plotcore/barplot.go | 2 + plot/plotcore/enumgen.go | 50 ---- plot/plotcore/metadata.go | 122 --------- plot/plotcore/options.go | 221 ---------------- plot/plotcore/plot.go | 22 +- plot/plotcore/ploteditor.go | 441 +++++++++++--------------------- plot/plotcore/plotterchooser.go | 38 +++ plot/plotcore/tablexy.go | 2 + plot/plotcore/typegen.go | 25 +- plot/plotcore/xyplot.go | 2 + plot/plots/labels.go | 3 +- plot/plots/plot_test.go | 12 +- plot/plotter.go | 3 + plot/style.go | 14 +- plot/table.go | 4 +- plot/text.go | 56 ++-- plot/typegen.go | 31 ++- styles/font.go | 21 +- 23 files changed, 374 insertions(+), 770 deletions(-) delete mode 100644 plot/plotcore/enumgen.go delete mode 100644 plot/plotcore/metadata.go delete mode 100644 plot/plotcore/options.go create mode 100644 plot/plotcore/plotterchooser.go diff --git a/core/form.go b/core/form.go index 6cdbe7a85e..6bdd38c7db 100644 --- a/core/form.go +++ b/core/form.go @@ -30,6 +30,11 @@ type Form struct { // Inline is whether to display the form in one line. Inline bool + // Modified optionally highlights and tracks fields that have been modified + // through an OnChange event. If present, it replaces the default value highlighting + // and resetting logic. Ignored if nil. + Modified map[string]bool + // structFields are the fields of the current struct. structFields []*structField @@ -179,13 +184,20 @@ func (fm *Form) Init() { // (see https://github.com/cogentcore/core/issues/1098). doc, _ := types.GetDoc(f.value, f.parent, f.field, label) w.SetTooltip(doc) - if hasDef { - w.SetTooltip("(Default: " + def + ") " + w.Tooltip) + if hasDef || fm.Modified != nil { + if hasDef { + w.SetTooltip("(Default: " + def + ") " + w.Tooltip) + } var isDef bool w.Styler(func(s *styles.Style) { f := fm.structFields[i] - isDef = reflectx.ValueIsDefault(f.value, def) dcr := "(Double click to reset to default) " + if fm.Modified != nil { + isDef = !fm.Modified[f.path] + dcr = "(Double click to mark as non-modified) " + } else { + isDef = reflectx.ValueIsDefault(f.value, def) + } if !isDef { s.Color = colors.Scheme.Primary.Base s.Cursor = cursors.Poof @@ -202,13 +214,20 @@ func (fm *Form) Init() { return } e.SetHandled() - err := reflectx.SetFromDefaultTag(f.value, def) + var err error + if fm.Modified != nil { + fm.Modified[f.path] = false + } else { + err = reflectx.SetFromDefaultTag(f.value, def) + } if err != nil { ErrorSnackbar(w, err, "Error setting default value") } else { w.Update() valueWidget.AsWidget().Update() - valueWidget.AsWidget().SendChange(e) + if fm.Modified == nil { + valueWidget.AsWidget().SendChange(e) + } } }) } @@ -243,8 +262,11 @@ func (fm *Form) Init() { }) if !fm.IsReadOnly() && !readOnlyTag { wb.OnChange(func(e events.Event) { + if fm.Modified != nil { + fm.Modified[f.path] = true + } fm.SendChange(e) - if hasDef { + if hasDef || fm.Modified != nil { labelWidget.Update() } if fm.isShouldDisplayer { diff --git a/examples/plot/plot.go b/examples/plot/plot.go index 32e1c00954..99ada3fe88 100644 --- a/examples/plot/plot.go +++ b/examples/plot/plot.go @@ -8,7 +8,9 @@ import ( "embed" "cogentcore.org/core/core" + "cogentcore.org/core/plot" "cogentcore.org/core/plot/plotcore" + _ "cogentcore.org/core/plot/plots" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) @@ -21,13 +23,22 @@ func main() { epc := table.New("epc") epc.OpenFS(tsv, "ra25epoch.tsv", tensor.Tab) + epcc := epc.Column("Epoch") + plot.SetStylersTo(epcc, plot.Stylers{func(s *plot.Style) { + s.Role = plot.X + }}) + perr := epc.Column("PctErr") + plot.SetStylersTo(perr, plot.Stylers{func(s *plot.Style) { + s.On = true + s.Role = plot.Y + }}) pl := plotcore.NewPlotEditor(b) - pl.Options.Title = "RA25 Epoch Train" - pl.Options.XAxis = "Epoch" - pl.Options.Points = true + // pl.Options.Title = "RA25 Epoch Train" + // pl.Options.XAxis = "Epoch" + // pl.Options.Points = true + // pl.ColumnOptions("UnitErr").On = true pl.SetTable(epc) - pl.ColumnOptions("UnitErr").On = true b.AddTopBar(func(bar *core.Frame) { core.NewToolbar(bar).Maker(pl.MakeToolbar) }) diff --git a/plot/draw.go b/plot/draw.go index 2a015bd9ea..a572509fca 100644 --- a/plot/draw.go +++ b/plot/draw.go @@ -394,16 +394,16 @@ func (lg *Legend) draw(pt *Plot) { ptb := pc.Bounds lg.Style.ThumbnailWidth.ToDots(uc) - lg.Style.Text.ToDots(uc) lg.Style.Position.XOffs.ToDots(uc) lg.Style.Position.YOffs.ToDots(uc) - lg.Style.Text.openFont(pt) - - em := lg.Style.Text.Font.Face.Metrics.Em - pad := math32.Ceil(lg.Style.Text.Padding.Dots) var ltxt Text + ltxt.Defaults() ltxt.Style = lg.Style.Text + ltxt.ToDots(uc) + pad := math32.Ceil(ltxt.Style.Padding.Dots) + ltxt.openFont(pt) + em := ltxt.font.Face.Metrics.Em var sz image.Point maxTht := 0 for _, e := range lg.Entries { diff --git a/plot/legend.go b/plot/legend.go index dc825b8a12..695d07edca 100644 --- a/plot/legend.go +++ b/plot/legend.go @@ -19,13 +19,13 @@ type LegendStyle struct { //types:add -setters Column string // Text is the style given to the legend entry texts. - Text TextStyle + Text TextStyle `display:"add-fields"` // position of the legend Position LegendPosition `display:"inline"` // ThumbnailWidth is the width of legend thumbnails. - ThumbnailWidth units.Value + ThumbnailWidth units.Value `display:"inline"` // Fill specifies the background fill color for the legend box, // if non-nil. @@ -35,7 +35,7 @@ type LegendStyle struct { //types:add -setters func (ls *LegendStyle) Defaults() { ls.Text.Defaults() ls.Text.Padding.Dp(2) - ls.Text.Font.Size.Dp(20) + ls.Text.Size.Dp(20) ls.Position.Defaults() ls.ThumbnailWidth.Pt(20) ls.Fill = gradient.ApplyOpacity(colors.Scheme.Surface, 0.75) diff --git a/plot/plot.go b/plot/plot.go index d6d06b3cad..1dc87cba57 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -67,13 +67,13 @@ type PlotStyle struct { //types:add -setters Scale float32 `default:"1,2"` // Legend has the styling properties for the Legend. - Legend LegendStyle + Legend LegendStyle `display:"add-fields"` // Axis has the styling properties for the Axes. - Axis AxisStyle + Axis AxisStyle `display:"add-fields"` // XAxis has plot-level XAxis style properties. - XAxis XAxisStyle + XAxis XAxisStyle `display:"add-fields"` // YAxisLabel is the optional label to use for the YAxis instead of the default. YAxisLabel string diff --git a/plot/plotcore/barplot.go b/plot/plotcore/barplot.go index fe0461bf3d..e9964d7f8c 100644 --- a/plot/plotcore/barplot.go +++ b/plot/plotcore/barplot.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build not + package plotcore import ( diff --git a/plot/plotcore/enumgen.go b/plot/plotcore/enumgen.go deleted file mode 100644 index 3ea13f9f1c..0000000000 --- a/plot/plotcore/enumgen.go +++ /dev/null @@ -1,50 +0,0 @@ -// Code generated by "core generate"; DO NOT EDIT. - -package plotcore - -import ( - "cogentcore.org/core/enums" -) - -var _PlotTypesValues = []PlotTypes{0, 1} - -// PlotTypesN is the highest valid value for type PlotTypes, plus one. -const PlotTypesN PlotTypes = 2 - -var _PlotTypesValueMap = map[string]PlotTypes{`XY`: 0, `Bar`: 1} - -var _PlotTypesDescMap = map[PlotTypes]string{0: `XY is a standard line / point plot.`, 1: `Bar plots vertical bars.`} - -var _PlotTypesMap = map[PlotTypes]string{0: `XY`, 1: `Bar`} - -// String returns the string representation of this PlotTypes value. -func (i PlotTypes) String() string { return enums.String(i, _PlotTypesMap) } - -// SetString sets the PlotTypes value from its string representation, -// and returns an error if the string is invalid. -func (i *PlotTypes) SetString(s string) error { - return enums.SetString(i, s, _PlotTypesValueMap, "PlotTypes") -} - -// Int64 returns the PlotTypes value as an int64. -func (i PlotTypes) Int64() int64 { return int64(i) } - -// SetInt64 sets the PlotTypes value from an int64. -func (i *PlotTypes) SetInt64(in int64) { *i = PlotTypes(in) } - -// Desc returns the description of the PlotTypes value. -func (i PlotTypes) Desc() string { return enums.Desc(i, _PlotTypesDescMap) } - -// PlotTypesValues returns all possible values for the type PlotTypes. -func PlotTypesValues() []PlotTypes { return _PlotTypesValues } - -// Values returns all possible values for the type PlotTypes. -func (i PlotTypes) Values() []enums.Enum { return enums.Values(_PlotTypesValues) } - -// MarshalText implements the [encoding.TextMarshaler] interface. -func (i PlotTypes) MarshalText() ([]byte, error) { return []byte(i.String()), nil } - -// UnmarshalText implements the [encoding.TextUnmarshaler] interface. -func (i *PlotTypes) UnmarshalText(text []byte) error { - return enums.UnmarshalText(i, text, "PlotTypes") -} diff --git a/plot/plotcore/metadata.go b/plot/plotcore/metadata.go deleted file mode 100644 index 9b0009f0f7..0000000000 --- a/plot/plotcore/metadata.go +++ /dev/null @@ -1,122 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package plotcore - -import ( - "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/metadata" - "cogentcore.org/core/tensor/table" -) - -// todo: make a more systematic version driven by reflect fields -// key issue is the same nullable type issue: you don't want to set everything -// in md. - -// also the Column name and IsString should not be in the struct! - -// fromMeta sets plot options from meta data. -func (po *PlotOptions) fromMeta(dt *table.Table) { - po.FromMetaMap(dt.Meta) -} - -// FromMetaMap sets plot options from meta data map. -func (po *PlotOptions) FromMetaMap(meta metadata.Data) { - if typ, err := metadata.Get[string](meta, "Type"); err == nil { - po.Type.SetString(typ) - } - if op, err := metadata.Get[bool](meta, "Lines"); err == nil { - po.Lines = op - } - if op, err := metadata.Get[bool](meta, "Points"); err == nil { - po.Points = op - } - if lw, err := metadata.Get[float64](meta, "LineWidth"); err == nil { - po.LineWidth = float32(lw) - } - if ps, err := metadata.Get[float64](meta, "PointSize"); err == nil { - po.PointSize = float32(ps) - } - if bw, err := metadata.Get[float64](meta, "BarWidth"); err == nil { - po.BarWidth = float32(bw) - } - if op, err := metadata.Get[bool](meta, "NegativeXDraw"); err == nil { - po.NegativeXDraw = op - } - if scl, err := metadata.Get[float64](meta, "Scale"); err == nil { - po.Scale = float32(scl) - } - if xc, err := metadata.Get[string](meta, "XAxis"); err == nil { - po.XAxis = xc - } - if lc, err := metadata.Get[string](meta, "Legend"); err == nil { - po.Legend = lc - } - if xrot, err := metadata.Get[float64](meta, "XAxisRotation"); err == nil { - po.XAxisRotation = float32(xrot) - } - if lb, err := metadata.Get[string](meta, "XAxisLabel"); err == nil { - po.XAxisLabel = lb - } - if lb, err := metadata.Get[string](meta, "YAxisLabel"); err == nil { - po.YAxisLabel = lb - } -} - -// fromMetaMap sets column options from meta data map. -func (co *ColumnOptions) fromMetaMap(meta metadata.Data) { - if op, err := metadata.Get[bool](meta, co.Column+":On"); err == nil { - co.On = op - } - if op, err := metadata.Get[bool](meta, co.Column+":Off"); err == nil { - co.On = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FixMin"); err == nil { - co.Range.FixMin = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FixMax"); err == nil { - co.Range.FixMax = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FloatMin"); err == nil { - co.Range.FixMin = op - } - if op, err := metadata.Get[bool](meta, co.Column+":FloatMax"); err == nil { - co.Range.FixMax = op - } - if vl, err := metadata.Get[float64](meta, co.Column+":Max"); err == nil { - co.Range.Max = float32(vl) - } - if vl, err := metadata.Get[float64](meta, co.Column+":Min"); err == nil { - co.Range.Min = float32(vl) - } - if lb, err := metadata.Get[string](meta, co.Column+":Label"); err == nil { - co.Label = lb - } - if lb, err := metadata.Get[string](meta, co.Column+":ErrColumn"); err == nil { - co.ErrColumn = lb - } - if vl, err := metadata.Get[int](meta, co.Column+":TensorIndex"); err == nil { - co.TensorIndex = vl - } -} - -// PlotColumnZeroOne returns plot options with a fixed 0-1 range, -// and the standard key: "PlotColumnOptions" -func PlotColumnZeroOne() (*ColumnOptions, string) { - opts := &ColumnOptions{} - opts.Range.SetMin(0) - opts.Range.SetMax(1) - return opts, "PlotColumnOptions" -} - -// SetPlotColumnOptions sets given plotting options for named items -// within this directory, in Metadata. -func SetPlotColumnOptions(md metadata.Data, opts *ColumnOptions) { - md.Set("PlotColumnOptions", opts) -} - -// PlotColumnOptions returns plotting options if they have been set, else nil. -func PlotColumnOptions(md metadata.Data) *ColumnOptions { - return errors.Ignore1(metadata.Get[*ColumnOptions](md, "PlotColumnOptions")) -} diff --git a/plot/plotcore/options.go b/plot/plotcore/options.go deleted file mode 100644 index e1349131d2..0000000000 --- a/plot/plotcore/options.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package plotcore - -import ( - "image" - - "cogentcore.org/core/base/metadata" - "cogentcore.org/core/base/option" - "cogentcore.org/core/math32/minmax" - "cogentcore.org/core/plot" - "cogentcore.org/core/plot/plots" -) - -// PlotOptions are options for the overall plot. -type PlotOptions struct { //types:add - - // optional title at top of plot - Title string - - // type of plot to generate. For a Bar plot, items are plotted ordinally by row and the XAxis is optional - Type PlotTypes - - // whether to plot lines - Lines bool `default:"true"` - - // whether to plot points with symbols - Points bool - - // width of lines - LineWidth float32 `default:"1"` - - // size of points - PointSize float32 `default:"3"` - - // the shape used to draw points - PointShape plots.Shapes - - // width of bars for bar plot, as fraction of available space (1 = no gaps) - BarWidth float32 `min:"0.01" max:"1" default:"0.8"` - - // if true, draw lines that connect points with a negative X-axis direction; - // otherwise there is a break in the line. - // default is false, so that repeated series of data across the X axis - // are plotted separately. - NegativeXDraw bool - - // Scale multiplies the plot DPI value, to change the overall scale - // of the rendered plot. Larger numbers produce larger scaling. - // Typically use larger numbers when generating plots for inclusion in - // documents or other cases where the overall plot size will be small. - Scale float32 `default:"1,2"` - - // what column to use for the common X axis. if empty or not found, - // the row number is used. This optional for Bar plots, if present and - // Legend is also present, then an extra space will be put between X values. - XAxis string - - // optional column for adding a separate colored / styled line or bar - // according to this value, and acts just like a separate Y variable, - // crossed with Y variables. - Legend string - - // position of the Legend - LegendPosition plot.LegendPosition `display:"inline"` - - // rotation of the X Axis labels, in degrees - XAxisRotation float32 - - // optional label to use for XAxis instead of column name - XAxisLabel string - - // optional label to use for YAxis -- if empty, first column name is used - YAxisLabel string -} - -// Defaults sets defaults if unset values are present. -func (po *PlotOptions) Defaults() { - if po.LineWidth == 0 { - po.LineWidth = 1 - po.Lines = true - po.Points = false - po.PointSize = 3 - po.BarWidth = .8 - po.LegendPosition.Defaults() - } - if po.Scale == 0 { - po.Scale = 1 - } -} - -// ColumnOptions are options for plotting one column of data. -type ColumnOptions struct { //types:add - // whether to plot this column - On bool - - // name of column being plotted - Column string - - // whether to plot lines; uses the overall plot option if unset - Lines option.Option[bool] - - // whether to plot points with symbols; uses the overall plot option if unset - Points option.Option[bool] - - // the width of lines; uses the overall plot option if unset - LineWidth option.Option[float32] - - // the size of points; uses the overall plot option if unset - PointSize option.Option[float32] - - // the shape used to draw points; uses the overall plot option if unset - PointShape option.Option[plots.Shapes] - - // effective range of data to plot -- either end can be fixed - Range minmax.Range32 `display:"inline"` - - // full actual range of data -- only valid if specifically computed - FullRange minmax.F32 `display:"inline"` - - // color to use when plotting the line / column - Color image.Image - - // desired number of ticks - NTicks int - - // if specified, this is an alternative label to use when plotting - Label string - - // if column has n-dimensional tensor cells in each row, this is the index within each cell to plot -- use -1 to plot *all* indexes as separate lines - TensorIndex int - - // specifies a column containing error bars for this column - ErrColumn string - - // if true this is a string column -- plots as labels - IsString bool `edit:"-"` -} - -// Defaults sets defaults if unset values are present. -func (co *ColumnOptions) Defaults() { - if co.NTicks == 0 { - co.NTicks = 10 - } -} - -// getLabel returns the effective label of the column. -func (co *ColumnOptions) getLabel() string { - if co.Label != "" { - return co.Label - } - return co.Column -} - -// PlotTypes are different types of plots. -type PlotTypes int32 //enums:enum - -const ( - // XY is a standard line / point plot. - XY PlotTypes = iota - - // Bar plots vertical bars. - Bar -) - -//////// Stylers - -// PlotStylers are plot styling functions. -type PlotStylers struct { - Plot []func(po *PlotOptions) - Column map[string][]func(co *ColumnOptions) -} - -func NewPlotStylers() *PlotStylers { - return &PlotStylers{} -} - -// PlotStyler adds a plot styling function. -func (ps *PlotStylers) PlotStyler(f func(po *PlotOptions)) { - ps.Plot = append(ps.Plot, f) -} - -// ColumnStyler adds a column styling function for given column name. -func (ps *PlotStylers) ColumnStyler(col string, f func(co *ColumnOptions)) { - if ps.Column == nil { - ps.Column = make(map[string][]func(co *ColumnOptions)) - } - cs := ps.Column[col] - ps.Column[col] = append(cs, f) -} - -// ApplyToPlot applies stylers to plot options. -func (ps *PlotStylers) ApplyToPlot(po *PlotOptions) { - for _, f := range ps.Plot { - f(po) - } -} - -// ApplyToColumn applies stylers to column of given name -func (ps *PlotStylers) ApplyToColumn(co *ColumnOptions) { - if ps.Column == nil { - return - } - fs := ps.Column[co.Column] - for _, f := range fs { - f(co) - } -} - -// SetPlotStylers sets the PlotStylers into given metadata. -func SetPlotStylers(md *metadata.Data, ps *PlotStylers) { - md.Set("PlotStylers", ps) -} - -// GetPlotStylers gets the PlotStylers from given metadata (nil if none). -func GetPlotStylers(md *metadata.Data) *PlotStylers { - ps, _ := metadata.Get[*PlotStylers](*md, "PlotStylers") - return ps -} diff --git a/plot/plotcore/plot.go b/plot/plotcore/plot.go index 61b3e27fb2..f12f1515c0 100644 --- a/plot/plotcore/plot.go +++ b/plot/plotcore/plot.go @@ -94,12 +94,12 @@ func (pt *Plot) Init() { return } del := e.PrevDelta() - dx := -float32(del.X) * (pt.Plot.X.Max - pt.Plot.X.Min) * 0.0008 - dy := float32(del.Y) * (pt.Plot.Y.Max - pt.Plot.Y.Min) * 0.0008 - pt.Plot.X.Min += dx - pt.Plot.X.Max += dx - pt.Plot.Y.Min += dy - pt.Plot.Y.Max += dy + dx := -float64(del.X) * (pt.Plot.X.Range.Range()) * 0.0008 + dy := float64(del.Y) * (pt.Plot.Y.Range.Range()) * 0.0008 + pt.Plot.X.Range.Min += dx + pt.Plot.X.Range.Max += dx + pt.Plot.Y.Range.Min += dy + pt.Plot.Y.Range.Max += dy pt.updatePlot() pt.NeedsRender() }) @@ -110,11 +110,11 @@ func (pt *Plot) Init() { return } se := e.(*events.MouseScroll) - sc := 1 + (float32(se.Delta.Y) * 0.002) - pt.Plot.X.Min *= sc - pt.Plot.X.Max *= sc - pt.Plot.Y.Min *= sc - pt.Plot.Y.Max *= sc + sc := 1 + (float64(se.Delta.Y) * 0.002) + pt.Plot.X.Range.Min *= sc + pt.Plot.X.Range.Max *= sc + pt.Plot.Y.Range.Min *= sc + pt.Plot.Y.Range.Max *= sc pt.updatePlot() pt.NeedsRender() }) diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index f2602a25cc..655901ac84 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -8,22 +8,20 @@ package plotcore //go:generate core generate import ( + "fmt" "io/fs" "log/slog" "path/filepath" - "reflect" "strings" "time" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/iox/imagex" - "cogentcore.org/core/base/metadata" "cogentcore.org/core/colors" "cogentcore.org/core/core" "cogentcore.org/core/events" "cogentcore.org/core/icons" - "cogentcore.org/core/math32" "cogentcore.org/core/plot" "cogentcore.org/core/styles" "cogentcore.org/core/styles/states" @@ -44,14 +42,8 @@ type PlotEditor struct { //types:add // table is the table of data being plotted. table *table.Table - // Options are the overall plot options. - Options PlotOptions - - // Columns are the options for each column of the table. - Columns []*ColumnOptions `set:"-"` - - // Stylers are plot styling functions. - Stylers PlotStylers + // PlotStyle has the overall plot style parameters. + PlotStyle plot.PlotStyle // plot is the plot object. plot *plot.Plot @@ -72,12 +64,8 @@ type PlotEditor struct { //types:add func (pl *PlotEditor) CopyFieldsFrom(frm tree.Node) { fr := frm.(*PlotEditor) pl.Frame.CopyFieldsFrom(&fr.Frame) - pl.Options = fr.Options + pl.PlotStyle = fr.PlotStyle pl.setTable(fr.table) - mx := min(len(pl.Columns), len(fr.Columns)) - for i := 0; i < mx; i++ { - *pl.Columns[i] = *fr.Columns[i] - } } // NewSubPlot returns a [PlotEditor] with its own separate [core.Toolbar], @@ -97,8 +85,7 @@ func NewSubPlot(parent ...tree.Node) *PlotEditor { func (pl *PlotEditor) Init() { pl.Frame.Init() - pl.Options.Defaults() - pl.Stylers.ApplyToPlot(&pl.Options) + pl.PlotStyle.Defaults() pl.Styler(func(s *styles.Style) { s.Grow.Set(1, 1) if pl.SizeClass() == core.SizeCompact { @@ -110,11 +97,11 @@ func (pl *PlotEditor) Init() { pl.UpdatePlot() }) - pl.Updater(func() { - if pl.table != nil { - pl.Options.fromMeta(pl.table) - } - }) + // pl.Updater(func() { + // if pl.table != nil { + // pl.Options.fromMeta(pl.table) + // } + // }) tree.AddChildAt(pl, "columns", func(w *core.Frame) { pl.columnsFrame = w w.Styler(func(s *styles.Style) { @@ -144,9 +131,6 @@ func (pl *PlotEditor) Init() { // This is safe to call from a different goroutine. func (pl *PlotEditor) setTable(tab *table.Table) *PlotEditor { pl.table = tab - if ps := GetPlotStylers(&tab.Meta); ps != nil { - pl.Stylers = *ps - } pl.Update() return pl } @@ -157,9 +141,6 @@ func (pl *PlotEditor) setTable(tab *table.Table) *PlotEditor { // This is safe to call from a different goroutine. func (pl *PlotEditor) SetTable(tab *table.Table) *PlotEditor { pl.table = table.NewView(tab) - if ps := GetPlotStylers(&tab.Meta); ps != nil { - pl.Stylers = *ps - } pl.Update() return pl } @@ -170,46 +151,6 @@ func (pl *PlotEditor) SetSlice(sl any) *PlotEditor { return pl.SetTable(errors.Log1(table.NewSliceTable(sl))) } -// ColumnOptions returns the current column options by name -// (to access by index, just use Columns directly). -func (pl *PlotEditor) ColumnOptions(column string) *ColumnOptions { - for _, co := range pl.Columns { - if co.Column == column { - return co - } - } - return nil -} - -// Bool constants for [PlotEditor.SetColumnOptions]. -const ( - On = true - Off = false - FixMin = true - FloatMin = false - FixMax = true - FloatMax = false -) - -// SetColumnOptions sets the main parameters for one column. -func (pl *PlotEditor) SetColumnOptions(column string, on bool, fixMin bool, min float32, fixMax bool, max float32) *ColumnOptions { - co := pl.ColumnOptions(column) - if co == nil { - slog.Error("plotcore.PlotEditor.SetColumnOptions: column not found", "column", column) - return nil - } - co.On = on - co.Range.FixMin = fixMin - if fixMin { - co.Range.Min = min - } - co.Range.FixMax = fixMax - if fixMax { - co.Range.Max = max - } - return co -} - // SaveSVG saves the plot to an svg -- first updates to ensure that plot is current func (pl *PlotEditor) SaveSVG(fname core.Filename) { //types:add pl.UpdatePlot() @@ -256,34 +197,6 @@ func (pl *PlotEditor) OpenFS(fsys fs.FS, filename core.Filename, delim tensor.De pl.UpdatePlot() } -// yLabel returns the Y-axis label -func (pl *PlotEditor) yLabel() string { - if pl.Options.YAxisLabel != "" { - return pl.Options.YAxisLabel - } - for _, cp := range pl.Columns { - if cp.On { - return cp.getLabel() - } - } - return "Y" -} - -// xLabel returns the X-axis label -func (pl *PlotEditor) xLabel() string { - if pl.Options.XAxisLabel != "" { - return pl.Options.XAxisLabel - } - if pl.Options.XAxis != "" { - cp := pl.ColumnOptions(pl.Options.XAxis) - if cp != nil { - return cp.getLabel() - } - return pl.Options.XAxis - } - return "X" -} - // GoUpdatePlot updates the display based on current Indexed view into table. // This version can be called from goroutines. It does Sequential() on // the [table.Table], under the assumption that it is used for tracking a @@ -315,7 +228,7 @@ func (pl *PlotEditor) UpdatePlot() { if pl.table == nil || pl.inPlot { return } - if len(pl.Children) != 2 || len(pl.Columns) != pl.table.NumColumns() { + if len(pl.Children) != 2 { // || len(pl.Columns) != pl.table.NumColumns() { // todo: pl.Update() } if pl.table.NumRows() == 0 { @@ -344,181 +257,105 @@ func (pl *PlotEditor) genPlot() { pl.table.Sequential() } } - pl.plot = nil - switch pl.Options.Type { - case XY: - pl.genPlotXY() - case Bar: - pl.genPlotBar() - } - pl.plotWidget.Scale = pl.Options.Scale - pl.plotWidget.SetRangesFunc = func() { - plt := pl.plotWidget.Plot - xi := pl.table.ColumnIndex(pl.Options.XAxis) - if xi >= 0 { - xp := pl.Columns[xi] - if xp.Range.FixMin { - plt.X.Min = math32.Min(plt.X.Min, float32(xp.Range.Min)) - } - if xp.Range.FixMax { - plt.X.Max = math32.Max(plt.X.Max, float32(xp.Range.Max)) - } - } - for _, cp := range pl.Columns { // key that this comes at the end, to actually stick - if !cp.On || cp.IsString { - continue - } - if cp.Range.FixMin { - plt.Y.Min = math32.Min(plt.Y.Min, float32(cp.Range.Min)) - } - if cp.Range.FixMax { - plt.Y.Max = math32.Max(plt.Y.Max, float32(cp.Range.Max)) - } - } - } + pl.plot = errors.Log1(plot.NewTablePlot(pl.table)) pl.plotWidget.SetPlot(pl.plot) // redraws etc pl.inPlot = false } -// configPlot configures the given plot based on the plot options. -func (pl *PlotEditor) configPlot(plt *plot.Plot) { - plt.Title.Text = pl.Options.Title - plt.X.Label.Text = pl.xLabel() - plt.Y.Label.Text = pl.yLabel() - plt.Legend.Position = pl.Options.LegendPosition - plt.X.TickText.Style.Rotation = float32(pl.Options.XAxisRotation) -} - -// plotXAxis processes the XAxis and returns its index -func (pl *PlotEditor) plotXAxis(plt *plot.Plot, ixvw *table.Table) (xi int, xview *table.Table, err error) { - xi = ixvw.ColumnIndex(pl.Options.XAxis) - if xi < 0 { - // log.Println("plot.PlotXAxis: " + err.Error()) - return - } - xview = ixvw - xc := ixvw.ColumnByIndex(xi) - xp := pl.Columns[xi] - sz := 1 - if xp.Range.FixMin { - plt.X.Min = math32.Min(plt.X.Min, float32(xp.Range.Min)) - } - if xp.Range.FixMax { - plt.X.Max = math32.Max(plt.X.Max, float32(xp.Range.Max)) - } - if xc.Tensor.NumDims() > 1 { - sz = xc.NumRows() / xc.Tensor.DimSize(0) - if xp.TensorIndex > sz || xp.TensorIndex < 0 { - slog.Error("plotcore.PlotEditor.plotXAxis: TensorIndex invalid -- reset to 0") - xp.TensorIndex = 0 - } - } - return -} - const plotColumnsHeaderN = 2 // columnsListUpdate updates the list of columns -func (pl *PlotEditor) columnsListUpdate() { - if pl.table == nil { - pl.Columns = nil - return - } - dt := pl.table - nc := dt.NumColumns() - if nc == len(pl.Columns) { - return - } - pl.Columns = make([]*ColumnOptions, nc) - clri := 0 - hasOn := false - for ci := range dt.NumColumns() { - cn := dt.ColumnName(ci) - if pl.Options.XAxis == "" && ci == 0 { - pl.Options.XAxis = cn // x-axis defaults to the first column - } - cp := &ColumnOptions{Column: cn} - cp.Defaults() - pl.Stylers.ApplyToColumn(cp) - tcol := dt.ColumnByIndex(ci) - tc := tcol.Tensor - if tc.IsString() { - cp.IsString = true - } else { - cp.IsString = false - // we enable the first non-string, non-x-axis, non-first column by default - if !hasOn && cn != pl.Options.XAxis && ci != 0 { - cp.On = true - hasOn = true - } - } - cp.fromMetaMap(pl.table.Meta) - inc := 1 - if cn == pl.Options.XAxis || tc.IsString() || tc.DataType() == reflect.Int || tc.DataType() == reflect.Int64 || tc.DataType() == reflect.Int32 || tc.DataType() == reflect.Uint8 { - inc = 0 - } - cp.Color = colors.Uniform(colors.Spaced(clri)) - pl.Columns[ci] = cp - clri += inc - } -} - -// ColumnsFromMetaMap updates all the column settings from given meta map -func (pl *PlotEditor) ColumnsFromMetaMap(meta metadata.Data) { - for _, cp := range pl.Columns { - cp.fromMetaMap(meta) - } -} +// func (pl *PlotEditor) columnsListUpdate() { +// if pl.table == nil { +// return +// } +// dt := pl.table +// nc := dt.NumColumns() +// if nc == len(pl.Columns) { +// return +// } +// pl.Columns = make([]*ColumnOptions, nc) +// clri := 0 +// hasOn := false +// for ci := range dt.NumColumns() { +// cn := dt.ColumnName(ci) +// if pl.Options.XAxis == "" && ci == 0 { +// pl.Options.XAxis = cn // x-axis defaults to the first column +// } +// cp := &ColumnOptions{Column: cn} +// cp.Defaults() +// pl.Stylers.ApplyToColumn(cp) +// tcol := dt.ColumnByIndex(ci) +// tc := tcol.Tensor +// if tc.IsString() { +// cp.IsString = true +// } else { +// cp.IsString = false +// // we enable the first non-string, non-x-axis, non-first column by default +// if !hasOn && cn != pl.Options.XAxis && ci != 0 { +// cp.On = true +// hasOn = true +// } +// } +// cp.fromMetaMap(pl.table.Meta) +// inc := 1 +// if cn == pl.Options.XAxis || tc.IsString() || tc.DataType() == reflect.Int || tc.DataType() == reflect.Int64 || tc.DataType() == reflect.Int32 || tc.DataType() == reflect.Uint8 { +// inc = 0 +// } +// cp.Color = colors.Uniform(colors.Spaced(clri)) +// pl.Columns[ci] = cp +// clri += inc +// } +// } // setAllColumns turns all Columns on or off (except X axis) func (pl *PlotEditor) setAllColumns(on bool) { - fr := pl.columnsFrame - for i, cli := range fr.Children { - if i < plotColumnsHeaderN { - continue - } - ci := i - plotColumnsHeaderN - cp := pl.Columns[ci] - if cp.Column == pl.Options.XAxis { - continue - } - cp.On = on - cl := cli.(*core.Frame) - sw := cl.Child(0).(*core.Switch) - sw.SetChecked(cp.On) - } - pl.UpdatePlot() - pl.NeedsRender() + // fr := pl.columnsFrame + // for i, cli := range fr.Children { + // if i < plotColumnsHeaderN { + // continue + // } + // ci := i - plotColumnsHeaderN + // cp := pl.Columns[ci] + // if cp.Column == pl.Options.XAxis { + // continue + // } + // cp.On = on + // cl := cli.(*core.Frame) + // sw := cl.Child(0).(*core.Switch) + // sw.SetChecked(cp.On) + // } + // pl.UpdatePlot() + // pl.NeedsRender() } // setColumnsByName turns columns on or off if their name contains // the given string. func (pl *PlotEditor) setColumnsByName(nameContains string, on bool) { //types:add - fr := pl.columnsFrame - for i, cli := range fr.Children { - if i < plotColumnsHeaderN { - continue - } - ci := i - plotColumnsHeaderN - cp := pl.Columns[ci] - if cp.Column == pl.Options.XAxis { - continue - } - if !strings.Contains(cp.Column, nameContains) { - continue - } - cp.On = on - cl := cli.(*core.Frame) - sw := cl.Child(0).(*core.Switch) - sw.SetChecked(cp.On) - } - pl.UpdatePlot() - pl.NeedsRender() + // fr := pl.columnsFrame + // for i, cli := range fr.Children { + // if i < plotColumnsHeaderN { + // continue + // } + // ci := i - plotColumnsHeaderN + // cp := pl.Columns[ci] + // if cp.Column == pl.Options.XAxis { + // continue + // } + // if !strings.Contains(cp.Column, nameContains) { + // continue + // } + // cp.On = on + // cl := cli.(*core.Frame) + // sw := cl.Child(0).(*core.Switch) + // sw.SetChecked(cp.On) + // } + // pl.UpdatePlot() + // pl.NeedsRender() } // makeColumns makes the Plans for columns func (pl *PlotEditor) makeColumns(p *tree.Plan) { - pl.columnsListUpdate() tree.Add(p, func(w *core.Frame) { tree.AddChild(w, func(w *core.Button) { w.SetText("Clear").SetIcon(icons.ClearAll).SetType(core.ButtonAction) @@ -536,29 +373,55 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { }) }) tree.Add(p, func(w *core.Separator) {}) - for _, cp := range pl.Columns { - tree.AddAt(p, cp.Column, func(w *core.Frame) { + if pl.table == nil { + return + } + for ci, cl := range pl.table.Columns.Values { + cnm := pl.table.Columns.Keys[ci] + cst := &plot.Style{} + cst.Defaults() + psty := plot.GetStylersFrom(cl) + // nsty := 0 + if psty != nil { + // nsty = len(psty) + psty.Run(cst) + } + mods := map[string]bool{} + // metadata.SetTo(cl, "PlotEditorNStylers", nsty) + // metadata.SetTo(cl, "PlotEditorModsMap", mods) + updateStyle := func() { + if len(mods) == 0 { + return + } + sty := psty + sty = append(sty, func(s *plot.Style) { + }) + } + _ = updateStyle + tree.AddAt(p, cnm, func(w *core.Frame) { w.Styler(func(s *styles.Style) { s.CenterAll() }) tree.AddChild(w, func(w *core.Switch) { w.SetType(core.SwitchCheckbox).SetTooltip("Turn this column on or off") + // todo: set color according to style w.OnChange(func(e events.Event) { - cp.On = w.IsChecked() + mods["On"] = true + cst.On = w.IsChecked() pl.UpdatePlot() }) w.Updater(func() { - xaxis := cp.Column == pl.Options.XAxis || cp.Column == pl.Options.Legend + xaxis := cst.Role == plot.X // || cp.Column == pl.Options.Legend w.SetState(xaxis, states.Disabled, states.Indeterminate) if xaxis { - cp.On = false + cst.On = false } else { - w.SetChecked(cp.On) + w.SetChecked(cst.On) } }) }) tree.AddChild(w, func(w *core.Button) { - w.SetText(cp.Column).SetType(core.ButtonAction).SetTooltip("Edit column options including setting it as the x-axis or legend") + w.SetText(cnm).SetType(core.ButtonAction).SetTooltip("Edit all styling options for this column") w.OnClick(func(e events.Event) { update := func() { if core.TheApp.Platform().IsMobile() { @@ -571,27 +434,30 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { pl.Update() pl.AsyncUnlock() } - d := core.NewBody("Column options") - core.NewForm(d).SetStruct(cp). - OnChange(func(e events.Event) { - update() - }) - d.AddTopBar(func(bar *core.Frame) { - core.NewToolbar(bar).Maker(func(p *tree.Plan) { - tree.Add(p, func(w *core.Button) { - w.SetText("Set x-axis").OnClick(func(e events.Event) { - pl.Options.XAxis = cp.Column - update() - }) - }) - tree.Add(p, func(w *core.Button) { - w.SetText("Set legend").OnClick(func(e events.Event) { - pl.Options.Legend = cp.Column - update() - }) - }) - }) + d := core.NewBody("Column " + cnm + " style properties") + fm := core.NewForm(d).SetStruct(cst) + fm.Modified = mods + fm.OnChange(func(e events.Event) { + // todo: compile style based on changed items + fmt.Println(mods) + update() }) + // d.AddTopBar(func(bar *core.Frame) { + // core.NewToolbar(bar).Maker(func(p *tree.Plan) { + // tree.Add(p, func(w *core.Button) { + // w.SetText("Set x-axis").OnClick(func(e events.Event) { + // pl.Options.XAxis = cp.Column + // update() + // }) + // }) + // tree.Add(p, func(w *core.Button) { + // w.SetText("Set legend").OnClick(func(e events.Event) { + // pl.Options.Legend = cp.Column + // update() + // }) + // }) + // }) + // }) d.RunWindowDialog(pl) }) }) @@ -632,11 +498,14 @@ func (pl *PlotEditor) MakeToolbar(p *tree.Plan) { w.SetText("Options").SetIcon(icons.Settings). SetTooltip("Options for how the plot is rendered"). OnClick(func(e events.Event) { - d := core.NewBody("Plot options") - core.NewForm(d).SetStruct(&pl.Options). - OnChange(func(e events.Event) { - pl.GoUpdatePlot() - }) + d := core.NewBody("Plot style") + fm := core.NewForm(d).SetStruct(&pl.PlotStyle) + fm.Modified = map[string]bool{} + fm.OnChange(func(e events.Event) { + // todo: get modified and make style + fmt.Println(fm.Modified) + pl.GoUpdatePlot() + }) d.RunWindowDialog(pl) }) }) diff --git a/plot/plotcore/plotterchooser.go b/plot/plotcore/plotterchooser.go new file mode 100644 index 0000000000..e46372112b --- /dev/null +++ b/plot/plotcore/plotterchooser.go @@ -0,0 +1,38 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package plotcore + +import ( + "reflect" + "slices" + + "cogentcore.org/core/core" + "cogentcore.org/core/plot" + _ "cogentcore.org/core/plot/plots" + "golang.org/x/exp/maps" +) + +func init() { + core.AddValueType[plot.PlotterName, PlotterChooser]() +} + +// PlotterChooser represents a [Plottername] value with a [core.Chooser] +// for selecting a plotter. +type PlotterChooser struct { + core.Chooser + PlotterName string +} + +func (fc *PlotterChooser) WidgetValue() any { return &fc.PlotterName } + +func (fc *PlotterChooser) OnBind(value any, tags reflect.StructTag) { +} + +func (fc *PlotterChooser) Init() { + fc.Chooser.Init() + pnms := maps.Keys(plot.Plotters) + slices.Sort(pnms) + fc.Chooser.SetStrings(pnms...) +} diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go index 403a2a74d7..13f7805369 100644 --- a/plot/plotcore/tablexy.go +++ b/plot/plotcore/tablexy.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build not + package plotcore import ( diff --git a/plot/plotcore/typegen.go b/plot/plotcore/typegen.go index 9db4fb21c9..8ea5a24262 100644 --- a/plot/plotcore/typegen.go +++ b/plot/plotcore/typegen.go @@ -3,14 +3,11 @@ package plotcore import ( + "cogentcore.org/core/plot" "cogentcore.org/core/tree" "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotOptions", IDName: "plot-options", Doc: "PlotOptions are options for the overall plot.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "Title", Doc: "optional title at top of plot"}, {Name: "Type", Doc: "type of plot to generate. For a Bar plot, items are plotted ordinally by row and the XAxis is optional"}, {Name: "Lines", Doc: "whether to plot lines"}, {Name: "Points", Doc: "whether to plot points with symbols"}, {Name: "LineWidth", Doc: "width of lines"}, {Name: "PointSize", Doc: "size of points"}, {Name: "PointShape", Doc: "the shape used to draw points"}, {Name: "BarWidth", Doc: "width of bars for bar plot, as fraction of available space (1 = no gaps)"}, {Name: "NegativeXDraw", Doc: "if true, draw lines that connect points with a negative X-axis direction;\notherwise there is a break in the line.\ndefault is false, so that repeated series of data across the X axis\nare plotted separately."}, {Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "XAxis", Doc: "what column to use for the common X axis. if empty or not found,\nthe row number is used. This optional for Bar plots, if present and\nLegend is also present, then an extra space will be put between X values."}, {Name: "Legend", Doc: "optional column for adding a separate colored / styled line or bar\naccording to this value, and acts just like a separate Y variable,\ncrossed with Y variables."}, {Name: "LegendPosition", Doc: "position of the Legend"}, {Name: "XAxisRotation", Doc: "rotation of the X Axis labels, in degrees"}, {Name: "XAxisLabel", Doc: "optional label to use for XAxis instead of column name"}, {Name: "YAxisLabel", Doc: "optional label to use for YAxis -- if empty, first column name is used"}}}) - -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.ColumnOptions", IDName: "column-options", Doc: "ColumnOptions are options for plotting one column of data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Fields: []types.Field{{Name: "On", Doc: "whether to plot this column"}, {Name: "Column", Doc: "name of column being plotted"}, {Name: "Lines", Doc: "whether to plot lines; uses the overall plot option if unset"}, {Name: "Points", Doc: "whether to plot points with symbols; uses the overall plot option if unset"}, {Name: "LineWidth", Doc: "the width of lines; uses the overall plot option if unset"}, {Name: "PointSize", Doc: "the size of points; uses the overall plot option if unset"}, {Name: "PointShape", Doc: "the shape used to draw points; uses the overall plot option if unset"}, {Name: "Range", Doc: "effective range of data to plot -- either end can be fixed"}, {Name: "FullRange", Doc: "full actual range of data -- only valid if specifically computed"}, {Name: "Color", Doc: "color to use when plotting the line / column"}, {Name: "NTicks", Doc: "desired number of ticks"}, {Name: "Label", Doc: "if specified, this is an alternative label to use when plotting"}, {Name: "TensorIndex", Doc: "if column has n-dimensional tensor cells in each row, this is the index within each cell to plot -- use -1 to plot *all* indexes as separate lines"}, {Name: "ErrColumn", Doc: "specifies a column containing error bars for this column"}, {Name: "IsString", Doc: "if true this is a string column -- plots as labels"}}}) - var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plot", IDName: "plot", Doc: "Plot is a widget that renders a [plot.Plot] object.\nIf it is not [states.ReadOnly], the user can pan and zoom the graph.\nSee [PlotEditor] for an interactive interface for selecting columns to view.", Embeds: []types.Field{{Name: "WidgetBase"}}, Fields: []types.Field{{Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Plot", Doc: "Plot is the Plot to display in this widget"}, {Name: "SetRangesFunc", Doc: "SetRangesFunc, if set, is called to adjust the data ranges\nafter the point when these ranges are updated based on the plot data."}}}) // NewPlot returns a new [Plot] with the given optional parent: @@ -31,7 +28,7 @@ func (t *Plot) SetScale(v float32) *Plot { t.Scale = v; return t } // after the point when these ranges are updated based on the plot data. func (t *Plot) SetSetRangesFunc(v func()) *Plot { t.SetRangesFunc = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Table] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "Options", Doc: "Options are the overall plot options."}, {Name: "Columns", Doc: "Columns are the options for each column of the table."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Table] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "makeColumns", Doc: "makeColumns makes the Plans for columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"p"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "PlotStyle", Doc: "PlotStyle has the overall plot style parameters."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) // NewPlotEditor returns a new [PlotEditor] with the given optional parent: // PlotEditor is a widget that provides an interactive 2D plot @@ -40,6 +37,18 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotE // The user can change various options for the plot and also modify the underlying data. func NewPlotEditor(parent ...tree.Node) *PlotEditor { return tree.New[PlotEditor](parent...) } -// SetOptions sets the [PlotEditor.Options]: -// Options are the overall plot options. -func (t *PlotEditor) SetOptions(v PlotOptions) *PlotEditor { t.Options = v; return t } +// SetPlotStyle sets the [PlotEditor.PlotStyle]: +// PlotStyle has the overall plot style parameters. +func (t *PlotEditor) SetPlotStyle(v plot.PlotStyle) *PlotEditor { t.PlotStyle = v; return t } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotterChooser", IDName: "plotter-chooser", Doc: "PlotterChooser represents a [Plottername] value with a [core.Chooser]\nfor selecting a plotter.", Embeds: []types.Field{{Name: "Chooser"}}, Fields: []types.Field{{Name: "PlotterName"}}}) + +// NewPlotterChooser returns a new [PlotterChooser] with the given optional parent: +// PlotterChooser represents a [Plottername] value with a [core.Chooser] +// for selecting a plotter. +func NewPlotterChooser(parent ...tree.Node) *PlotterChooser { + return tree.New[PlotterChooser](parent...) +} + +// SetPlotterName sets the [PlotterChooser.PlotterName] +func (t *PlotterChooser) SetPlotterName(v string) *PlotterChooser { t.PlotterName = v; return t } diff --git a/plot/plotcore/xyplot.go b/plot/plotcore/xyplot.go index 9050f15a00..05612065c1 100644 --- a/plot/plotcore/xyplot.go +++ b/plot/plotcore/xyplot.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build not + package plotcore import ( diff --git a/plot/plots/labels.go b/plot/plots/labels.go index 1ecde00e18..d377804e03 100644 --- a/plot/plots/labels.go +++ b/plot/plots/labels.go @@ -104,9 +104,10 @@ func (lb *Labels) Plot(plt *plot.Plot) { lb.PY = plot.PlotY(plt, lb.Y) st := &lb.Style.Text st.Offset.ToDots(uc) - st.ToDots(uc) var ltxt plot.Text + ltxt.Defaults() ltxt.Style = *st + ltxt.ToDots(uc) for i, label := range lb.Labels { if label == "" { continue diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 6895ba7f1f..0349c988e7 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -103,7 +103,7 @@ func ExampleTable() { s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.Off) } plot.SetStylersTo(ty, plot.Stylers{genst, func(s *plot.Style) { - s.On = plot.On + s.On = true s.Plotter = "XY" s.Role = plot.Y s.Line.Color = colors.Uniform(colors.Red) @@ -114,12 +114,12 @@ func ExampleTable() { s.Role = plot.X }}) plot.SetStylersTo(th, plot.Stylers{func(s *plot.Style) { - s.On = plot.On + s.On = true s.Plotter = "YErrorBars" s.Role = plot.High }}) plot.SetStylersTo(lbls, plot.Stylers{func(s *plot.Style) { - s.On = plot.On + s.On = true s.Plotter = "Labels" s.Role = plot.Label s.Text.Offset.X.Dp(6) @@ -526,7 +526,7 @@ func TestTable(t *testing.T) { s.Range.SetMin(0).SetMax(100) } plot.SetStylersTo(ty, plot.Stylers{genst, func(s *plot.Style) { - s.On = plot.On + s.On = true s.Role = plot.Y s.Group = "Y" }}) @@ -540,7 +540,7 @@ func TestTable(t *testing.T) { s.Group = "Y" }}) plot.SetStylersTo(th, plot.Stylers{genst, func(s *plot.Style) { - s.On = plot.On + s.On = true s.Role = plot.High s.Group = "Y" }}) @@ -553,7 +553,7 @@ func TestTable(t *testing.T) { s.Group = "Y" }}) plot.SetStylersTo(lbls, plot.Stylers{genst, func(s *plot.Style) { - s.On = plot.On + s.On = true s.Role = plot.Label s.Group = "Y" }}) diff --git a/plot/plotter.go b/plot/plotter.go index 3d59874068..5d2bbfa4cf 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -52,6 +52,9 @@ type PlotterType struct { New func(data Data) Plotter } +// PlotterName is the name of a specific plotter type. +type PlotterName string + // Plotters is the registry of [Plotter] types. var Plotters = map[string]PlotterType{} diff --git a/plot/style.go b/plot/style.go index 529586ff1a..b0eb91292f 100644 --- a/plot/style.go +++ b/plot/style.go @@ -17,14 +17,14 @@ type Style struct { //types:add -setters // Plot has overall plot-level properties, which can be set by any // plot element, and are updated first, before applying element-wise styles. - Plot PlotStyle + Plot PlotStyle `display:"-"` // On specifies whether to plot this item, for table-based plots. - On DefaultOffOn + On bool // Plotter is the type of plotter to use in plotting this data, // for table-based plots. Blank means use default ([plots.XY] is overall default). - Plotter string + Plotter PlotterName // Role specifies a role for this item, used for table-based plots to indicate // how a particular column of data should be used. @@ -44,16 +44,16 @@ type Style struct { //types:add -setters NTicks int // Line has style properties for drawing lines. - Line LineStyle + Line LineStyle `display:"add-fields"` // Point has style properties for drawing points. - Point PointStyle + Point PointStyle `display:"add-fields"` // Text has style properties for rendering text. - Text TextStyle + Text TextStyle `display:"add-fields"` // Width has various plot width properties. - Width WidthStyle + Width WidthStyle `display:"inline"` } // NewStyle returns a new Style object with defaults applied. diff --git a/plot/table.go b/plot/table.go index 2158fe8ecd..4452707ab8 100644 --- a/plot/table.go +++ b/plot/table.go @@ -52,7 +52,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { for ci, cl := range dt.Columns.Values { cnm := dt.Columns.Keys[ci] st := csty[cl] - if st == nil || st.On != On || st.Role == X { + if st == nil || !st.On || st.Role == X { continue } gp := st.Group @@ -64,7 +64,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { } ptyp := "XY" if st.Plotter != "" { - ptyp = st.Plotter + ptyp = string(st.Plotter) } pt, err := PlotterByType(ptyp) if err != nil { diff --git a/plot/text.go b/plot/text.go index 98f65a99d3..040e6f0cd1 100644 --- a/plot/text.go +++ b/plot/text.go @@ -5,6 +5,8 @@ package plot import ( + "image" + "cogentcore.org/core/colors" "cogentcore.org/core/math32" "cogentcore.org/core/paint" @@ -16,17 +18,27 @@ import ( // if not set, the standard Cogent Core default font is used. var DefaultFontFamily = "" -// TextStyle specifies styling parameters for Text elements +// TextStyle specifies styling parameters for Text elements. type TextStyle struct { //types:add -setters - styles.FontRender + // Size of font to render. Default is 16dp + Size units.Value + + // Family name for font (inherited): ordered list of comma-separated names + // from more general to more specific to use. Use split on, to parse. + Family string - // how to align text along the relevant dimension for the text element + // Color of text. + Color image.Image + + // Align specifies how to align text along the relevant + // dimension for the text element. Align styles.Aligns - // Padding is used in a case-dependent manner to add space around text elements + // Padding is used in a case-dependent manner to add + // space around text elements. Padding units.Value - // rotation of the text, in Degrees + // Rotation of the text, in degrees. Rotation float32 // Offset is added directly to the final label location. @@ -34,25 +46,14 @@ type TextStyle struct { //types:add -setters } func (ts *TextStyle) Defaults() { - ts.FontRender.Defaults() + ts.Size.Dp(16) ts.Color = colors.Scheme.OnSurface ts.Align = styles.Center if DefaultFontFamily != "" { - ts.FontRender.Family = DefaultFontFamily - } -} - -func (ts *TextStyle) openFont(pt *Plot) { - if ts.Font.Face == nil { - paint.OpenFont(&ts.FontRender, &pt.Paint.UnitContext) // calls SetUnContext after updating metrics + ts.Family = DefaultFontFamily } } -func (ts *TextStyle) ToDots(uc *units.Context) { - ts.FontRender.ToDots(uc) - ts.Padding.ToDots(uc) -} - // Text specifies a single text element in a plot type Text struct { @@ -62,6 +63,9 @@ type Text struct { // styling for this text element Style TextStyle + // font has the full font rendering styles. + font styles.FontRender + // PaintText is the [paint.Text] for the text. PaintText paint.Text } @@ -73,7 +77,10 @@ func (tx *Text) Defaults() { // config is called during the layout of the plot, prior to drawing func (tx *Text) Config(pt *Plot) { uc := &pt.Paint.UnitContext - fs := &tx.Style.FontRender + fs := &tx.font + fs.Size = tx.Style.Size + fs.Family = tx.Style.Family + fs.Color = tx.Style.Color if math32.Abs(tx.Style.Rotation) > 10 { tx.Style.Align = styles.End } @@ -91,6 +98,17 @@ func (tx *Text) Config(pt *Plot) { } } +func (tx *Text) openFont(pt *Plot) { + if tx.font.Face == nil { + paint.OpenFont(&tx.font, &pt.Paint.UnitContext) // calls SetUnContext after updating metrics + } +} + +func (tx *Text) ToDots(uc *units.Context) { + tx.font.ToDots(uc) + tx.Style.Padding.ToDots(uc) +} + // PosX returns the starting position for a horizontally-aligned text element, // based on given width. Text must have been config'd already. func (tx *Text) PosX(width float32) math32.Vector2 { diff --git a/plot/typegen.go b/plot/typegen.go index 2025794cbb..978ebbbfd2 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -244,6 +244,8 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Plotter", IDNa var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotterType", IDName: "plotter-type", Doc: "PlotterType registers a Plotter so that it can be created with appropriate data.", Fields: []types.Field{{Name: "Name", Doc: "Name of the plot type."}, {Name: "Doc", Doc: "Doc is the documentation for this Plotter."}, {Name: "Required", Doc: "Required Data roles for this plot. Data for these Roles must be provided."}, {Name: "Optional", Doc: "Optional Data roles for this plot."}, {Name: "New", Doc: "New returns a new plotter of this type with given data in given roles."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PlotterName", IDName: "plotter-name", Doc: "PlotterName is the name of a specific plotter type."}) + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.PointStyle", IDName: "point-style", Doc: "PointStyle has style properties for drawing points as different shapes.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "On", Doc: "On indicates whether to plot points."}, {Name: "Shape", Doc: "Shape to draw."}, {Name: "Color", Doc: "Color is the stroke color image specification.\nSetting to nil turns line off."}, {Name: "Fill", Doc: "Fill is the color to fill solid regions, in a plot-specific\nway (e.g., the area below a Line plot, the bar color).\nUse nil to disable filling."}, {Name: "Width", Doc: "Width is the line width for point glyphs, with a default of 1 Pt (point).\nSetting to 0 turns line off."}, {Name: "Size", Doc: "Size of shape to draw for each point.\nDefaults to 4 Pt (point)."}}}) // SetOn sets the [PointStyle.On]: @@ -288,12 +290,12 @@ func (t *Style) SetPlot(v PlotStyle) *Style { t.Plot = v; return t } // SetOn sets the [Style.On]: // On specifies whether to plot this item, for table-based plots. -func (t *Style) SetOn(v DefaultOffOn) *Style { t.On = v; return t } +func (t *Style) SetOn(v bool) *Style { t.On = v; return t } // SetPlotter sets the [Style.Plotter]: // Plotter is the type of plotter to use in plotting this data, // for table-based plots. Blank means use default ([plots.XY] is overall default). -func (t *Style) SetPlotter(v string) *Style { t.Plotter = v; return t } +func (t *Style) SetPlotter(v PlotterName) *Style { t.Plotter = v; return t } // SetRole sets the [Style.Role]: // Role specifies a role for this item, used for table-based plots to indicate @@ -367,25 +369,40 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Stylers", IDNa var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.DefaultOffOn", IDName: "default-off-on", Doc: "DefaultOffOn specifies whether to use the default value for a bool option,\nor to override the default and set Off or On."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TextStyle", IDName: "text-style", Doc: "TextStyle specifies styling parameters for Text elements", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Embeds: []types.Field{{Name: "FontRender"}}, Fields: []types.Field{{Name: "Align", Doc: "how to align text along the relevant dimension for the text element"}, {Name: "Padding", Doc: "Padding is used in a case-dependent manner to add space around text elements"}, {Name: "Rotation", Doc: "rotation of the text, in Degrees"}, {Name: "Offset", Doc: "Offset is added directly to the final label location."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.TextStyle", IDName: "text-style", Doc: "TextStyle specifies styling parameters for Text elements.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Size", Doc: "Size of font to render. Default is 16dp"}, {Name: "Family", Doc: "Family name for font (inherited): ordered list of comma-separated names\nfrom more general to more specific to use. Use split on, to parse."}, {Name: "Color", Doc: "Color of text."}, {Name: "Align", Doc: "Align specifies how to align text along the relevant\ndimension for the text element."}, {Name: "Padding", Doc: "Padding is used in a case-dependent manner to add\nspace around text elements."}, {Name: "Rotation", Doc: "Rotation of the text, in degrees."}, {Name: "Offset", Doc: "Offset is added directly to the final label location."}}}) + +// SetSize sets the [TextStyle.Size]: +// Size of font to render. Default is 16dp +func (t *TextStyle) SetSize(v units.Value) *TextStyle { t.Size = v; return t } + +// SetFamily sets the [TextStyle.Family]: +// Family name for font (inherited): ordered list of comma-separated names +// from more general to more specific to use. Use split on, to parse. +func (t *TextStyle) SetFamily(v string) *TextStyle { t.Family = v; return t } + +// SetColor sets the [TextStyle.Color]: +// Color of text. +func (t *TextStyle) SetColor(v image.Image) *TextStyle { t.Color = v; return t } // SetAlign sets the [TextStyle.Align]: -// how to align text along the relevant dimension for the text element +// Align specifies how to align text along the relevant +// dimension for the text element. func (t *TextStyle) SetAlign(v styles.Aligns) *TextStyle { t.Align = v; return t } // SetPadding sets the [TextStyle.Padding]: -// Padding is used in a case-dependent manner to add space around text elements +// Padding is used in a case-dependent manner to add +// space around text elements. func (t *TextStyle) SetPadding(v units.Value) *TextStyle { t.Padding = v; return t } // SetRotation sets the [TextStyle.Rotation]: -// rotation of the text, in Degrees +// Rotation of the text, in degrees. func (t *TextStyle) SetRotation(v float32) *TextStyle { t.Rotation = v; return t } // SetOffset sets the [TextStyle.Offset]: // Offset is added directly to the final label location. func (t *TextStyle) SetOffset(v units.XY) *TextStyle { t.Offset = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Text", IDName: "text", Doc: "Text specifies a single text element in a plot", Fields: []types.Field{{Name: "Text", Doc: "text string, which can use HTML formatting"}, {Name: "Style", Doc: "styling for this text element"}, {Name: "PaintText", Doc: "PaintText is the [paint.Text] for the text."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Text", IDName: "text", Doc: "Text specifies a single text element in a plot", Fields: []types.Field{{Name: "Text", Doc: "text string, which can use HTML formatting"}, {Name: "Style", Doc: "styling for this text element"}, {Name: "font", Doc: "font has the full font rendering styles."}, {Name: "PaintText", Doc: "PaintText is the [paint.Text] for the text."}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Tick", IDName: "tick", Doc: "A Tick is a single tick mark on an axis.", Fields: []types.Field{{Name: "Value", Doc: "Value is the data value marked by this Tick."}, {Name: "Label", Doc: "Label is the text to display at the tick mark.\nIf Label is an empty string then this is a minor tick mark."}}}) diff --git a/styles/font.go b/styles/font.go index 34bc3cd804..3712adc600 100644 --- a/styles/font.go +++ b/styles/font.go @@ -21,22 +21,24 @@ import ( // for rendering -- see [FontRender] for that. type Font struct { //types:add - // size of font to render (inherited); converted to points when getting font to use + // Size of font to render (inherited). + // Converted to points when getting font to use. Size units.Value - // font family (inherited): ordered list of comma-separated names from more general to more specific to use; use split on , to parse + // Family name for font (inherited): ordered list of comma-separated names + // from more general to more specific to use. Use split on, to parse. Family string - // style (inherited): normal, italic, etc + // Style (inherited): normal, italic, etc. Style FontStyles - // weight (inherited): normal, bold, etc + // Weight (inherited): normal, bold, etc. Weight FontWeights - // font stretch / condense options (inherited) + // Stretch / condense options (inherited). Stretch FontStretch - // normal or small caps (inherited) + // Variant specifies normal or small caps (inherited). Variant FontVariants // Decoration contains the bit flag [TextDecorations] @@ -45,15 +47,16 @@ type Font struct { //types:add // It is not inherited. Decoration TextDecorations - // super / sub script (not inherited) + // Shift is the super / sub script (not inherited). Shift BaselineShifts - // full font information including enhanced metrics and actual font codes for drawing text; this is a pointer into FontLibrary of loaded fonts + // Face has full font information including enhanced metrics and actual + // font codes for drawing text; this is a pointer into FontLibrary of loaded fonts. Face *FontFace `display:"-"` } func (fs *Font) Defaults() { - fs.Size = units.Dp(16) + fs.Size.Dp(16) } // InheritFields from parent From 22c985b7df7e9d6724267ccda8ecd118aff1f02e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 12:52:26 -0800 Subject: [PATCH 285/311] ploteditor: CopyFields is working to set changed values, but color copying is bad. maybe just copy the whole damn thing afterall? --- base/reflectx/pointers_test.go | 8 +- base/reflectx/structs.go | 51 +++++++++++ base/reflectx/structs_test.go | 21 +++++ plot/plotcore/ploteditor.go | 151 ++++++++++++++++++++------------ plot/plotcore/plotterchooser.go | 9 +- plot/plotcore/typegen.go | 5 +- 6 files changed, 172 insertions(+), 73 deletions(-) diff --git a/base/reflectx/pointers_test.go b/base/reflectx/pointers_test.go index f9169b4010..bb062d84e9 100644 --- a/base/reflectx/pointers_test.go +++ b/base/reflectx/pointers_test.go @@ -28,14 +28,14 @@ func InitPointerTest() { pt.Mbr2 = 2 } -func FieldValue(obj any, fld reflect.StructField) reflect.Value { +func fieldValue(obj any, fld reflect.StructField) reflect.Value { ov := reflect.ValueOf(obj) f := unsafe.Pointer(ov.Pointer() + fld.Offset) nw := reflect.NewAt(fld.Type, f) return nw } -func SubFieldValue(obj any, fld reflect.StructField, sub reflect.StructField) reflect.Value { +func subFieldValue(obj any, fld reflect.StructField, sub reflect.StructField) reflect.Value { ov := reflect.ValueOf(obj) f := unsafe.Pointer(ov.Pointer() + fld.Offset + sub.Offset) nw := reflect.NewAt(sub.Type, f) @@ -47,7 +47,7 @@ func TestNewAt(t *testing.T) { InitPointerTest() typ := reflect.TypeOf(pt) fld, _ := typ.FieldByName("Mbr2") - vf := FieldValue(&pt, fld) + vf := fieldValue(&pt, fld) // fmt.Printf("Fld: %v Typ: %v vf: %v vfi: %v vfT: %v vfp: %v canaddr: %v canset: %v caninterface: %v\n", fld.Name, vf.Type().String(), vf.String(), vf.Interface(), vf.Interface(), vf.Interface(), vf.CanAddr(), vf.CanSet(), vf.CanInterface()) @@ -58,7 +58,7 @@ func TestNewAt(t *testing.T) { } fld, _ = typ.FieldByName("Mbr1") - vf = FieldValue(&pt, fld) + vf = fieldValue(&pt, fld) // fmt.Printf("Fld: %v Typ: %v vf: %v vfi: %v vfT: %v vfp: %v canaddr: %v canset: %v caninterface: %v\n", fld.Name, vf.Type().String(), vf.String(), vf.Interface(), vf.Interface(), vf.Interface(), vf.CanAddr(), vf.CanSet(), vf.CanInterface()) diff --git a/base/reflectx/structs.go b/base/reflectx/structs.go index 7cc05c30a3..69a4943642 100644 --- a/base/reflectx/structs.go +++ b/base/reflectx/structs.go @@ -232,3 +232,54 @@ func StructTags(tags reflect.StructTag) map[string]string { func StringJSON(v any) string { return string(errors.Log1(jsonx.WriteBytesIndent(v))) } + +// FieldValue returns the [reflect.Value] of given field within given struct value, +// where the field can be a path with . separators, for fields within struct fields. +func FieldValue(s reflect.Value, fieldPath string) (reflect.Value, error) { + sv := UnderlyingPointer(s) + var zv reflect.Value + if sv.Elem().Kind() != reflect.Struct { + return zv, errors.New("reflectx.FieldValue: kind is not struct") + } + fps := strings.Split(fieldPath, ".") + fv := sv.Elem().FieldByName(fps[0]) + if fv == zv { + return zv, errors.New("reflectx.FieldValue: field name not found: " + fps[0]) + } + if len(fps) == 1 { + return fv, nil + } + return FieldValue(fv, strings.Join(fps[1:], ".")) +} + +// CopyFields copies the named fields from src struct into dest struct. +// Fields can be paths with . separators for sub-fields of fields. +func CopyFields(dest, src any, fields ...string) error { + dsv := UnderlyingPointer(reflect.ValueOf(dest)) + if dsv.Elem().Kind() != reflect.Struct { + return errors.New("reflectx.CopyFields: destination kind is not struct") + } + ssv := UnderlyingPointer(reflect.ValueOf(src)) + if ssv.Elem().Kind() != reflect.Struct { + return errors.New("reflectx.CopyFields: source kind is not struct") + } + var errs []error + for _, f := range fields { + dfv, err := FieldValue(dsv, f) + if err != nil { + errs = append(errs, err) + continue + } + sfv, err := FieldValue(ssv, f) + if err != nil { + errs = append(errs, err) + continue + } + err = SetRobust(UnderlyingPointer(dfv).Interface(), sfv.Interface()) + if err != nil { + errs = append(errs, err) + continue + } + } + return errors.Join(errs...) +} diff --git a/base/reflectx/structs_test.go b/base/reflectx/structs_test.go index e42b07f175..2fcdb494d3 100644 --- a/base/reflectx/structs_test.go +++ b/base/reflectx/structs_test.go @@ -7,6 +7,8 @@ package reflectx import ( "reflect" "testing" + + "github.com/stretchr/testify/assert" ) type person struct { @@ -53,3 +55,22 @@ func TestNonDefaultFields(t *testing.T) { t.Errorf("expected\n%v\n\tbut got\n%v", want, have) } } + +func TestCopyFields(t *testing.T) { + sp := &person{ + Name: "Go Gopher", + Age: 23, + ProgrammingLanguage: "Go", + FavoriteFruit: "Peach", + Data: "abcdef", + Pet: pet{ + Name: "Pet Gopher", + Type: "Dog", + Age: 7, + }, + } + dp := &person{} + CopyFields(dp, sp, "Name", "Pet.Age") + assert.Equal(t, sp.Name, dp.Name) + assert.Equal(t, sp.Pet.Age, dp.Pet.Age) +} diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 655901ac84..2bfa718c3f 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -12,12 +12,14 @@ import ( "io/fs" "log/slog" "path/filepath" + "slices" "strings" "time" "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/iox/imagex" + "cogentcore.org/core/base/reflectx" "cogentcore.org/core/colors" "cogentcore.org/core/core" "cogentcore.org/core/events" @@ -30,6 +32,7 @@ import ( "cogentcore.org/core/tensor/table" "cogentcore.org/core/tensor/tensorcore" "cogentcore.org/core/tree" + "golang.org/x/exp/maps" ) // PlotEditor is a widget that provides an interactive 2D plot @@ -264,50 +267,6 @@ func (pl *PlotEditor) genPlot() { const plotColumnsHeaderN = 2 -// columnsListUpdate updates the list of columns -// func (pl *PlotEditor) columnsListUpdate() { -// if pl.table == nil { -// return -// } -// dt := pl.table -// nc := dt.NumColumns() -// if nc == len(pl.Columns) { -// return -// } -// pl.Columns = make([]*ColumnOptions, nc) -// clri := 0 -// hasOn := false -// for ci := range dt.NumColumns() { -// cn := dt.ColumnName(ci) -// if pl.Options.XAxis == "" && ci == 0 { -// pl.Options.XAxis = cn // x-axis defaults to the first column -// } -// cp := &ColumnOptions{Column: cn} -// cp.Defaults() -// pl.Stylers.ApplyToColumn(cp) -// tcol := dt.ColumnByIndex(ci) -// tc := tcol.Tensor -// if tc.IsString() { -// cp.IsString = true -// } else { -// cp.IsString = false -// // we enable the first non-string, non-x-axis, non-first column by default -// if !hasOn && cn != pl.Options.XAxis && ci != 0 { -// cp.On = true -// hasOn = true -// } -// } -// cp.fromMetaMap(pl.table.Meta) -// inc := 1 -// if cn == pl.Options.XAxis || tc.IsString() || tc.DataType() == reflect.Int || tc.DataType() == reflect.Int64 || tc.DataType() == reflect.Int32 || tc.DataType() == reflect.Uint8 { -// inc = 0 -// } -// cp.Color = colors.Uniform(colors.Spaced(clri)) -// pl.Columns[ci] = cp -// clri += inc -// } -// } - // setAllColumns turns all Columns on or off (except X axis) func (pl *PlotEditor) setAllColumns(on bool) { // fr := pl.columnsFrame @@ -378,26 +337,19 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { } for ci, cl := range pl.table.Columns.Values { cnm := pl.table.Columns.Keys[ci] - cst := &plot.Style{} - cst.Defaults() psty := plot.GetStylersFrom(cl) - // nsty := 0 - if psty != nil { - // nsty = len(psty) - psty.Run(cst) - } - mods := map[string]bool{} - // metadata.SetTo(cl, "PlotEditorNStylers", nsty) - // metadata.SetTo(cl, "PlotEditorModsMap", mods) + cst, mods := pl.defaultColumnStyle(ci, cl, psty) updateStyle := func() { if len(mods) == 0 { return } + mf := modFields(mods) sty := psty sty = append(sty, func(s *plot.Style) { + errors.Log(reflectx.CopyFields(s, cst, mf...)) }) + plot.SetStylersTo(cl, sty) } - _ = updateStyle tree.AddAt(p, cnm, func(w *core.Frame) { w.Styler(func(s *styles.Style) { s.CenterAll() @@ -408,6 +360,7 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { w.OnChange(func(e events.Event) { mods["On"] = true cst.On = w.IsChecked() + updateStyle() pl.UpdatePlot() }) w.Updater(func() { @@ -424,6 +377,7 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { w.SetText(cnm).SetType(core.ButtonAction).SetTooltip("Edit all styling options for this column") w.OnClick(func(e events.Event) { update := func() { + updateStyle() if core.TheApp.Platform().IsMobile() { pl.Update() return @@ -438,8 +392,6 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { fm := core.NewForm(d).SetStruct(cst) fm.Modified = mods fm.OnChange(func(e events.Event) { - // todo: compile style based on changed items - fmt.Println(mods) update() }) // d.AddTopBar(func(bar *core.Frame) { @@ -465,6 +417,91 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { } } +// defaultColumnStyle initializes the column style with any existing stylers +// plus additional general defaults, returning the initially modified field names. +func (pl *PlotEditor) defaultColumnStyle(ci int, cl tensor.Values, psty plot.Stylers) (*plot.Style, map[string]bool) { + cst := &plot.Style{} + cst.Defaults() + if psty != nil { + psty.Run(cst) + } + mods := map[string]bool{} + // todo: qualify based on type? + if cst.Plotter == "" { + cst.Plotter = plot.PlotterName("XY") + mods["Plotter"] = true + } + if cst.Role == plot.NoRole { + cst.Role = plot.Y + mods["Role"] = true + } + // if cst.Line.Color == nil { // todo: detect default + // todo: skip over non-float types here + cst.Line.Color = colors.Uniform(colors.Spaced(ci)) + mods["Line.Color"] = true + // } + return cst, mods +} + +// columnsListUpdate updates the list of columns +// func (pl *PlotEditor) columnsListUpdate() { +// if pl.table == nil { +// return +// } +// dt := pl.table +// nc := dt.NumColumns() +// if nc == len(pl.Columns) { +// return +// } +// pl.Columns = make([]*ColumnOptions, nc) +// clri := 0 +// hasOn := false +// for ci := range dt.NumColumns() { +// cn := dt.ColumnName(ci) +// if pl.Options.XAxis == "" && ci == 0 { +// pl.Options.XAxis = cn // x-axis defaults to the first column +// } +// cp := &ColumnOptions{Column: cn} +// cp.Defaults() +// pl.Stylers.ApplyToColumn(cp) +// tcol := dt.ColumnByIndex(ci) +// tc := tcol.Tensor +// if tc.IsString() { +// cp.IsString = true +// } else { +// cp.IsString = false +// // we enable the first non-string, non-x-axis, non-first column by default +// if !hasOn && cn != pl.Options.XAxis && ci != 0 { +// cp.On = true +// hasOn = true +// } +// } +// cp.fromMetaMap(pl.table.Meta) +// inc := 1 +// if cn == pl.Options.XAxis || tc.IsString() || tc.DataType() == reflect.Int || tc.DataType() == reflect.Int64 || tc.DataType() == reflect.Int32 || tc.DataType() == reflect.Uint8 { +// inc = 0 +// } +// cp.Color = colors.Uniform(colors.Spaced(clri)) +// pl.Columns[ci] = cp +// clri += inc +// } +// } + +// modFields returns the modified fields as field paths using . separators +func modFields(mods map[string]bool) []string { + fns := maps.Keys(mods) + rf := make([]string, 0, len(fns)) + for _, f := range fns { + if mods[f] == false { + continue + } + fc := strings.ReplaceAll(f, " • ", ".") + rf = append(rf, fc) + } + slices.Sort(rf) + return rf +} + func (pl *PlotEditor) MakeToolbar(p *tree.Plan) { if pl.table == nil { return diff --git a/plot/plotcore/plotterchooser.go b/plot/plotcore/plotterchooser.go index e46372112b..bbc121928a 100644 --- a/plot/plotcore/plotterchooser.go +++ b/plot/plotcore/plotterchooser.go @@ -5,7 +5,6 @@ package plotcore import ( - "reflect" "slices" "cogentcore.org/core/core" @@ -22,17 +21,11 @@ func init() { // for selecting a plotter. type PlotterChooser struct { core.Chooser - PlotterName string -} - -func (fc *PlotterChooser) WidgetValue() any { return &fc.PlotterName } - -func (fc *PlotterChooser) OnBind(value any, tags reflect.StructTag) { } func (fc *PlotterChooser) Init() { fc.Chooser.Init() pnms := maps.Keys(plot.Plotters) slices.Sort(pnms) - fc.Chooser.SetStrings(pnms...) + fc.SetStrings(pnms...) } diff --git a/plot/plotcore/typegen.go b/plot/plotcore/typegen.go index 8ea5a24262..c270c56ffd 100644 --- a/plot/plotcore/typegen.go +++ b/plot/plotcore/typegen.go @@ -41,7 +41,7 @@ func NewPlotEditor(parent ...tree.Node) *PlotEditor { return tree.New[PlotEditor // PlotStyle has the overall plot style parameters. func (t *PlotEditor) SetPlotStyle(v plot.PlotStyle) *PlotEditor { t.PlotStyle = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotterChooser", IDName: "plotter-chooser", Doc: "PlotterChooser represents a [Plottername] value with a [core.Chooser]\nfor selecting a plotter.", Embeds: []types.Field{{Name: "Chooser"}}, Fields: []types.Field{{Name: "PlotterName"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotterChooser", IDName: "plotter-chooser", Doc: "PlotterChooser represents a [Plottername] value with a [core.Chooser]\nfor selecting a plotter.", Embeds: []types.Field{{Name: "Chooser"}}}) // NewPlotterChooser returns a new [PlotterChooser] with the given optional parent: // PlotterChooser represents a [Plottername] value with a [core.Chooser] @@ -49,6 +49,3 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plott func NewPlotterChooser(parent ...tree.Node) *PlotterChooser { return tree.New[PlotterChooser](parent...) } - -// SetPlotterName sets the [PlotterChooser.PlotterName] -func (t *PlotterChooser) SetPlotterName(v string) *PlotterChooser { t.PlotterName = v; return t } From ee023f47a6582f07d27832a0962ecfa8ff8b3978 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 16:02:55 -0800 Subject: [PATCH 286/311] reflectx image pointer copying now working properly -- CopyFields needs to explicitly get the pointer value to the fields. --- base/reflectx/structs.go | 2 +- base/reflectx/structs_test.go | 13 +++++++++++++ base/reflectx/values.go | 9 +++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/base/reflectx/structs.go b/base/reflectx/structs.go index 69a4943642..567acd2106 100644 --- a/base/reflectx/structs.go +++ b/base/reflectx/structs.go @@ -275,7 +275,7 @@ func CopyFields(dest, src any, fields ...string) error { errs = append(errs, err) continue } - err = SetRobust(UnderlyingPointer(dfv).Interface(), sfv.Interface()) + err = SetRobust(PointerValue(dfv).Interface(), sfv.Interface()) if err != nil { errs = append(errs, err) continue diff --git a/base/reflectx/structs_test.go b/base/reflectx/structs_test.go index 2fcdb494d3..622c50d9fb 100644 --- a/base/reflectx/structs_test.go +++ b/base/reflectx/structs_test.go @@ -5,9 +5,11 @@ package reflectx import ( + "image" "reflect" "testing" + "cogentcore.org/core/colors" "github.com/stretchr/testify/assert" ) @@ -56,6 +58,10 @@ func TestNonDefaultFields(t *testing.T) { } } +type imgfield struct { + Mycolor image.Image +} + func TestCopyFields(t *testing.T) { sp := &person{ Name: "Go Gopher", @@ -73,4 +79,11 @@ func TestCopyFields(t *testing.T) { CopyFields(dp, sp, "Name", "Pet.Age") assert.Equal(t, sp.Name, dp.Name) assert.Equal(t, sp.Pet.Age, dp.Pet.Age) + + sif := &imgfield{ + Mycolor: colors.Uniform(colors.Black), + } + dif := &imgfield{} + CopyFields(dif, sif, "Mycolor") + assert.Equal(t, sif.Mycolor, dif.Mycolor) } diff --git a/base/reflectx/values.go b/base/reflectx/values.go index fd54eb10d7..e8c1bd3bbb 100644 --- a/base/reflectx/values.go +++ b/base/reflectx/values.go @@ -961,6 +961,15 @@ func SetRobust(to, from any) error { return fmt.Errorf("destination value cannot be set; it must be a variable or field, not a const or tmp or other value that cannot be set (value: %v of type %T)", pointer, pointer) } + // images should not be copied per content: just set the pointer! + // otherwise the original images (esp colors!) are altered. + if img, ok := to.(*image.Image); ok { + if fimg, ok := from.(image.Image); ok { + *img = fimg + return nil + } + } + // first we do the generic AssignableTo case if v.Kind() == reflect.Pointer { fv := reflect.ValueOf(from) From 915f134a0bcede9906b6759240cb52f436f3739f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 16:29:51 -0800 Subject: [PATCH 287/311] ploteditor default styling reasonable --- plot/plotcore/ploteditor.go | 41 +++++++++++++++++++++++++++---------- plot/table.go | 13 +++++++++--- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 2bfa718c3f..29295ad51c 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -25,6 +25,7 @@ import ( "cogentcore.org/core/events" "cogentcore.org/core/icons" "cogentcore.org/core/plot" + "cogentcore.org/core/plot/plots" "cogentcore.org/core/styles" "cogentcore.org/core/styles/states" "cogentcore.org/core/system" @@ -335,10 +336,11 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { if pl.table == nil { return } + colorIdx := 0 // index for color sequence -- skips various types for ci, cl := range pl.table.Columns.Values { cnm := pl.table.Columns.Keys[ci] psty := plot.GetStylersFrom(cl) - cst, mods := pl.defaultColumnStyle(ci, cl, psty) + cst, mods := pl.defaultColumnStyle(cl, ci, &colorIdx, psty) updateStyle := func() { if len(mods) == 0 { return @@ -350,6 +352,7 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { }) plot.SetStylersTo(cl, sty) } + updateStyle() tree.AddAt(p, cnm, func(w *core.Frame) { w.Styler(func(s *styles.Style) { s.CenterAll() @@ -419,27 +422,43 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { // defaultColumnStyle initializes the column style with any existing stylers // plus additional general defaults, returning the initially modified field names. -func (pl *PlotEditor) defaultColumnStyle(ci int, cl tensor.Values, psty plot.Stylers) (*plot.Style, map[string]bool) { +func (pl *PlotEditor) defaultColumnStyle(cl tensor.Values, ci int, colorIdx *int, psty plot.Stylers) (*plot.Style, map[string]bool) { cst := &plot.Style{} cst.Defaults() if psty != nil { psty.Run(cst) } mods := map[string]bool{} - // todo: qualify based on type? + isfloat := reflectx.KindIsFloat(cl.DataType()) if cst.Plotter == "" { - cst.Plotter = plot.PlotterName("XY") - mods["Plotter"] = true + if isfloat { + cst.Plotter = plots.XYType + mods["Plotter"] = true + } else if cl.IsString() { + cst.Plotter = plot.PlotterName(plots.LabelsType) + mods["Plotter"] = true + } } if cst.Role == plot.NoRole { - cst.Role = plot.Y mods["Role"] = true + if isfloat { + cst.Role = plot.Y + } else if cl.IsString() { + cst.Role = plot.Label + } else { + cst.Role = plot.X + } + } + if cst.Line.Color == colors.Scheme.OnSurface { + if cst.Role == plot.Y && isfloat { + spclr := colors.Uniform(colors.Spaced(*colorIdx)) + cst.Line.Color = spclr + mods["Line.Color"] = true + cst.Point.Color = spclr + mods["Point.Color"] = true + (*colorIdx)++ + } } - // if cst.Line.Color == nil { // todo: detect default - // todo: skip over non-float types here - cst.Line.Color = colors.Uniform(colors.Spaced(ci)) - mods["Line.Color"] = true - // } return cst, mods } diff --git a/plot/table.go b/plot/table.go index 4452707ab8..3cb4fddad0 100644 --- a/plot/table.go +++ b/plot/table.go @@ -31,7 +31,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { } csty := make(map[tensor.Values]*Style, nc) gps := make(map[string][]tensor.Values, nc) - var xt tensor.Values + var xt tensor.Values // get the _last_ role = X column -- most specific counter var errs []error for _, cl := range dt.Columns.Values { st := &Style{} @@ -43,7 +43,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { csty[cl] = st stl.Run(st) gps[st.Group] = append(gps[st.Group], cl) - if xt == nil && st.Role == X { + if st.Role == X { xt = cl } } @@ -82,9 +82,16 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { for _, gc := range gcols { gst := csty[gc] if gst.Role == rl { + if rl == Y { + if !gst.On { + continue + } + } data[rl] = gc got = true - break + if rl != X { // get the last one for X + break + } } } if !got { From b4b827877cc4536758f15d734918c62935b51664 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 20:27:54 -0800 Subject: [PATCH 288/311] overall plot styling working --- examples/plot/plot.go | 1 - plot/plotcore/ploteditor.go | 98 ++++++++++++++++--------------------- 2 files changed, 42 insertions(+), 57 deletions(-) diff --git a/examples/plot/plot.go b/examples/plot/plot.go index 99ada3fe88..0d78671eaf 100644 --- a/examples/plot/plot.go +++ b/examples/plot/plot.go @@ -10,7 +10,6 @@ import ( "cogentcore.org/core/core" "cogentcore.org/core/plot" "cogentcore.org/core/plot/plotcore" - _ "cogentcore.org/core/plot/plots" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 29295ad51c..12c458ed43 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -61,8 +61,9 @@ type PlotEditor struct { //types:add // currently doing a plot inPlot bool - columnsFrame *core.Frame - plotWidget *Plot + columnsFrame *core.Frame + plotWidget *Plot + plotStyleModified map[string]bool } func (pl *PlotEditor) CopyFieldsFrom(frm tree.Node) { @@ -90,6 +91,7 @@ func (pl *PlotEditor) Init() { pl.Frame.Init() pl.PlotStyle.Defaults() + pl.Styler(func(s *styles.Style) { s.Grow.Set(1, 1) if pl.SizeClass() == core.SizeCompact { @@ -101,11 +103,11 @@ func (pl *PlotEditor) Init() { pl.UpdatePlot() }) - // pl.Updater(func() { - // if pl.table != nil { - // pl.Options.fromMeta(pl.table) - // } - // }) + pl.Updater(func() { + if pl.table != nil { + pl.plotStyleFromTable(pl.table) + } + }) tree.AddChildAt(pl, "columns", func(w *core.Frame) { pl.columnsFrame = w w.Styler(func(s *styles.Style) { @@ -349,6 +351,7 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { sty := psty sty = append(sty, func(s *plot.Style) { errors.Log(reflectx.CopyFields(s, cst, mf...)) + errors.Log(reflectx.CopyFields(&s.Plot, &pl.PlotStyle, modFields(pl.plotStyleModified)...)) }) plot.SetStylersTo(cl, sty) } @@ -432,7 +435,7 @@ func (pl *PlotEditor) defaultColumnStyle(cl tensor.Values, ci int, colorIdx *int isfloat := reflectx.KindIsFloat(cl.DataType()) if cst.Plotter == "" { if isfloat { - cst.Plotter = plots.XYType + cst.Plotter = plot.PlotterName(plots.XYType) mods["Plotter"] = true } else if cl.IsString() { cst.Plotter = plot.PlotterName(plots.LabelsType) @@ -456,55 +459,40 @@ func (pl *PlotEditor) defaultColumnStyle(cl tensor.Values, ci int, colorIdx *int mods["Line.Color"] = true cst.Point.Color = spclr mods["Point.Color"] = true + if cst.Plotter == plots.BarType { + cst.Line.Fill = spclr + mods["Line.Fill"] = true + } (*colorIdx)++ } } return cst, mods } -// columnsListUpdate updates the list of columns -// func (pl *PlotEditor) columnsListUpdate() { -// if pl.table == nil { -// return -// } -// dt := pl.table -// nc := dt.NumColumns() -// if nc == len(pl.Columns) { -// return -// } -// pl.Columns = make([]*ColumnOptions, nc) -// clri := 0 -// hasOn := false -// for ci := range dt.NumColumns() { -// cn := dt.ColumnName(ci) -// if pl.Options.XAxis == "" && ci == 0 { -// pl.Options.XAxis = cn // x-axis defaults to the first column -// } -// cp := &ColumnOptions{Column: cn} -// cp.Defaults() -// pl.Stylers.ApplyToColumn(cp) -// tcol := dt.ColumnByIndex(ci) -// tc := tcol.Tensor -// if tc.IsString() { -// cp.IsString = true -// } else { -// cp.IsString = false -// // we enable the first non-string, non-x-axis, non-first column by default -// if !hasOn && cn != pl.Options.XAxis && ci != 0 { -// cp.On = true -// hasOn = true -// } -// } -// cp.fromMetaMap(pl.table.Meta) -// inc := 1 -// if cn == pl.Options.XAxis || tc.IsString() || tc.DataType() == reflect.Int || tc.DataType() == reflect.Int64 || tc.DataType() == reflect.Int32 || tc.DataType() == reflect.Uint8 { -// inc = 0 -// } -// cp.Color = colors.Uniform(colors.Spaced(clri)) -// pl.Columns[ci] = cp -// clri += inc -// } -// } +func (pl *PlotEditor) plotStyleFromTable(dt *table.Table) { + if pl.plotStyleModified != nil { // already set + return + } + pst := &pl.PlotStyle + mods := map[string]bool{} + pl.plotStyleModified = mods + tst := &plot.Style{} + tst.Defaults() + tst.Plot.Defaults() + for _, cl := range pl.table.Columns.Values { + stl := plot.GetStylersFrom(cl) + if stl == nil { + continue + } + stl.Run(tst) + } + *pst = tst.Plot + if pst.PointsOn == plot.Default { + pst.PointsOn = plot.Off + mods["PointsOn"] = true + } + fmt.Println(pl.PlotStyle.Scale) +} // modFields returns the modified fields as field paths using . separators func modFields(mods map[string]bool) []string { @@ -551,15 +539,13 @@ func (pl *PlotEditor) MakeToolbar(p *tree.Plan) { }) }) tree.Add(p, func(w *core.Button) { - w.SetText("Options").SetIcon(icons.Settings). - SetTooltip("Options for how the plot is rendered"). + w.SetText("Style").SetIcon(icons.Settings). + SetTooltip("Style for how the plot is rendered"). OnClick(func(e events.Event) { d := core.NewBody("Plot style") fm := core.NewForm(d).SetStruct(&pl.PlotStyle) - fm.Modified = map[string]bool{} + fm.Modified = pl.plotStyleModified fm.OnChange(func(e events.Event) { - // todo: get modified and make style - fmt.Println(fm.Modified) pl.GoUpdatePlot() }) d.RunWindowDialog(pl) From f3bd93c52fddca09776fd52f9733cf048c8ed388 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 22:42:10 -0800 Subject: [PATCH 289/311] set columns buttons working; style switches with line color --- base/metadata/metadata.go | 22 +++++++++ examples/plot/plot.go | 14 ++---- plot/plotcore/ploteditor.go | 95 ++++++++++++++++++++----------------- 3 files changed, 79 insertions(+), 52 deletions(-) diff --git a/base/metadata/metadata.go b/base/metadata/metadata.go index 9eeea554fc..472561a92d 100644 --- a/base/metadata/metadata.go +++ b/base/metadata/metadata.go @@ -124,3 +124,25 @@ func SetTo(obj any, key string, value any) error { md.Set(key, value) return nil } + +// SetName sets the "Name" standard key. +func SetName(obj any, name string) { + SetTo(obj, "Name", name) +} + +// Name returns the "Name" standard key value (empty if not set). +func Name(obj any) string { + nm, _ := GetFrom[string](obj, "Name") + return nm +} + +// SetDoc sets the "Doc" standard key. +func SetDoc(obj any, doc string) { + SetTo(obj, "Doc", doc) +} + +// Doc returns the "Doc" standard key value (empty if not set). +func Doc(obj any) string { + doc, _ := GetFrom[string](obj, "Doc") + return doc +} diff --git a/examples/plot/plot.go b/examples/plot/plot.go index 0d78671eaf..bff8fe87ce 100644 --- a/examples/plot/plot.go +++ b/examples/plot/plot.go @@ -22,21 +22,17 @@ func main() { epc := table.New("epc") epc.OpenFS(tsv, "ra25epoch.tsv", tensor.Tab) - epcc := epc.Column("Epoch") - plot.SetStylersTo(epcc, plot.Stylers{func(s *plot.Style) { - s.Role = plot.X - }}) + pst := func(s *plot.Style) { + s.Plot.Title = "RA25 Epoch Train" + s.Plot.XAxis.Label = "Epoch" + } perr := epc.Column("PctErr") - plot.SetStylersTo(perr, plot.Stylers{func(s *plot.Style) { + plot.SetStylersTo(perr, plot.Stylers{pst, func(s *plot.Style) { s.On = true s.Role = plot.Y }}) pl := plotcore.NewPlotEditor(b) - // pl.Options.Title = "RA25 Epoch Train" - // pl.Options.XAxis = "Epoch" - // pl.Options.Points = true - // pl.ColumnOptions("UnitErr").On = true pl.SetTable(epc) b.AddTopBar(func(bar *core.Frame) { core.NewToolbar(bar).Maker(pl.MakeToolbar) diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 12c458ed43..e46995f62e 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -19,6 +19,7 @@ import ( "cogentcore.org/core/base/errors" "cogentcore.org/core/base/fsx" "cogentcore.org/core/base/iox/imagex" + "cogentcore.org/core/base/metadata" "cogentcore.org/core/base/reflectx" "cogentcore.org/core/colors" "cogentcore.org/core/core" @@ -268,52 +269,40 @@ func (pl *PlotEditor) genPlot() { pl.inPlot = false } -const plotColumnsHeaderN = 2 - -// setAllColumns turns all Columns on or off (except X axis) -func (pl *PlotEditor) setAllColumns(on bool) { - // fr := pl.columnsFrame - // for i, cli := range fr.Children { - // if i < plotColumnsHeaderN { - // continue - // } - // ci := i - plotColumnsHeaderN - // cp := pl.Columns[ci] - // if cp.Column == pl.Options.XAxis { - // continue - // } - // cp.On = on - // cl := cli.(*core.Frame) - // sw := cl.Child(0).(*core.Switch) - // sw.SetChecked(cp.On) - // } - // pl.UpdatePlot() - // pl.NeedsRender() +const plotColumnsHeaderN = 3 + +// allColumnsOff turns all columns off. +func (pl *PlotEditor) allColumnsOff() { + fr := pl.columnsFrame + for i, cli := range fr.Children { + if i < plotColumnsHeaderN { + continue + } + cl := cli.(*core.Frame) + sw := cl.Child(0).(*core.Switch) + sw.SetChecked(false) + sw.SendChange() + } + pl.Update() } // setColumnsByName turns columns on or off if their name contains // the given string. func (pl *PlotEditor) setColumnsByName(nameContains string, on bool) { //types:add - // fr := pl.columnsFrame - // for i, cli := range fr.Children { - // if i < plotColumnsHeaderN { - // continue - // } - // ci := i - plotColumnsHeaderN - // cp := pl.Columns[ci] - // if cp.Column == pl.Options.XAxis { - // continue - // } - // if !strings.Contains(cp.Column, nameContains) { - // continue - // } - // cp.On = on - // cl := cli.(*core.Frame) - // sw := cl.Child(0).(*core.Switch) - // sw.SetChecked(cp.On) - // } - // pl.UpdatePlot() - // pl.NeedsRender() + fr := pl.columnsFrame + for i, cli := range fr.Children { + if i < plotColumnsHeaderN { + continue + } + cl := cli.(*core.Frame) + if !strings.Contains(cl.Name, nameContains) { + continue + } + sw := cl.Child(0).(*core.Switch) + sw.SetChecked(on) + sw.SendChange() + } + pl.Update() } // makeColumns makes the Plans for columns @@ -323,7 +312,7 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { w.SetText("Clear").SetIcon(icons.ClearAll).SetType(core.ButtonAction) w.SetTooltip("Turn all columns off") w.OnClick(func(e events.Event) { - pl.setAllColumns(false) + pl.allColumnsOff() }) }) tree.AddChild(w, func(w *core.Button) { @@ -362,7 +351,21 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { }) tree.AddChild(w, func(w *core.Switch) { w.SetType(core.SwitchCheckbox).SetTooltip("Turn this column on or off") - // todo: set color according to style + w.Styler(func(s *styles.Style) { + s.Color = cst.Line.Color + }) + tree.AddChildInit(w, "stack", func(w *core.Frame) { + f := func(name string) { + tree.AddChildInit(w, name, func(w *core.Icon) { + w.Styler(func(s *styles.Style) { + s.Color = cst.Line.Color + }) + }) + } + f("icon-on") + f("icon-off") + f("icon-indeterminate") + }) w.OnChange(func(e events.Event) { mods["On"] = true cst.On = w.IsChecked() @@ -491,6 +494,12 @@ func (pl *PlotEditor) plotStyleFromTable(dt *table.Table) { pst.PointsOn = plot.Off mods["PointsOn"] = true } + if pst.Title == "" { + pst.Title = metadata.Name(pl.table) + if pst.Title != "" { + mods["Title"] = true + } + } fmt.Println(pl.PlotStyle.Scale) } From 05fc34866436d4cd0897a85952c3c77917829192 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Tue, 12 Nov 2024 23:43:45 -0800 Subject: [PATCH 290/311] global x axis and auto legend --- examples/plot/plot.go | 1 - plot/legend.go | 12 +++++------- plot/plot.go | 8 ++++---- plot/plotcore/ploteditor.go | 8 ++++++-- plot/plots/plot_test.go | 2 +- plot/plotter.go | 1 + plot/style.go | 17 ++++++++++++----- plot/table.go | 36 +++++++++++++++++++++++++++++++++++- 8 files changed, 64 insertions(+), 21 deletions(-) diff --git a/examples/plot/plot.go b/examples/plot/plot.go index bff8fe87ce..64ff53e62c 100644 --- a/examples/plot/plot.go +++ b/examples/plot/plot.go @@ -24,7 +24,6 @@ func main() { epc.OpenFS(tsv, "ra25epoch.tsv", tensor.Tab) pst := func(s *plot.Style) { s.Plot.Title = "RA25 Epoch Train" - s.Plot.XAxis.Label = "Epoch" } perr := epc.Column("PctErr") plot.SetStylersTo(perr, plot.Stylers{pst, func(s *plot.Style) { diff --git a/plot/legend.go b/plot/legend.go index 695d07edca..bb0cacc049 100644 --- a/plot/legend.go +++ b/plot/legend.go @@ -93,14 +93,12 @@ func (lg *Legend) LegendForPlotter(plt Plotter) string { return "" } -// Thumbnailer wraps the Thumbnail method, which -// draws the small image in a legend representing the -// style of data. +// Thumbnailer wraps the Thumbnail method, which draws the small +// image in a legend representing the style of data. type Thumbnailer interface { - // Thumbnail draws an thumbnail representing - // a legend entry. The thumbnail will usually show - // a smaller representation of the style used - // to plot the corresponding data. + // Thumbnail draws an thumbnail representing a legend entry. + // The thumbnail will usually show a smaller representation + // of the style used to plot the corresponding data. Thumbnail(pt *Plot) } diff --git a/plot/plot.go b/plot/plot.go index 1dc87cba57..6af47b1bd0 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -25,10 +25,10 @@ import ( // XAxisStyle has overall plot level styling properties for the XAxis. type XAxisStyle struct { //types:add -setters - // Column specifies the column to use for the common X axis in a table based plot. - // if empty or not found, the row number is used. - // This optional for Bar plots, if present and Legend is also present, - // then an extra space will be put between X values. + // Column specifies the column to use for the common X axis, + // for [plot.NewTablePlot] [table.Table] driven plots. + // If empty, standard Group-based role binding is used: the last column + // within the same group with Role=X is used. Column string // Rotation is the rotation of the X Axis labels, in degrees. diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index e46995f62e..ba6ea4601b 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -264,7 +264,11 @@ func (pl *PlotEditor) genPlot() { pl.table.Sequential() } } - pl.plot = errors.Log1(plot.NewTablePlot(pl.table)) + var err error + pl.plot, err = plot.NewTablePlot(pl.table) + if err != nil { + core.ErrorSnackbar(pl, err) + } pl.plotWidget.SetPlot(pl.plot) // redraws etc pl.inPlot = false } @@ -397,7 +401,7 @@ func (pl *PlotEditor) makeColumns(p *tree.Plan) { pl.Update() pl.AsyncUnlock() } - d := core.NewBody("Column " + cnm + " style properties") + d := core.NewBody(cnm + " style properties") fm := core.NewForm(d).SetStruct(cst) fm.Modified = mods fm.OnChange(func(e events.Event) { diff --git a/plot/plots/plot_test.go b/plot/plots/plot_test.go index 0349c988e7..3026a5737e 100644 --- a/plot/plots/plot_test.go +++ b/plot/plots/plot_test.go @@ -518,7 +518,7 @@ func TestTable(t *testing.T) { s.Plot.Title = "Test " + ttyp s.Plot.XAxis.Label = "X Axis" s.Plot.YAxisLabel = "Y Axis" - s.Plotter = ttyp + s.Plotter = plot.PlotterName(ttyp) s.Plot.Scale = 2 s.Plot.SetLinesOn(plot.On).SetPointsOn(plot.On) s.Line.Color = colors.Uniform(colors.Red) diff --git a/plot/plotter.go b/plot/plotter.go index 5d2bbfa4cf..6a74c431ed 100644 --- a/plot/plotter.go +++ b/plot/plotter.go @@ -14,6 +14,7 @@ import ( // Plotter is an interface that wraps the Plot method. // Standard implementations of Plotter are in the [plots] package. type Plotter interface { + // Plot draws the data to the Plot Paint. Plot(pt *Plot) diff --git a/plot/style.go b/plot/style.go index b0eb91292f..30ce167d25 100644 --- a/plot/style.go +++ b/plot/style.go @@ -23,15 +23,17 @@ type Style struct { //types:add -setters On bool // Plotter is the type of plotter to use in plotting this data, - // for table-based plots. Blank means use default ([plots.XY] is overall default). + // for [plot.NewTablePlot] [table.Table] driven plots. + // Blank means use default ([plots.XY] is overall default). Plotter PlotterName - // Role specifies a role for this item, used for table-based plots to indicate - // how a particular column of data should be used. + // Role specifies how a particular column of data should be used, + // for [plot.NewTablePlot] [table.Table] driven plots. Role Roles - // Group specifies a group of related data items, used for table-based plots - // where different columns of data within the same Group play different Roles + // Group specifies a group of related data items, + // for [plot.NewTablePlot] [table.Table] driven plots, + // where different columns of data within the same Group play different Roles. Group string // Range is the effective range of data to plot, where either end can be fixed. @@ -40,6 +42,11 @@ type Style struct { //types:add -setters // Label provides an alternative label to use for axis, if set. Label string + // NoLegend excludes this item from the legend when it otherwise would be included, + // for [plot.NewTablePlot] [table.Table] driven plots. + // Role = Y values are included in the Legend by default. + NoLegend bool + // NTicks sets the desired number of ticks for the axis, if > 0. NTicks int diff --git a/plot/table.go b/plot/table.go index 3cb4fddad0..d143d615c5 100644 --- a/plot/table.go +++ b/plot/table.go @@ -33,6 +33,8 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { gps := make(map[string][]tensor.Values, nc) var xt tensor.Values // get the _last_ role = X column -- most specific counter var errs []error + var pstySt Style // overall PlotStyle accumulator + pstySt.Defaults() for _, cl := range dt.Columns.Values { st := &Style{} st.Defaults() @@ -42,19 +44,37 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { } csty[cl] = st stl.Run(st) + stl.Run(&pstySt) gps[st.Group] = append(gps[st.Group], cl) if st.Role == X { xt = cl } } + psty := pstySt.Plot + globalX := false + if psty.XAxis.Column != "" { + xc := dt.Columns.At(psty.XAxis.Column) + if xc != nil { + xt = xc + globalX = true + } else { + errs = append(errs, errors.New("XAxis.Column name not found: "+psty.XAxis.Column)) + } + } doneGps := map[string]bool{} plt := New() + var legends []Thumbnailer // candidates for legend adding -- only add if > 1 + var legLabels []string for ci, cl := range dt.Columns.Values { cnm := dt.Columns.Keys[ci] st := csty[cl] if st == nil || !st.On || st.Role == X { continue } + lbl := cnm + if st.Label != "" { + lbl = st.Label + } gp := st.Group if doneGps[gp] { continue @@ -74,8 +94,11 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { data := Data{st.Role: cl} gcols := gps[gp] gotReq := true + if globalX { + data[X] = xt + } for _, rl := range pt.Required { - if rl == st.Role { + if rl == st.Role || (rl == X && globalX) { continue } got := false @@ -123,10 +146,21 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { pl := pt.New(data) if pl != nil { plt.Add(pl) + if !st.NoLegend { + if tn, ok := pl.(Thumbnailer); ok { + legends = append(legends, tn) + legLabels = append(legLabels, lbl) + } + } } else { err = fmt.Errorf("plot.NewTablePlot: error in creating plotter type: %q", ptyp) errs = append(errs, err) } } + if len(legends) > 1 { + for i, l := range legends { + plt.Legend.Add(legLabels[i], l) + } + } return plt, errors.Join(errs...) } From bc7464d2f2ba94e1ed4f4ea939f5306452542cc1 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 13 Nov 2024 00:22:04 -0800 Subject: [PATCH 291/311] plot table: tricky logic for automatic X axis label working --- plot/plotcore/ploteditor.go | 2 -- plot/style.go | 7 ++++ plot/table.go | 71 +++++++++++++++++++++++-------------- 3 files changed, 52 insertions(+), 28 deletions(-) diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index ba6ea4601b..909d11dd59 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -8,7 +8,6 @@ package plotcore //go:generate core generate import ( - "fmt" "io/fs" "log/slog" "path/filepath" @@ -504,7 +503,6 @@ func (pl *PlotEditor) plotStyleFromTable(dt *table.Table) { mods["Title"] = true } } - fmt.Println(pl.PlotStyle.Scale) } // modFields returns the modified fields as field paths using . separators diff --git a/plot/style.go b/plot/style.go index 30ce167d25..24c369f672 100644 --- a/plot/style.go +++ b/plot/style.go @@ -160,6 +160,13 @@ func GetStylersFrom(obj any) Stylers { return st } +// AddStylerTo adds the given [Styler] function into given object's [metadata]. +func AddStylerTo(obj any, f func(s *Style)) { + st := GetStylersFrom(obj) + st.Add(f) + SetStylersTo(obj, st) +} + // GetStylersFromData returns [Stylers] from given role // in given [Data]. nil if not present. func GetStylersFromData(data Data, role Roles) Stylers { diff --git a/plot/table.go b/plot/table.go index d143d615c5..2225a1ba1d 100644 --- a/plot/table.go +++ b/plot/table.go @@ -8,8 +8,8 @@ import ( "fmt" "cogentcore.org/core/base/errors" - "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" + "golang.org/x/exp/maps" ) // NewTablePlot returns a new Plot with all configuration based on given @@ -29,34 +29,35 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { if nc == 0 { return nil, errors.New("plot.NewTablePlot: no columns in data table") } - csty := make(map[tensor.Values]*Style, nc) - gps := make(map[string][]tensor.Values, nc) - var xt tensor.Values // get the _last_ role = X column -- most specific counter + csty := make([]*Style, nc) + gps := make(map[string][]int, nc) + xi := -1 // get the _last_ role = X column -- most specific counter var errs []error var pstySt Style // overall PlotStyle accumulator pstySt.Defaults() - for _, cl := range dt.Columns.Values { + for ci, cl := range dt.Columns.Values { st := &Style{} st.Defaults() stl := GetStylersFrom(cl) - if stl == nil { - continue + if stl != nil { + stl.Run(st) } - csty[cl] = st - stl.Run(st) + csty[ci] = st stl.Run(&pstySt) - gps[st.Group] = append(gps[st.Group], cl) + gps[st.Group] = append(gps[st.Group], ci) if st.Role == X { - xt = cl + xi = ci } } psty := pstySt.Plot globalX := false + xidxs := map[int]bool{} // map of all the _unique_ x indexes used if psty.XAxis.Column != "" { - xc := dt.Columns.At(psty.XAxis.Column) - if xc != nil { - xt = xc + xc := dt.Columns.IndexByKey(psty.XAxis.Column) + if xc >= 0 { + xi = xc globalX = true + xidxs[xi] = true } else { errs = append(errs, errors.New("XAxis.Column name not found: "+psty.XAxis.Column)) } @@ -67,8 +68,8 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { var legLabels []string for ci, cl := range dt.Columns.Values { cnm := dt.Columns.Keys[ci] - st := csty[cl] - if st == nil || !st.On || st.Role == X { + st := csty[ci] + if !st.On || st.Role == X { continue } lbl := cnm @@ -95,31 +96,35 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { gcols := gps[gp] gotReq := true if globalX { - data[X] = xt + data[X] = dt.Columns.Values[xi] } + gotX := -1 for _, rl := range pt.Required { if rl == st.Role || (rl == X && globalX) { continue } got := false - for _, gc := range gcols { - gst := csty[gc] + for _, gi := range gcols { + gst := csty[gi] if gst.Role == rl { if rl == Y { if !gst.On { continue } } - data[rl] = gc + data[rl] = dt.Columns.Values[gi] got = true - if rl != X { // get the last one for X + if rl == X { + gotX = gi + } else { break } } } if !got { - if rl == X && xt != nil { - data[rl] = xt + if rl == X && xi >= 0 { + gotX = xi + data[rl] = dt.Columns.Values[xi] } else { err = fmt.Errorf("plot.NewTablePlot: Required Role %q not found in Group %q, Plotter %q not added for Column: %q", rl.String(), gp, ptyp, cnm) errs = append(errs, err) @@ -131,14 +136,17 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { if !gotReq { continue } + if gotX >= 0 { + xidxs[gotX] = true + } for _, rl := range pt.Optional { if rl == st.Role { // should not happen continue } - for _, gc := range gcols { - gst := csty[gc] + for _, gi := range gcols { + gst := csty[gi] if gst.Role == rl { - data[rl] = gc + data[rl] = dt.Columns.Values[gi] break } } @@ -162,5 +170,16 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { plt.Legend.Add(legLabels[i], l) } } + if psty.XAxis.Label == "" && len(xidxs) == 1 { + xi := maps.Keys(xidxs)[0] + lbl := dt.Columns.Keys[xi] + if csty[xi].Label != "" { + lbl = csty[xi].Label + } + pl0 := plt.Plotters[0] + pl0.Stylers().Add(func(s *Style) { + s.Plot.XAxis.Label = lbl + }) + } return plt, errors.Join(errs...) } From a2d67fd673db852b9592d06511d09651f727e913 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 13 Nov 2024 00:35:41 -0800 Subject: [PATCH 292/311] readme --- plot/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plot/README.md b/plot/README.md index 080dacb35e..1e1114357f 100644 --- a/plot/README.md +++ b/plot/README.md @@ -153,6 +153,10 @@ The `statplot` package provides functions taking `tensor` data that produce stat TODO: add a Data scatter that plots points to overlay on top of Violin or Box. +## LegendGroups + +* implements current legend grouping logic -- ends up being a multi-table output -- not sure how to interface. + ## Histogram ## Quartiles From 4bded0c8c0acb517bf87850cf14fc21ada332675 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 13 Nov 2024 02:04:50 -0800 Subject: [PATCH 293/311] bar plot striding logic; increase default nticks; cleanup old code --- plot/README.md | 2 + plot/axis.go | 4 +- plot/line.go | 12 ++ plot/plotcore/barplot.go | 199 ------------------------ plot/plotcore/ploteditor.go | 18 ++- plot/plotcore/ploteditor_test.go | 36 +++-- plot/plotcore/tablexy.go | 252 ------------------------------- plot/plots/xy.go | 3 +- plot/style.go | 7 +- plot/table.go | 61 ++++++-- 10 files changed, 111 insertions(+), 483 deletions(-) delete mode 100644 plot/plotcore/barplot.go delete mode 100644 plot/plotcore/tablexy.go diff --git a/plot/README.md b/plot/README.md index 1e1114357f..a6bbeff852 100644 --- a/plot/README.md +++ b/plot/README.md @@ -180,5 +180,7 @@ Here is the copyright notice for that package: # TODO +* points size incorporated into UpdateRange in XY +* tensor index * Grid? in styling. diff --git a/plot/axis.go b/plot/axis.go index 43cdb36711..2e648f0a1d 100644 --- a/plot/axis.go +++ b/plot/axis.go @@ -50,7 +50,7 @@ type AxisStyle struct { //types:add -setters // on the axis, thus making it easier to see. Padding units.Value - // NTicks is the desired number of ticks. + // NTicks is the desired number of ticks (actual likely will be different). NTicks int // Scale specifies how values are scaled along the axis: @@ -73,7 +73,7 @@ func (ax *AxisStyle) Defaults() { ax.Text.Defaults() ax.Text.Size.Dp(20) ax.Padding.Pt(5) - ax.NTicks = 3 + ax.NTicks = 5 ax.TickText.Defaults() ax.TickText.Size.Dp(16) ax.TickText.Padding.Dp(2) diff --git a/plot/line.go b/plot/line.go index b40de6c4fe..fba304e5ea 100644 --- a/plot/line.go +++ b/plot/line.go @@ -46,6 +46,7 @@ type LineStyle struct { //types:add -setters func (ls *LineStyle) Defaults() { ls.Color = colors.Scheme.OnSurface + ls.Fill = colors.Uniform(colors.Transparent) ls.Width.Pt(1) } @@ -67,6 +68,17 @@ func (ls *LineStyle) SetStroke(pt *Plot) bool { return true } +func (ls *LineStyle) HasFill() bool { + if ls.Fill == nil { + return false + } + clr := colors.ToUniform(ls.Fill) + if clr == colors.Transparent { + return false + } + return true +} + // Draw draws a line between given coordinates, setting the stroke style // to current parameters. Returns false if either Width = 0 or Color = nil func (ls *LineStyle) Draw(pt *Plot, start, end math32.Vector2) bool { diff --git a/plot/plotcore/barplot.go b/plot/plotcore/barplot.go deleted file mode 100644 index e9964d7f8c..0000000000 --- a/plot/plotcore/barplot.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build not - -package plotcore - -import ( - "fmt" - "log" - - "cogentcore.org/core/colors" - "cogentcore.org/core/math32" - "cogentcore.org/core/math32/minmax" - "cogentcore.org/core/plot" - "cogentcore.org/core/plot/plots" -) - -// bar plot is on integer positions, with different Y values and / or -// legend values interleaved - -// genPlotBar generates a Bar plot, setting GPlot variable -func (pl *PlotEditor) genPlotBar() { - plt := plot.New() // note: not clear how to re-use, due to newtablexynames - if pl.Options.BarWidth > 1 { - pl.Options.BarWidth = .8 - } - - // process xaxis first - xi, xview, err := pl.plotXAxis(plt, pl.table) - if err != nil { - return - } - xp := pl.Columns[xi] - - // var lsplit *table.Splits - nleg := 1 - if pl.Options.Legend != "" { - lcol := pl.table.ColumnIndex(pl.Options.Legend) - if lcol < 0 { - log.Println("plot.Legend not found: " + pl.Options.Legend) - } else { - // xview.SortColumnNames([]string{pl.Options.Legend, xp.Column}, tensor.Ascending) // make it fit! - // lsplit = split.GroupBy(xview, pl.Options.Legend) - // nleg = max(lsplit.Len(), 1) - } - } - - var firstXY *tableXY - var strCols []*ColumnOptions - nys := 0 - for _, cp := range pl.Columns { - if !cp.On { - continue - } - if cp.IsString { - strCols = append(strCols, cp) - continue - } - if cp.TensorIndex < 0 { - yc := pl.table.Column(cp.Column) - _, sz := yc.RowCellSize() - nys += sz - } else { - nys++ - } - } - - if nys == 0 { - return - } - - stride := nys * nleg - if stride > 1 { - stride += 1 // extra gap - } - - yoff := 0 - yidx := 0 - maxx := 0 // max number of x values - for _, cp := range pl.Columns { - if !cp.On || cp == xp { - continue - } - if cp.IsString { - continue - } - start := yoff - for li := 0; li < nleg; li++ { - lview := xview - leg := "" - // if lsplit != nil && len(lsplit.Values) > li { - // leg = lsplit.Values[li][0] - // lview = lsplit.Splits[li] - // } - nidx := 1 - stidx := cp.TensorIndex - if cp.TensorIndex < 0 { // do all - yc := pl.table.Column(cp.Column) - _, sz := yc.RowCellSize() - nidx = sz - stidx = 0 - } - for ii := 0; ii < nidx; ii++ { - idx := stidx + ii - xy, _ := newTableXYName(lview, xi, xp.TensorIndex, cp.Column, idx, cp.Range) - if xy == nil { - continue - } - maxx = max(maxx, lview.NumRows()) - if firstXY == nil { - firstXY = xy - } - lbl := cp.getLabel() - clr := cp.Color - if leg != "" { - lbl = leg + " " + lbl - } - if nleg > 1 { - cidx := yidx*nleg + li - clr = colors.Uniform(colors.Spaced(cidx)) - } - if nidx > 1 { - clr = colors.Uniform(colors.Spaced(idx)) - lbl = fmt.Sprintf("%s_%02d", lbl, idx) - } - ec := -1 - if cp.ErrColumn != "" { - ec = pl.table.ColumnIndex(cp.ErrColumn) - } - var bar *plots.BarChart - if ec >= 0 { - exy, _ := newTableXY(lview, ec, 0, ec, 0, minmax.Range32{}) - bar = plots.NewBarChart(xy, exy) - if bar == nil { - continue - } - } else { - bar = plots.NewBarChart(xy, nil) - if bar == nil { - continue - } - } - bar.Color = clr - bar.Stride = float32(stride) - bar.Offset = float32(start) - bar.Width = pl.Options.BarWidth - plt.Add(bar) - plt.Legend.Add(lbl, bar) - start++ - } - } - yidx++ - yoff += nleg - } - mid := (stride - 1) / 2 - if stride > 1 { - mid = (stride - 2) / 2 - } - if firstXY != nil && len(strCols) > 0 { - firstXY.table = xview - n := xview.NumRows() - for _, cp := range strCols { - xy, _ := newTableXY(xview, xi, xp.TensorIndex, firstXY.yColumn, cp.TensorIndex, firstXY.yRange) - xy.labelColumn = xview.ColumnIndex(cp.Column) - xy.yIndex = firstXY.yIndex - - xyl := plots.XYLabels{} - xyl.XYs = make(plot.XYs, n) - xyl.Labels = make([]string, n) - - for i := range xview.Indexes { - y := firstXY.Float1D(i) - x := float32(mid + (i%maxx)*stride) - xyl.XYs[i] = math32.Vec2(x, y) - xyl.Labels[i] = xy.Label(i) - } - lbls := plots.NewLabels(xyl) - if lbls != nil { - plt.Add(lbls) - } - } - } - - netn := pl.table.NumRows() * stride - xc := pl.table.ColumnByIndex(xi) - vals := make([]string, netn) - for i, dx := range pl.table.Indexes { - pi := mid + i*stride - if pi < netn && dx < xc.Len() { - vals[pi] = xc.String1D(dx) - } - } - plt.NominalX(vals...) - - pl.configPlot(plt) - pl.plot = plt -} diff --git a/plot/plotcore/ploteditor.go b/plot/plotcore/ploteditor.go index 909d11dd59..fc10c1bc3b 100644 --- a/plot/plotcore/ploteditor.go +++ b/plot/plotcore/ploteditor.go @@ -151,10 +151,20 @@ func (pl *PlotEditor) SetTable(tab *table.Table) *PlotEditor { return pl } -// SetSlice sets the table to a [table.NewSliceTable] -// from the given slice. -func (pl *PlotEditor) SetSlice(sl any) *PlotEditor { - return pl.SetTable(errors.Log1(table.NewSliceTable(sl))) +// SetSlice sets the table to a [table.NewSliceTable] from the given slice. +// Optional styler functions are used for each struct field in sequence, +// and any can contain global plot style. +func (pl *PlotEditor) SetSlice(sl any, stylers ...func(s *plot.Style)) *PlotEditor { + dt, err := table.NewSliceTable(sl) + errors.Log(err) + if dt == nil { + return nil + } + mx := min(dt.NumColumns(), len(stylers)) + for i := range mx { + plot.SetStylersTo(dt.Columns.Values[i], plot.Stylers{stylers[i]}) + } + return pl.SetTable(dt) } // SaveSVG saves the plot to an svg -- first updates to ensure that plot is current diff --git a/plot/plotcore/ploteditor_test.go b/plot/plotcore/ploteditor_test.go index 301991d56b..7e89f8b0be 100644 --- a/plot/plotcore/ploteditor_test.go +++ b/plot/plotcore/ploteditor_test.go @@ -8,11 +8,13 @@ import ( "testing" "cogentcore.org/core/core" + "cogentcore.org/core/plot" + "cogentcore.org/core/plot/plots" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/table" ) -type Data struct { +type data struct { City string Population float32 Area float32 @@ -25,12 +27,16 @@ func TestTablePlotEditor(t *testing.T) { epc.OpenCSV("testdata/ra25epoch.tsv", tensor.Tab) pl := NewPlotEditor(b) - pl.Options.Title = "RA25 Epoch Train" - pl.Options.XAxis = "Epoch" - // pl.Options.Scale = 2 - pl.Options.Points = true + pst := func(s *plot.Style) { + s.Plot.Title = "RA25 Epoch Train" + s.Plot.PointsOn = plot.On + } + perr := epc.Column("PctErr") + plot.SetStylersTo(perr, plot.Stylers{pst, func(s *plot.Style) { + s.On = true + s.Role = plot.Y + }}) pl.SetTable(epc) - pl.ColumnOptions("UnitErr").On = true b.AddTopBar(func(bar *core.Frame) { core.NewToolbar(bar).Maker(pl.MakeToolbar) }) @@ -38,18 +44,24 @@ func TestTablePlotEditor(t *testing.T) { } func TestSlicePlotEditor(t *testing.T) { - t.Skip("TODO: this test randomly hangs on CI") - data := []Data{ + dt := []data{ {"Davis", 62000, 500}, {"Boulder", 85000, 800}, } b := core.NewBody() - pl := NewPlotEditor(b) - pl.Options.Title = "Slice Data" - pl.Options.Points = true - pl.SetSlice(data) + pst := func(s *plot.Style) { + s.Plot.Title = "Test Data" + s.Plot.PointsOn = plot.On + } + onst := func(s *plot.Style) { + pst(s) + s.Plotter = plots.BarType + s.On = true + s.Role = plot.Y + } + pl.SetSlice(dt, pst, onst) b.AddTopBar(func(bar *core.Frame) { core.NewToolbar(bar).Maker(pl.MakeToolbar) }) diff --git a/plot/plotcore/tablexy.go b/plot/plotcore/tablexy.go deleted file mode 100644 index 13f7805369..0000000000 --- a/plot/plotcore/tablexy.go +++ /dev/null @@ -1,252 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build not - -package plotcore - -import ( - "cogentcore.org/core/base/errors" - "cogentcore.org/core/math32" - "cogentcore.org/core/math32/minmax" - "cogentcore.org/core/plot" - "cogentcore.org/core/plot/plots" - "cogentcore.org/core/tensor/table" -) - -// tableXY selects two columns from a [table.Table] data table to plot in a [plot.Plot], -// satisfying the [plot.XYer], [plot.Valuer], [plot.Labeler], and [plots.YErrorer] interfaces. -// For Tensor-valued cells, Index's specify tensor cell. -// Also satisfies the plot/plots.Labeler interface for labels attached to a line, and -// plot/plots.YErrorer for error bars. -type tableXY struct { - - // the index view of data table to plot from - table *table.Table - - // the indexes of the tensor columns to use for the X and Y data, respectively - xColumn, yColumn int - - // numer of elements in each row of data -- 1 for scalar, > 1 for multi-dimensional - xRowSize, yRowSize int - - // the indexes of the element within each tensor cell if cells are n-dimensional, respectively - xIndex, yIndex int - - // the column to use for returning a label using Label interface -- for string cols - labelColumn int - - // the column to use for returning errorbars (+/- given value) -- if YColumn is tensor then this must also be a tensor and given YIndex used - errColumn int - - // range constraints on Y values - yRange minmax.Range32 -} - -var _ plot.XYer = &tableXY{} -var _ plot.Valuer = &tableXY{} -var _ plot.Labeler = &tableXY{} -var _ plots.YErrorer = &tableXY{} - -// newTableXY returns a new XY plot view onto the given Indexed view of table.Table (makes a copy), -// from given column indexes, and tensor indexes within each cell. -// Column indexes are enforced to be valid, with an error message if they are not. -func newTableXY(dt *table.Table, xcol, xtsrIndex, ycol, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { - txy := &tableXY{table: dt.Clone(), xColumn: xcol, yColumn: ycol, xIndex: xtsrIndex, yIndex: ytsrIndex, yRange: yrng} - return txy, txy.validate() -} - -// newTableXYName returns a new XY plot view onto the given Indexed view of table.Table (makes a copy), -// from given column name and tensor indexes within each cell. -// Column indexes are enforced to be valid, with an error message if they are not. -func newTableXYName(dt *table.Table, xi, xtsrIndex int, ycol string, ytsrIndex int, yrng minmax.Range32) (*tableXY, error) { - yi := dt.ColumnIndex(ycol) - if yi < 0 { - return nil, nil // todo: err - } - txy := &tableXY{table: dt.Clone(), xColumn: xi, yColumn: yi, xIndex: xtsrIndex, yIndex: ytsrIndex, yRange: yrng} - return txy, txy.validate() -} - -// validate returns error message if column indexes are invalid, else nil -// it also sets column indexes to 0 so nothing crashes. -func (txy *tableXY) validate() error { - if txy.table == nil { - return errors.New("eplot.TableXY table is nil") - } - nc := txy.table.NumColumns() - if txy.xColumn >= nc || txy.xColumn < 0 { - txy.xColumn = 0 - return errors.New("eplot.TableXY XColumn index invalid -- reset to 0") - } - if txy.yColumn >= nc || txy.yColumn < 0 { - txy.yColumn = 0 - return errors.New("eplot.TableXY YColumn index invalid -- reset to 0") - } - xc := txy.table.ColumnByIndex(txy.xColumn) - yc := txy.table.ColumnByIndex(txy.yColumn) - if xc.Tensor.NumDims() > 1 { - _, txy.xRowSize = xc.RowCellSize() - // note: index already validated - } - if yc.Tensor.NumDims() > 1 { - _, txy.yRowSize = yc.RowCellSize() - if txy.yIndex >= txy.yRowSize || txy.yIndex < 0 { - txy.yIndex = 0 - return errors.New("eplot.TableXY Y TensorIndex invalid -- reset to 0") - } - } - txy.filterValues() - return nil -} - -// filterValues removes items with NaN values, and out of Y range -func (txy *tableXY) filterValues() { - txy.table.Filter(func(et *table.Table, row int) bool { - xv := txy.tRowXValue(row) - yv := txy.tRowValue(row) - if math32.IsNaN(yv) || math32.IsNaN(xv) { - return false - } - if txy.yRange.FixMin && yv < txy.yRange.Min { - return false - } - if txy.yRange.FixMax && yv > txy.yRange.Max { - return false - } - return true - }) -} - -// Len returns the number of rows in the view of table -func (txy *tableXY) Len() int { - if txy.table == nil { - return 0 - } - return txy.table.NumRows() -} - -// tRowValue returns the y value at given true table row in table -func (txy *tableXY) tRowValue(row int) float32 { - yc := txy.table.ColumnByIndex(txy.yColumn) - y := float32(0.0) - switch { - case yc.Tensor.IsString(): - y = float32(row) - case yc.Tensor.NumDims() > 1: - _, sz := yc.RowCellSize() - if txy.yIndex < sz && txy.yIndex >= 0 { - y = float32(yc.FloatRowCell(row, txy.yIndex)) - } - default: - y = float32(yc.Float1D(row)) - } - return y -} - -// Value returns the y value at given row in table -func (txy *tableXY) Value(row int) float32 { - if txy.table == nil || row >= txy.table.NumRows() { - return 0 - } - trow := txy.table.Indexes[row] // true table row - yc := txy.table.ColumnByIndex(txy.yColumn) - y := float32(0.0) - switch { - case yc.Tensor.IsString(): - y = float32(row) - case yc.Tensor.NumDims() > 1: - _, sz := yc.RowCellSize() - if txy.yIndex < sz && txy.yIndex >= 0 { - y = float32(yc.FloatRowCell(trow, txy.yIndex)) - } - default: - y = float32(yc.Float1D(trow)) - } - return y -} - -// tRowXValue returns an x value at given actual row in table -func (txy *tableXY) tRowXValue(row int) float32 { - if txy.table == nil { - return 0 - } - xc := txy.table.ColumnByIndex(txy.xColumn) - x := float32(0.0) - switch { - case xc.Tensor.IsString(): - x = float32(row) - case xc.Tensor.NumDims() > 1: - _, sz := xc.RowCellSize() - if txy.xIndex < sz && txy.xIndex >= 0 { - x = float32(xc.FloatRowCell(row, txy.xIndex)) - } - default: - x = float32(xc.Float1D(row)) - } - return x -} - -// xValue returns an x value at given row in table -func (txy *tableXY) xValue(row int) float32 { - if txy.table == nil || row >= txy.table.NumRows() { - return 0 - } - trow := txy.table.Indexes[row] // true table row - xc := txy.table.ColumnByIndex(txy.xColumn) - x := float32(0.0) - switch { - case xc.Tensor.IsString(): - x = float32(row) - case xc.Tensor.NumDims() > 1: - _, sz := xc.RowCellSize() - if txy.xIndex < sz && txy.xIndex >= 0 { - x = float32(xc.FloatRowCell(trow, txy.xIndex)) - } - default: - x = float32(xc.Float1D(trow)) - } - return x -} - -// XY returns an x, y pair at given row in table -func (txy *tableXY) XY(row int) (x, y float32) { - if txy.table == nil { - return 0, 0 - } - x = txy.Value(row) - y = txy.Value(row) - return -} - -// Label returns a label for given row in table, implementing [plot.Labeler] interface -func (txy *tableXY) Label(row int) string { - if txy.table == nil || row >= txy.table.NumRows() { - return "" - } - trow := txy.table.Indexes[row] // true table row - return txy.table.ColumnByIndex(txy.labelColumn).String1D(trow) -} - -// YError returns error bars, implementing [plots.YErrorer] interface. -func (txy *tableXY) YError(row int) (float32, float32) { - if txy.table == nil || row >= txy.table.NumRows() { - return 0, 0 - } - trow := txy.table.Indexes[row] // true table row - ec := txy.table.ColumnByIndex(txy.errColumn) - eval := float32(0.0) - switch { - case ec.Tensor.IsString(): - eval = float32(row) - case ec.Tensor.NumDims() > 1: - _, sz := ec.RowCellSize() - if txy.yIndex < sz && txy.yIndex >= 0 { - eval = float32(ec.FloatRowCell(trow, txy.yIndex)) - } - default: - eval = float32(ec.Float1D(trow)) - } - return -eval, eval -} diff --git a/plot/plots/xy.go b/plot/plots/xy.go index b018af27fc..176e7d33e3 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -120,8 +120,7 @@ func (ln *XY) Plot(plt *plot.Plot) { ln.PX = plot.PlotX(plt, ln.X) ln.PY = plot.PlotY(plt, ln.Y) np := len(ln.PX) - - if ln.Style.Line.Fill != nil { + if ln.Style.Line.HasFill() { pc.FillStyle.Color = ln.Style.Line.Fill minY := plt.PY(plt.Y.Range.Min) prevX := ln.PX[0] diff --git a/plot/style.go b/plot/style.go index 24c369f672..2008d5e385 100644 --- a/plot/style.go +++ b/plot/style.go @@ -71,6 +71,7 @@ func NewStyle() *Style { } func (st *Style) Defaults() { + st.Plot.Defaults() st.Line.Defaults() st.Point.Defaults() st.Text.Defaults() @@ -86,14 +87,14 @@ type WidthStyle struct { //types:add -setters // Offset for Bar plot is the offset added to each X axis value // relative to the Stride computed value (X = offset + index * Stride) - // Defaults to 1. + // Defaults to 0. Offset float64 // Stride for Bar plot is distance between bars. Defaults to 1. Stride float64 - // Width for Bar plot is the width of the bars, which should be less than - // the Stride (1 typically) to prevent bar overlap. Defaults to .8. + // Width for Bar plot is the width of the bars, as a fraction of the Stride, + // to prevent bar overlap. Defaults to .8. Width float64 `min:"0.01" max:"1" default:"0.8"` // Pad for Bar plot is additional space at start / end of data range, diff --git a/plot/table.go b/plot/table.go index 2225a1ba1d..2c4d7d26de 100644 --- a/plot/table.go +++ b/plot/table.go @@ -66,6 +66,8 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { plt := New() var legends []Thumbnailer // candidates for legend adding -- only add if > 1 var legLabels []string + var barCols []int // column indexes of bar plots + var barPlots []int // plotter indexes of bar plots for ci, cl := range dt.Columns.Values { cnm := dt.Columns.Keys[ci] st := csty[ci] @@ -152,17 +154,21 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { } } pl := pt.New(data) - if pl != nil { - plt.Add(pl) - if !st.NoLegend { - if tn, ok := pl.(Thumbnailer); ok { - legends = append(legends, tn) - legLabels = append(legLabels, lbl) - } - } - } else { + if pl == nil { err = fmt.Errorf("plot.NewTablePlot: error in creating plotter type: %q", ptyp) errs = append(errs, err) + continue + } + plt.Add(pl) + if !st.NoLegend { + if tn, ok := pl.(Thumbnailer); ok { + legends = append(legends, tn) + legLabels = append(legLabels, lbl) + } + } + if ptyp == "Bar" { + barCols = append(barCols, ci) + barPlots = append(barPlots, len(plt.Plotters)-1) } } if len(legends) > 1 { @@ -181,5 +187,42 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { s.Plot.XAxis.Label = lbl }) } + nbar := len(barCols) + if nbar > 1 { + sz := 1.0 / (float64(nbar) + 0.5) + for bi, bp := range barPlots { + pl := plt.Plotters[bp] + pl.Stylers().Add(func(s *Style) { + s.Width.Stride = 1 + s.Width.Offset = float64(bi) * sz + s.Width.Width = psty.BarWidth * sz + }) + } + } return plt, errors.Join(errs...) } + +// todo: bar chart rows, if needed +// +// netn := pl.table.NumRows() * stride +// xc := pl.table.ColumnByIndex(xi) +// vals := make([]string, netn) +// for i, dx := range pl.table.Indexes { +// pi := mid + i*stride +// if pi < netn && dx < xc.Len() { +// vals[pi] = xc.String1D(dx) +// } +// } +// plt.NominalX(vals...) + +// todo: +// Use string labels for X axis if X is a string +// xc := pl.table.ColumnByIndex(xi) +// if xc.Tensor.IsString() { +// xcs := xc.Tensor.(*tensor.String) +// vals := make([]string, pl.table.NumRows()) +// for i, dx := range pl.table.Indexes { +// vals[i] = xcs.Values[dx] +// } +// plt.NominalX(vals...) +// } From 4531b30122aa9cfb4ce4dd54fb9528f6f2d7a711 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 13 Nov 2024 02:29:12 -0800 Subject: [PATCH 294/311] panzoom post-processing -- better than before. also has shift = x, alt = y constraints --- plot/enumgen.go | 232 +++++++++++++++++++++++++++++++++++++++ plot/plot.go | 38 +++++++ plot/plotcore/plot.go | 40 +++---- plot/plotcore/typegen.go | 11 +- plot/typegen.go | 44 +++++--- 5 files changed, 319 insertions(+), 46 deletions(-) create mode 100644 plot/enumgen.go diff --git a/plot/enumgen.go b/plot/enumgen.go new file mode 100644 index 0000000000..4b47ce6566 --- /dev/null +++ b/plot/enumgen.go @@ -0,0 +1,232 @@ +// Code generated by "core generate -add-types"; DO NOT EDIT. + +package plot + +import ( + "cogentcore.org/core/enums" +) + +var _AxisScalesValues = []AxisScales{0, 1, 2, 3} + +// AxisScalesN is the highest valid value for type AxisScales, plus one. +// +//gosl:start +const AxisScalesN AxisScales = 4 + +//gosl:end + +var _AxisScalesValueMap = map[string]AxisScales{`Linear`: 0, `Log`: 1, `InverseLinear`: 2, `InverseLog`: 3} + +var _AxisScalesDescMap = map[AxisScales]string{0: `Linear is a linear axis scale.`, 1: `Log is a Logarithmic axis scale.`, 2: `InverseLinear is an inverted linear axis scale.`, 3: `InverseLog is an inverted log axis scale.`} + +var _AxisScalesMap = map[AxisScales]string{0: `Linear`, 1: `Log`, 2: `InverseLinear`, 3: `InverseLog`} + +// String returns the string representation of this AxisScales value. +func (i AxisScales) String() string { return enums.String(i, _AxisScalesMap) } + +// SetString sets the AxisScales value from its string representation, +// and returns an error if the string is invalid. +func (i *AxisScales) SetString(s string) error { + return enums.SetString(i, s, _AxisScalesValueMap, "AxisScales") +} + +// Int64 returns the AxisScales value as an int64. +func (i AxisScales) Int64() int64 { return int64(i) } + +// SetInt64 sets the AxisScales value from an int64. +func (i *AxisScales) SetInt64(in int64) { *i = AxisScales(in) } + +// Desc returns the description of the AxisScales value. +func (i AxisScales) Desc() string { return enums.Desc(i, _AxisScalesDescMap) } + +// AxisScalesValues returns all possible values for the type AxisScales. +func AxisScalesValues() []AxisScales { return _AxisScalesValues } + +// Values returns all possible values for the type AxisScales. +func (i AxisScales) Values() []enums.Enum { return enums.Values(_AxisScalesValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i AxisScales) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *AxisScales) UnmarshalText(text []byte) error { + return enums.UnmarshalText(i, text, "AxisScales") +} + +var _RolesValues = []Roles{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11} + +// RolesN is the highest valid value for type Roles, plus one. +// +//gosl:start +const RolesN Roles = 12 + +//gosl:end + +var _RolesValueMap = map[string]Roles{`NoRole`: 0, `X`: 1, `Y`: 2, `Z`: 3, `U`: 4, `V`: 5, `W`: 6, `Low`: 7, `High`: 8, `Size`: 9, `Color`: 10, `Label`: 11} + +var _RolesDescMap = map[Roles]string{0: `NoRole is the default no-role specified case.`, 1: `X axis`, 2: `Y axis`, 3: `Z axis`, 4: `U is the X component of a vector or first quartile in Box plot, etc.`, 5: `V is the Y component of a vector or third quartile in a Box plot, etc.`, 6: `W is the Z component of a vector`, 7: `Low is a lower error bar or region.`, 8: `High is an upper error bar or region.`, 9: `Size controls the size of points etc.`, 10: `Color controls the color of points or other elements.`, 11: `Label renders a label, typically from string data, but can also be used for values.`} + +var _RolesMap = map[Roles]string{0: `NoRole`, 1: `X`, 2: `Y`, 3: `Z`, 4: `U`, 5: `V`, 6: `W`, 7: `Low`, 8: `High`, 9: `Size`, 10: `Color`, 11: `Label`} + +// String returns the string representation of this Roles value. +func (i Roles) String() string { return enums.String(i, _RolesMap) } + +// SetString sets the Roles value from its string representation, +// and returns an error if the string is invalid. +func (i *Roles) SetString(s string) error { return enums.SetString(i, s, _RolesValueMap, "Roles") } + +// Int64 returns the Roles value as an int64. +func (i Roles) Int64() int64 { return int64(i) } + +// SetInt64 sets the Roles value from an int64. +func (i *Roles) SetInt64(in int64) { *i = Roles(in) } + +// Desc returns the description of the Roles value. +func (i Roles) Desc() string { return enums.Desc(i, _RolesDescMap) } + +// RolesValues returns all possible values for the type Roles. +func RolesValues() []Roles { return _RolesValues } + +// Values returns all possible values for the type Roles. +func (i Roles) Values() []enums.Enum { return enums.Values(_RolesValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i Roles) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *Roles) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Roles") } + +var _StepKindValues = []StepKind{0, 1, 2, 3} + +// StepKindN is the highest valid value for type StepKind, plus one. +// +//gosl:start +const StepKindN StepKind = 4 + +//gosl:end + +var _StepKindValueMap = map[string]StepKind{`NoStep`: 0, `PreStep`: 1, `MidStep`: 2, `PostStep`: 3} + +var _StepKindDescMap = map[StepKind]string{0: `NoStep connects two points by simple line.`, 1: `PreStep connects two points by following lines: vertical, horizontal.`, 2: `MidStep connects two points by following lines: horizontal, vertical, horizontal. Vertical line is placed in the middle of the interval.`, 3: `PostStep connects two points by following lines: horizontal, vertical.`} + +var _StepKindMap = map[StepKind]string{0: `NoStep`, 1: `PreStep`, 2: `MidStep`, 3: `PostStep`} + +// String returns the string representation of this StepKind value. +func (i StepKind) String() string { return enums.String(i, _StepKindMap) } + +// SetString sets the StepKind value from its string representation, +// and returns an error if the string is invalid. +func (i *StepKind) SetString(s string) error { + return enums.SetString(i, s, _StepKindValueMap, "StepKind") +} + +// Int64 returns the StepKind value as an int64. +func (i StepKind) Int64() int64 { return int64(i) } + +// SetInt64 sets the StepKind value from an int64. +func (i *StepKind) SetInt64(in int64) { *i = StepKind(in) } + +// Desc returns the description of the StepKind value. +func (i StepKind) Desc() string { return enums.Desc(i, _StepKindDescMap) } + +// StepKindValues returns all possible values for the type StepKind. +func StepKindValues() []StepKind { return _StepKindValues } + +// Values returns all possible values for the type StepKind. +func (i StepKind) Values() []enums.Enum { return enums.Values(_StepKindValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i StepKind) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *StepKind) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "StepKind") } + +var _ShapesValues = []Shapes{0, 1, 2, 3, 4, 5, 6, 7} + +// ShapesN is the highest valid value for type Shapes, plus one. +// +//gosl:start +const ShapesN Shapes = 8 + +//gosl:end + +var _ShapesValueMap = map[string]Shapes{`Ring`: 0, `Circle`: 1, `Square`: 2, `Box`: 3, `Triangle`: 4, `Pyramid`: 5, `Plus`: 6, `Cross`: 7} + +var _ShapesDescMap = map[Shapes]string{0: `Ring is the outline of a circle`, 1: `Circle is a solid circle`, 2: `Square is the outline of a square`, 3: `Box is a filled square`, 4: `Triangle is the outline of a triangle`, 5: `Pyramid is a filled triangle`, 6: `Plus is a plus sign`, 7: `Cross is a big X`} + +var _ShapesMap = map[Shapes]string{0: `Ring`, 1: `Circle`, 2: `Square`, 3: `Box`, 4: `Triangle`, 5: `Pyramid`, 6: `Plus`, 7: `Cross`} + +// String returns the string representation of this Shapes value. +func (i Shapes) String() string { return enums.String(i, _ShapesMap) } + +// SetString sets the Shapes value from its string representation, +// and returns an error if the string is invalid. +func (i *Shapes) SetString(s string) error { return enums.SetString(i, s, _ShapesValueMap, "Shapes") } + +// Int64 returns the Shapes value as an int64. +func (i Shapes) Int64() int64 { return int64(i) } + +// SetInt64 sets the Shapes value from an int64. +func (i *Shapes) SetInt64(in int64) { *i = Shapes(in) } + +// Desc returns the description of the Shapes value. +func (i Shapes) Desc() string { return enums.Desc(i, _ShapesDescMap) } + +// ShapesValues returns all possible values for the type Shapes. +func ShapesValues() []Shapes { return _ShapesValues } + +// Values returns all possible values for the type Shapes. +func (i Shapes) Values() []enums.Enum { return enums.Values(_ShapesValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i Shapes) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *Shapes) UnmarshalText(text []byte) error { return enums.UnmarshalText(i, text, "Shapes") } + +var _DefaultOffOnValues = []DefaultOffOn{0, 1, 2} + +// DefaultOffOnN is the highest valid value for type DefaultOffOn, plus one. +// +//gosl:start +const DefaultOffOnN DefaultOffOn = 3 + +//gosl:end + +var _DefaultOffOnValueMap = map[string]DefaultOffOn{`Default`: 0, `Off`: 1, `On`: 2} + +var _DefaultOffOnDescMap = map[DefaultOffOn]string{0: `Default means use the default value.`, 1: `Off means to override the default and turn Off.`, 2: `On means to override the default and turn On.`} + +var _DefaultOffOnMap = map[DefaultOffOn]string{0: `Default`, 1: `Off`, 2: `On`} + +// String returns the string representation of this DefaultOffOn value. +func (i DefaultOffOn) String() string { return enums.String(i, _DefaultOffOnMap) } + +// SetString sets the DefaultOffOn value from its string representation, +// and returns an error if the string is invalid. +func (i *DefaultOffOn) SetString(s string) error { + return enums.SetString(i, s, _DefaultOffOnValueMap, "DefaultOffOn") +} + +// Int64 returns the DefaultOffOn value as an int64. +func (i DefaultOffOn) Int64() int64 { return int64(i) } + +// SetInt64 sets the DefaultOffOn value from an int64. +func (i *DefaultOffOn) SetInt64(in int64) { *i = DefaultOffOn(in) } + +// Desc returns the description of the DefaultOffOn value. +func (i DefaultOffOn) Desc() string { return enums.Desc(i, _DefaultOffOnDescMap) } + +// DefaultOffOnValues returns all possible values for the type DefaultOffOn. +func DefaultOffOnValues() []DefaultOffOn { return _DefaultOffOnValues } + +// Values returns all possible values for the type DefaultOffOn. +func (i DefaultOffOn) Values() []enums.Enum { return enums.Values(_DefaultOffOnValues) } + +// MarshalText implements the [encoding.TextMarshaler] interface. +func (i DefaultOffOn) MarshalText() ([]byte, error) { return []byte(i.String()), nil } + +// UnmarshalText implements the [encoding.TextUnmarshaler] interface. +func (i *DefaultOffOn) UnmarshalText(text []byte) error { + return enums.UnmarshalText(i, text, "DefaultOffOn") +} diff --git a/plot/plot.go b/plot/plot.go index 6af47b1bd0..361374f29e 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -129,6 +129,27 @@ func (ps *PlotStyle) SetElementStyle(es *Style) { es.Text.Size = ps.LabelSize } +// PanZoom provides post-styling pan and zoom range manipulation. +type PanZoom struct { + + // XOffset adds offset to X range (pan). + XOffset float64 + + // XScale multiplies X range (zoom). + XScale float64 + + // YOffset adds offset to Y range (pan). + YOffset float64 + + // YScale multiplies Y range (zoom). + YScale float64 +} + +func (pz *PanZoom) Defaults() { + pz.XScale = 1 + pz.YScale = 1 +} + // Plot is the basic type representing a plot. // It renders into its own image.RGBA Pixels image, // and can also save a corresponding SVG version. @@ -161,6 +182,9 @@ type Plot struct { // which is strongly recommended for print (e.g., use 300 for print) DPI float32 `default:"96,160,300"` + // PanZoom provides post-styling pan and zoom range factors. + PanZoom PanZoom + // painter for rendering Paint *paint.Context @@ -187,6 +211,7 @@ func (pt *Plot) Defaults() { pt.Y.Defaults(math32.Y) pt.Legend.Defaults() pt.DPI = 96 + pt.PanZoom.Defaults() pt.Size = image.Point{1280, 1024} pt.StandardTextStyle.Defaults() pt.StandardTextStyle.WhiteSpace = styles.WhiteSpaceNowrap @@ -334,6 +359,19 @@ func (pt *Plot) UpdateRange() { for _, pl := range pt.Plotters { pl.UpdateRange(pt, &pt.X.Range, &pt.Y.Range, &pt.Z.Range) } + pt.X.Range.Sanitize() + pt.Y.Range.Sanitize() + pt.Z.Range.Sanitize() + + pt.X.Range.Min *= pt.PanZoom.XScale + pt.X.Range.Max *= pt.PanZoom.XScale + pt.X.Range.Min += pt.PanZoom.XOffset + pt.X.Range.Max += pt.PanZoom.XOffset + + pt.Y.Range.Min *= pt.PanZoom.YScale + pt.Y.Range.Max *= pt.PanZoom.YScale + pt.Y.Range.Min += pt.PanZoom.YOffset + pt.Y.Range.Max += pt.PanZoom.YOffset } // PX returns the X-axis plotting coordinate for given raw data value diff --git a/plot/plotcore/plot.go b/plot/plotcore/plot.go index f12f1515c0..4d770725a6 100644 --- a/plot/plotcore/plot.go +++ b/plot/plotcore/plot.go @@ -13,6 +13,7 @@ import ( "cogentcore.org/core/core" "cogentcore.org/core/cursors" "cogentcore.org/core/events" + "cogentcore.org/core/events/key" "cogentcore.org/core/plot" "cogentcore.org/core/styles" "cogentcore.org/core/styles/abilities" @@ -26,12 +27,6 @@ import ( type Plot struct { core.WidgetBase - // Scale multiplies the plot DPI value, to change the overall scale - // of the rendered plot. Larger numbers produce larger scaling. - // Typically use larger numbers when generating plots for inclusion in - // documents or other cases where the overall plot size will be small. - Scale float32 - // Plot is the Plot to display in this widget Plot *plot.Plot `set:"-"` @@ -44,7 +39,7 @@ type Plot struct { // drawn at the current size of this widget func (pt *Plot) SetPlot(pl *plot.Plot) { if pl != nil && pt.Plot != nil && pt.Plot.Pixels != nil { - pl.DPI = pt.Scale * pt.Styles.UnitContext.DPI + pl.DPI = pt.Styles.UnitContext.DPI pl.SetPixels(pt.Plot.Pixels) // re-use the image! } pt.Plot = pl @@ -62,7 +57,7 @@ func (pt *Plot) updatePlot() { if sz == (image.Point{}) { return } - pt.Plot.DPI = pt.Scale * pt.Styles.UnitContext.DPI + pt.Plot.DPI = pt.Styles.UnitContext.DPI pt.Plot.Resize(sz) if pt.SetRangesFunc != nil { pt.SetRangesFunc() @@ -73,7 +68,6 @@ func (pt *Plot) updatePlot() { func (pt *Plot) Init() { pt.WidgetBase.Init() - pt.Scale = 1 pt.Styler(func(s *styles.Style) { s.Min.Set(units.Dp(256)) ro := pt.IsReadOnly() @@ -93,13 +87,17 @@ func (pt *Plot) Init() { if pt.Plot == nil { return } + xf, yf := 1.0, 1.0 + if e.HasAnyModifier(key.Shift) { + yf = 0 + } else if e.HasAnyModifier(key.Alt) { + xf = 0 + } del := e.PrevDelta() - dx := -float64(del.X) * (pt.Plot.X.Range.Range()) * 0.0008 - dy := float64(del.Y) * (pt.Plot.Y.Range.Range()) * 0.0008 - pt.Plot.X.Range.Min += dx - pt.Plot.X.Range.Max += dx - pt.Plot.Y.Range.Min += dy - pt.Plot.Y.Range.Max += dy + dx := -float64(del.X) * (pt.Plot.X.Range.Range()) * 0.0008 * xf + dy := float64(del.Y) * (pt.Plot.Y.Range.Range()) * 0.0008 * yf + pt.Plot.PanZoom.XOffset += dx + pt.Plot.PanZoom.YOffset += dy pt.updatePlot() pt.NeedsRender() }) @@ -111,10 +109,14 @@ func (pt *Plot) Init() { } se := e.(*events.MouseScroll) sc := 1 + (float64(se.Delta.Y) * 0.002) - pt.Plot.X.Range.Min *= sc - pt.Plot.X.Range.Max *= sc - pt.Plot.Y.Range.Min *= sc - pt.Plot.Y.Range.Max *= sc + xsc, ysc := sc, sc + if e.HasAnyModifier(key.Shift) { + ysc = 1 + } else if e.HasAnyModifier(key.Alt) { + xsc = 1 + } + pt.Plot.PanZoom.XScale *= xsc + pt.Plot.PanZoom.YScale *= ysc pt.updatePlot() pt.NeedsRender() }) diff --git a/plot/plotcore/typegen.go b/plot/plotcore/typegen.go index c270c56ffd..ae170910ed 100644 --- a/plot/plotcore/typegen.go +++ b/plot/plotcore/typegen.go @@ -8,7 +8,7 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plot", IDName: "plot", Doc: "Plot is a widget that renders a [plot.Plot] object.\nIf it is not [states.ReadOnly], the user can pan and zoom the graph.\nSee [PlotEditor] for an interactive interface for selecting columns to view.", Embeds: []types.Field{{Name: "WidgetBase"}}, Fields: []types.Field{{Name: "Scale", Doc: "Scale multiplies the plot DPI value, to change the overall scale\nof the rendered plot. Larger numbers produce larger scaling.\nTypically use larger numbers when generating plots for inclusion in\ndocuments or other cases where the overall plot size will be small."}, {Name: "Plot", Doc: "Plot is the Plot to display in this widget"}, {Name: "SetRangesFunc", Doc: "SetRangesFunc, if set, is called to adjust the data ranges\nafter the point when these ranges are updated based on the plot data."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plot", IDName: "plot", Doc: "Plot is a widget that renders a [plot.Plot] object.\nIf it is not [states.ReadOnly], the user can pan and zoom the graph.\nSee [PlotEditor] for an interactive interface for selecting columns to view.", Embeds: []types.Field{{Name: "WidgetBase"}}, Fields: []types.Field{{Name: "Plot", Doc: "Plot is the Plot to display in this widget"}, {Name: "SetRangesFunc", Doc: "SetRangesFunc, if set, is called to adjust the data ranges\nafter the point when these ranges are updated based on the plot data."}}}) // NewPlot returns a new [Plot] with the given optional parent: // Plot is a widget that renders a [plot.Plot] object. @@ -16,19 +16,12 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.Plot" // See [PlotEditor] for an interactive interface for selecting columns to view. func NewPlot(parent ...tree.Node) *Plot { return tree.New[Plot](parent...) } -// SetScale sets the [Plot.Scale]: -// Scale multiplies the plot DPI value, to change the overall scale -// of the rendered plot. Larger numbers produce larger scaling. -// Typically use larger numbers when generating plots for inclusion in -// documents or other cases where the overall plot size will be small. -func (t *Plot) SetScale(v float32) *Plot { t.Scale = v; return t } - // SetSetRangesFunc sets the [Plot.SetRangesFunc]: // SetRangesFunc, if set, is called to adjust the data ranges // after the point when these ranges are updated based on the plot data. func (t *Plot) SetSetRangesFunc(v func()) *Plot { t.SetRangesFunc = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Table] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "makeColumns", Doc: "makeColumns makes the Plans for columns", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"p"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "PlotStyle", Doc: "PlotStyle has the overall plot style parameters."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot/plotcore.PlotEditor", IDName: "plot-editor", Doc: "PlotEditor is a widget that provides an interactive 2D plot\nof selected columns of tabular data, represented by a [table.Table] into\na [table.Table]. Other types of tabular data can be converted into this format.\nThe user can change various options for the plot and also modify the underlying data.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Methods: []types.Method{{Name: "SaveSVG", Doc: "SaveSVG saves the plot to an svg -- first updates to ensure that plot is current", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SavePNG", Doc: "SavePNG saves the current plot to a png, capturing current render", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "SaveCSV", Doc: "SaveCSV saves the Table data to a csv (comma-separated values) file with headers (any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname", "delim"}}, {Name: "SaveAll", Doc: "SaveAll saves the current plot to a png, svg, and the data to a tsv -- full save\nAny extension is removed and appropriate extensions are added", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"fname"}}, {Name: "OpenCSV", Doc: "OpenCSV opens the Table data from a csv (comma-separated values) file (or any delim)", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"filename", "delim"}}, {Name: "setColumnsByName", Doc: "setColumnsByName turns columns on or off if their name contains\nthe given string.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}, Args: []string{"nameContains", "on"}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "table", Doc: "table is the table of data being plotted."}, {Name: "PlotStyle", Doc: "PlotStyle has the overall plot style parameters."}, {Name: "plot", Doc: "plot is the plot object."}, {Name: "svgFile", Doc: "current svg file"}, {Name: "dataFile", Doc: "current csv data file"}, {Name: "inPlot", Doc: "currently doing a plot"}, {Name: "columnsFrame"}, {Name: "plotWidget"}, {Name: "plotStyleModified"}}}) // NewPlotEditor returns a new [PlotEditor] with the given optional parent: // PlotEditor is a widget that provides an interactive 2D plot diff --git a/plot/typegen.go b/plot/typegen.go index 978ebbbfd2..9e484189eb 100644 --- a/plot/typegen.go +++ b/plot/typegen.go @@ -13,7 +13,7 @@ import ( var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisScales", IDName: "axis-scales", Doc: "AxisScales are the scaling options for how values are distributed\nalong an axis: Linear, Log, etc."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "NTicks", Doc: "NTicks is the desired number of ticks."}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the axis:\nLinear, Log, Inverted"}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.AxisStyle", IDName: "axis-style", Doc: "AxisStyle has style properties for the axis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Text", Doc: "Text has the text style parameters for the text label."}, {Name: "Line", Doc: "Line has styling properties for the axis line."}, {Name: "Padding", Doc: "Padding between the axis line and the data. Having\nnon-zero padding ensures that the data is never drawn\non the axis, thus making it easier to see."}, {Name: "NTicks", Doc: "NTicks is the desired number of ticks (actual likely will be different)."}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the axis:\nLinear, Log, Inverted"}, {Name: "TickText", Doc: "TickText has the text style for rendering tick labels,\nand is shared for actual rendering."}, {Name: "TickLine", Doc: "TickLine has line style for drawing tick lines."}, {Name: "TickLength", Doc: "TickLength is the length of tick lines."}}}) // SetText sets the [AxisStyle.Text]: // Text has the text style parameters for the text label. @@ -30,7 +30,7 @@ func (t *AxisStyle) SetLine(v LineStyle) *AxisStyle { t.Line = v; return t } func (t *AxisStyle) SetPadding(v units.Value) *AxisStyle { t.Padding = v; return t } // SetNTicks sets the [AxisStyle.NTicks]: -// NTicks is the desired number of ticks. +// NTicks is the desired number of ticks (actual likely will be different). func (t *AxisStyle) SetNTicks(v int) *AxisStyle { t.NTicks = v; return t } // SetScale sets the [AxisStyle.Scale]: @@ -102,7 +102,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LegendPosition var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Legend", IDName: "legend", Doc: "A Legend gives a description of the meaning of different\ndata elements of the plot. Each legend entry has a name\nand a thumbnail, where the thumbnail shows a small\nsample of the display style of the corresponding data.", Fields: []types.Field{{Name: "Style", Doc: "Style has the legend styling parameters."}, {Name: "Entries", Doc: "Entries are all of the LegendEntries described by this legend."}}}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Thumbnailer", IDName: "thumbnailer", Doc: "Thumbnailer wraps the Thumbnail method, which\ndraws the small image in a legend representing the\nstyle of data.", Methods: []types.Method{{Name: "Thumbnail", Doc: "Thumbnail draws an thumbnail representing\na legend entry. The thumbnail will usually show\na smaller representation of the style used\nto plot the corresponding data.", Args: []string{"pt"}}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Thumbnailer", IDName: "thumbnailer", Doc: "Thumbnailer wraps the Thumbnail method, which draws the small\nimage in a legend representing the style of data.", Methods: []types.Method{{Name: "Thumbnail", Doc: "Thumbnail draws an thumbnail representing a legend entry.\nThe thumbnail will usually show a smaller representation\nof the style used to plot the corresponding data.", Args: []string{"pt"}}}}) var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.LegendEntry", IDName: "legend-entry", Doc: "A LegendEntry represents a single line of a legend, it\nhas a name and an icon.", Fields: []types.Field{{Name: "Text", Doc: "text is the text associated with this entry."}, {Name: "Thumbs", Doc: "thumbs is a slice of all of the thumbnails styles"}}}) @@ -146,13 +146,13 @@ func (t *LineStyle) SetStep(v StepKind) *LineStyle { t.Step = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.StepKind", IDName: "step-kind", Doc: "StepKind specifies a form of a connection of two consecutive points."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XAxisStyle", IDName: "x-axis-style", Doc: "XAxisStyle has overall plot level styling properties for the XAxis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Column", Doc: "Column specifies the column to use for the common X axis in a table based plot.\nif empty or not found, the row number is used.\nThis optional for Bar plots, if present and Legend is also present,\nthen an extra space will be put between X values."}, {Name: "Rotation", Doc: "Rotation is the rotation of the X Axis labels, in degrees."}, {Name: "Label", Doc: "Label is the optional label to use for the XAxis instead of the default."}, {Name: "Range", Doc: "Range is the effective range of XAxis data to plot, where either end can be fixed."}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the X axis:\nLinear, Log, Inverted"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.XAxisStyle", IDName: "x-axis-style", Doc: "XAxisStyle has overall plot level styling properties for the XAxis.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Column", Doc: "Column specifies the column to use for the common X axis,\nfor [plot.NewTablePlot] [table.Table] driven plots.\nIf empty, standard Group-based role binding is used: the last column\nwithin the same group with Role=X is used."}, {Name: "Rotation", Doc: "Rotation is the rotation of the X Axis labels, in degrees."}, {Name: "Label", Doc: "Label is the optional label to use for the XAxis instead of the default."}, {Name: "Range", Doc: "Range is the effective range of XAxis data to plot, where either end can be fixed."}, {Name: "Scale", Doc: "Scale specifies how values are scaled along the X axis:\nLinear, Log, Inverted"}}}) // SetColumn sets the [XAxisStyle.Column]: -// Column specifies the column to use for the common X axis in a table based plot. -// if empty or not found, the row number is used. -// This optional for Bar plots, if present and Legend is also present, -// then an extra space will be put between X values. +// Column specifies the column to use for the common X axis, +// for [plot.NewTablePlot] [table.Table] driven plots. +// If empty, standard Group-based role binding is used: the last column +// within the same group with Role=X is used. func (t *XAxisStyle) SetColumn(v string) *XAxisStyle { t.Column = v; return t } // SetRotation sets the [XAxisStyle.Rotation]: @@ -279,7 +279,7 @@ func (t *PointStyle) SetSize(v units.Value) *PointStyle { t.Size = v; return t } var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Shapes", IDName: "shapes", Doc: "Shapes has the options for how to draw points in the plot."}) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for table-based plots."}, {Name: "Plotter", Doc: "Plotter is the type of plotter to use in plotting this data,\nfor table-based plots. Blank means use default ([plots.XY] is overall default)."}, {Name: "Role", Doc: "Role specifies a role for this item, used for table-based plots to indicate\nhow a particular column of data should be used."}, {Name: "Group", Doc: "Group specifies a group of related data items, used for table-based plots\nwhere different columns of data within the same Group play different Roles"}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.Style", IDName: "style", Doc: "Style contains the plot styling properties relevant across\nmost plot types. These properties apply to individual plot elements\nwhile the Plot properties applies to the overall plot itself.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Plot", Doc: "\tPlot has overall plot-level properties, which can be set by any\nplot element, and are updated first, before applying element-wise styles."}, {Name: "On", Doc: "On specifies whether to plot this item, for table-based plots."}, {Name: "Plotter", Doc: "Plotter is the type of plotter to use in plotting this data,\nfor [plot.NewTablePlot] [table.Table] driven plots.\nBlank means use default ([plots.XY] is overall default)."}, {Name: "Role", Doc: "Role specifies how a particular column of data should be used,\nfor [plot.NewTablePlot] [table.Table] driven plots."}, {Name: "Group", Doc: "Group specifies a group of related data items,\nfor [plot.NewTablePlot] [table.Table] driven plots,\nwhere different columns of data within the same Group play different Roles."}, {Name: "Range", Doc: "Range is the effective range of data to plot, where either end can be fixed."}, {Name: "Label", Doc: "Label provides an alternative label to use for axis, if set."}, {Name: "NoLegend", Doc: "NoLegend excludes this item from the legend when it otherwise would be included,\nfor [plot.NewTablePlot] [table.Table] driven plots.\nRole = Y values are included in the Legend by default."}, {Name: "NTicks", Doc: "NTicks sets the desired number of ticks for the axis, if > 0."}, {Name: "Line", Doc: "Line has style properties for drawing lines."}, {Name: "Point", Doc: "Point has style properties for drawing points."}, {Name: "Text", Doc: "Text has style properties for rendering text."}, {Name: "Width", Doc: "Width has various plot width properties."}}}) // SetPlot sets the [Style.Plot]: // @@ -294,17 +294,19 @@ func (t *Style) SetOn(v bool) *Style { t.On = v; return t } // SetPlotter sets the [Style.Plotter]: // Plotter is the type of plotter to use in plotting this data, -// for table-based plots. Blank means use default ([plots.XY] is overall default). +// for [plot.NewTablePlot] [table.Table] driven plots. +// Blank means use default ([plots.XY] is overall default). func (t *Style) SetPlotter(v PlotterName) *Style { t.Plotter = v; return t } // SetRole sets the [Style.Role]: -// Role specifies a role for this item, used for table-based plots to indicate -// how a particular column of data should be used. +// Role specifies how a particular column of data should be used, +// for [plot.NewTablePlot] [table.Table] driven plots. func (t *Style) SetRole(v Roles) *Style { t.Role = v; return t } // SetGroup sets the [Style.Group]: -// Group specifies a group of related data items, used for table-based plots -// where different columns of data within the same Group play different Roles +// Group specifies a group of related data items, +// for [plot.NewTablePlot] [table.Table] driven plots, +// where different columns of data within the same Group play different Roles. func (t *Style) SetGroup(v string) *Style { t.Group = v; return t } // SetRange sets the [Style.Range]: @@ -315,6 +317,12 @@ func (t *Style) SetRange(v minmax.Range64) *Style { t.Range = v; return t } // Label provides an alternative label to use for axis, if set. func (t *Style) SetLabel(v string) *Style { t.Label = v; return t } +// SetNoLegend sets the [Style.NoLegend]: +// NoLegend excludes this item from the legend when it otherwise would be included, +// for [plot.NewTablePlot] [table.Table] driven plots. +// Role = Y values are included in the Legend by default. +func (t *Style) SetNoLegend(v bool) *Style { t.NoLegend = v; return t } + // SetNTicks sets the [Style.NTicks]: // NTicks sets the desired number of ticks for the axis, if > 0. func (t *Style) SetNTicks(v int) *Style { t.NTicks = v; return t } @@ -335,7 +343,7 @@ func (t *Style) SetText(v TextStyle) *Style { t.Text = v; return t } // Width has various plot width properties. func (t *Style) SetWidth(v WidthStyle) *Style { t.Width = v; return t } -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.WidthStyle", IDName: "width-style", Doc: "WidthStyle contains various plot width properties relevant across\ndifferent plot types.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Cap", Doc: "Cap is the width of the caps drawn at the top of error bars.\nThe default is 10dp"}, {Name: "Offset", Doc: "Offset for Bar plot is the offset added to each X axis value\nrelative to the Stride computed value (X = offset + index * Stride)\nDefaults to 1."}, {Name: "Stride", Doc: "Stride for Bar plot is distance between bars. Defaults to 1."}, {Name: "Width", Doc: "Width for Bar plot is the width of the bars, which should be less than\nthe Stride (1 typically) to prevent bar overlap. Defaults to .8."}, {Name: "Pad", Doc: "Pad for Bar plot is additional space at start / end of data range,\nto keep bars from overflowing ends. This amount is subtracted from Offset\nand added to (len(Values)-1)*Stride -- no other accommodation for bar\nwidth is provided, so that should be built into this value as well.\nDefaults to 1."}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/plot.WidthStyle", IDName: "width-style", Doc: "WidthStyle contains various plot width properties relevant across\ndifferent plot types.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Fields: []types.Field{{Name: "Cap", Doc: "Cap is the width of the caps drawn at the top of error bars.\nThe default is 10dp"}, {Name: "Offset", Doc: "Offset for Bar plot is the offset added to each X axis value\nrelative to the Stride computed value (X = offset + index * Stride)\nDefaults to 0."}, {Name: "Stride", Doc: "Stride for Bar plot is distance between bars. Defaults to 1."}, {Name: "Width", Doc: "Width for Bar plot is the width of the bars, as a fraction of the Stride,\nto prevent bar overlap. Defaults to .8."}, {Name: "Pad", Doc: "Pad for Bar plot is additional space at start / end of data range,\nto keep bars from overflowing ends. This amount is subtracted from Offset\nand added to (len(Values)-1)*Stride -- no other accommodation for bar\nwidth is provided, so that should be built into this value as well.\nDefaults to 1."}}}) // SetCap sets the [WidthStyle.Cap]: // Cap is the width of the caps drawn at the top of error bars. @@ -345,7 +353,7 @@ func (t *WidthStyle) SetCap(v units.Value) *WidthStyle { t.Cap = v; return t } // SetOffset sets the [WidthStyle.Offset]: // Offset for Bar plot is the offset added to each X axis value // relative to the Stride computed value (X = offset + index * Stride) -// Defaults to 1. +// Defaults to 0. func (t *WidthStyle) SetOffset(v float64) *WidthStyle { t.Offset = v; return t } // SetStride sets the [WidthStyle.Stride]: @@ -353,8 +361,8 @@ func (t *WidthStyle) SetOffset(v float64) *WidthStyle { t.Offset = v; return t } func (t *WidthStyle) SetStride(v float64) *WidthStyle { t.Stride = v; return t } // SetWidth sets the [WidthStyle.Width]: -// Width for Bar plot is the width of the bars, which should be less than -// the Stride (1 typically) to prevent bar overlap. Defaults to .8. +// Width for Bar plot is the width of the bars, as a fraction of the Stride, +// to prevent bar overlap. Defaults to .8. func (t *WidthStyle) SetWidth(v float64) *WidthStyle { t.Width = v; return t } // SetPad sets the [WidthStyle.Pad]: From de3e5e1f6a95dfe22ce85a613da773568522ab3d Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 13 Nov 2024 11:13:12 -0800 Subject: [PATCH 295/311] plot highlighting for kai --- plot/plot.go | 29 ++++++++++++++++++----------- plot/plotcore/plot.go | 12 +++++++++--- plot/plots/xy.go | 19 +++++++++++++++++++ 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/plot/plot.go b/plot/plot.go index 361374f29e..cec08d9224 100644 --- a/plot/plot.go +++ b/plot/plot.go @@ -170,11 +170,10 @@ type Plot struct { // Legend is the plot's legend. Legend Legend - // plotters are drawn by calling their Plot method - // after the axes are drawn. + // Plotters are drawn by calling their Plot method after the axes are drawn. Plotters []Plotter - // size is the target size of the image to render to + // Size is the target size of the image to render to. Size image.Point // DPI is the dots per inch for rendering the image. @@ -185,12 +184,19 @@ type Plot struct { // PanZoom provides post-styling pan and zoom range factors. PanZoom PanZoom - // painter for rendering - Paint *paint.Context + // HighlightPlotter is the Plotter to highlight. Used for mouse hovering for example. + // It is the responsibility of the Plotter Plot function to implement highlighting. + HighlightPlotter Plotter + + // HighlightIndex is the index of the data point to highlight, for HighlightPlotter. + HighlightIndex int // pixels that we render into Pixels *image.RGBA `copier:"-" json:"-" xml:"-" edit:"-"` + // Paint is the painter for rendering + Paint *paint.Context + // Current plot bounding box in image coordinates, for plotting coordinates PlotBox math32.Box2 } @@ -387,11 +393,11 @@ func (pt *Plot) PY(v float64) float32 { // ClosestDataToPixel returns the Plotter data point closest to given pixel point, // in the Pixels image. -func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float32, pixel math32.Vector2, data Data, legend string) { +func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, plotterIndex, pointIndex int, dist float32, pixel math32.Vector2, data Data, legend string) { tp := math32.Vec2(float32(px), float32(py)) dist = float32(math32.MaxFloat32) - for _, p := range pt.Plotters { - dts, pxX, pxY := p.Data() + for pi, pl := range pt.Plotters { + dts, pxX, pxY := pl.Data() for i, ptx := range pxX { pty := pxY[i] pxy := math32.Vec2(ptx, pty) @@ -399,10 +405,11 @@ func (pt *Plot) ClosestDataToPixel(px, py int) (plt Plotter, idx int, dist float if d < dist { dist = d pixel = pxy - plt = p - idx = i + plt = pl + plotterIndex = pi + pointIndex = i data = dts - legend = pt.Legend.LegendForPlotter(p) + legend = pt.Legend.LegendForPlotter(pl) } } } diff --git a/plot/plotcore/plot.go b/plot/plotcore/plot.go index 4d770725a6..e0d4c1b30b 100644 --- a/plot/plotcore/plot.go +++ b/plot/plotcore/plot.go @@ -99,7 +99,6 @@ func (pt *Plot) Init() { pt.Plot.PanZoom.XOffset += dx pt.Plot.PanZoom.YOffset += dy pt.updatePlot() - pt.NeedsRender() }) pt.On(events.Scroll, func(e events.Event) { @@ -118,7 +117,6 @@ func (pt *Plot) Init() { pt.Plot.PanZoom.XScale *= xsc pt.Plot.PanZoom.YScale *= ysc pt.updatePlot() - pt.NeedsRender() }) } @@ -130,9 +128,17 @@ func (pt *Plot) WidgetTooltip(pos image.Point) (string, image.Point) { return pt.Tooltip, pt.DefaultTooltipPos() } wpos := pos.Sub(pt.Geom.ContentBBox.Min) - _, idx, dist, data, _, legend := pt.Plot.ClosestDataToPixel(wpos.X, wpos.Y) + plt, _, idx, dist, data, _, legend := pt.Plot.ClosestDataToPixel(wpos.X, wpos.Y) if dist <= 10 { + pt.Plot.HighlightPlotter = plt + pt.Plot.HighlightIndex = idx + pt.updatePlot() return fmt.Sprintf("%s[%d]: (%g, %g)", legend, idx, data.X, data.Y), pos + } else { + if pt.Plot.HighlightPlotter != nil { + pt.Plot.HighlightPlotter = nil + pt.updatePlot() + } } return pt.Tooltip, pt.DefaultTooltipPos() } diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 176e7d33e3..765ec29911 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -177,6 +177,9 @@ func (ln *XY) Plot(plt *plot.Plot) { pc.FillStyle.Color = nil if ln.Style.Line.SetStroke(plt) { + if plt.HighlightPlotter == ln { + pc.StrokeStyle.Width.Dots *= 1.5 + } prevX, prevY := ln.PX[0], ln.PY[0] pc.MoveTo(prevX, prevY) for i := 1; i < np; i++ { @@ -206,10 +209,26 @@ func (ln *XY) Plot(plt *plot.Plot) { pc.Stroke() } if ln.Style.Point.SetStroke(plt) { + origWidth := pc.StrokeStyle.Width.Dots for i, ptx := range ln.PX { pty := ln.PY[i] + if plt.HighlightPlotter == ln { + if i == plt.HighlightIndex { + pc.StrokeStyle.Width.Dots *= 1.5 + } else { + pc.StrokeStyle.Width.Dots = origWidth + } + } ln.Style.Point.DrawShape(pc, math32.Vec2(ptx, pty)) } + } else if plt.HighlightPlotter == ln { + op := ln.Style.Point.On + ln.Style.Point.On = plot.On + ln.Style.Point.SetStroke(plt) + ptx := ln.PX[plt.HighlightIndex] + pty := ln.PY[plt.HighlightIndex] + ln.Style.Point.DrawShape(pc, math32.Vec2(ptx, pty)) + ln.Style.Point.On = op } pc.FillStyle.Color = nil } From 3e6241122c986f1deb1fcf940982caa1d3d5f8ed Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Wed, 13 Nov 2024 17:19:20 -0800 Subject: [PATCH 296/311] tensor: only support one set of Row access methods, always taking cell = 0 (for clarity) for scalar case; add AppendRow for everybody -- use as the primary api for sim logging --- tensor/README.md | 4 +- tensor/base.go | 2 +- tensor/bool.go | 41 +-- tensor/create.go | 15 +- tensor/databrowser/filetree.go | 3 +- tensor/datafs/data.go | 14 +- tensor/examples/datafs-sim/sim.go | 43 ++- tensor/examples/datafs-sim/sim.goal | 41 ++- tensor/examples/planets/planets.go | 4 +- tensor/number.go | 71 ++--- tensor/rowmajor.go | 83 ++---- tensor/rows.go | 79 ++--- tensor/stats/cluster/plot.go | 22 +- tensor/stats/glm/glm.go | 18 +- tensor/stats/stats/group.go | 16 +- tensor/stats/stats/group_test.go | 4 +- tensor/stats/stats/quantiles.go | 8 +- tensor/stats/stats/stats_test.go | 8 +- tensor/string.go | 49 ++-- tensor/table/README.md | 6 +- tensor/table/indexes.go | 4 +- tensor/table/slicetable.go | 4 +- tensor/table/table_test.go | 22 +- tensor/tensor.go | 4 +- tensor/tensor_test.go | 14 +- tensor/tensorcore/table.go | 2 +- tensor/tmath/ops_test.go | 8 +- .../cogentcore_org-core-base-reflectx.go | 2 + yaegicore/nogui/cogentcore_org-core-tensor.go | 272 ++++++++---------- .../cogentcore_org-core-plot-plotcore.go | 28 +- .../symbols/cogentcore_org-core-plot-plots.go | 122 ++------ yaegicore/symbols/cogentcore_org-core-plot.go | 211 ++++++++------ 32 files changed, 500 insertions(+), 724 deletions(-) diff --git a/tensor/README.md b/tensor/README.md index 30ea46945d..4e9117bb26 100644 --- a/tensor/README.md +++ b/tensor/README.md @@ -99,10 +99,10 @@ str := ix.String1D(2) ```Go // value at row 3, cell 2 (flat index into entire `SubSpace` tensor for this row) // The row index will be indirected through any `Indexes` present on the Rows view. -val := ix.FloatRowCell(3, 2) +val := ix.FloatRow(3, 2) // string value at row 2, cell 0. this is safe for 1D and 2D+ shapes // and is a robust way to get 1D data from tensors of unknown shapes. -str := ix.FloatRowCell(2, 0) +str := ix.FloatRow(2, 0) ``` ```Go diff --git a/tensor/base.go b/tensor/base.go index bd47054963..38903ed56e 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -129,7 +129,7 @@ func (tsr *Base[T]) String1D(i int) string { return reflectx.ToString(tsr.Values[tsr.shape.Header+i]) } -func (tsr *Base[T]) StringRowCell(row, cell int) string { +func (tsr *Base[T]) StringRow(row, cell int) string { _, sz := tsr.shape.RowCellSize() return reflectx.ToString(tsr.Values[tsr.shape.Header+row*sz+cell]) } diff --git a/tensor/bool.go b/tensor/bool.go index e1256322c9..db98b71795 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -16,7 +16,7 @@ import ( ) // Bool is a tensor of bits backed by a [bitslice.Slice] for efficient storage -// of binary, boolean data. Bool does not support [Tensor.SubSpace] access +// of binary, boolean data. Bool does not support [RowMajor.SubSpace] access // and related methods due to the nature of the underlying data representation. type Bool struct { shape Shape @@ -127,6 +127,9 @@ func (tsr *Bool) RowTensor(row int) Values { return nil } // SetRowTensor not possible with Bool. func (tsr *Bool) SetRowTensor(val Values, row int) {} +// AppendRow not possible with Bool. +func (tsr *Bool) AppendRow(val Values) {} + ///////////////////// Bool func (tsr *Bool) Value(i ...int) bool { @@ -163,26 +166,18 @@ func (tsr *Bool) SetString(val string, i ...int) { } } -func (tsr *Bool) StringRowCell(row, cell int) string { +func (tsr *Bool) StringRow(row, cell int) string { _, sz := tsr.shape.RowCellSize() return reflectx.ToString(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bool) SetStringRowCell(val string, row, cell int) { +func (tsr *Bool) SetStringRow(val string, row, cell int) { if bv, err := reflectx.ToBool(val); err == nil { _, sz := tsr.shape.RowCellSize() tsr.Values.Set(bv, row*sz+cell) } } -func (tsr *Bool) StringRow(row int) string { - return tsr.StringRowCell(row, 0) -} - -func (tsr *Bool) SetStringRow(val string, row int) { - tsr.SetStringRowCell(val, row, 0) -} - ///////////////////// Floats func (tsr *Bool) Float(i ...int) float64 { @@ -201,24 +196,16 @@ func (tsr *Bool) SetFloat1D(val float64, off int) { tsr.Values.Set(Float64ToBool(val), off) } -func (tsr *Bool) FloatRowCell(row, cell int) float64 { +func (tsr *Bool) FloatRow(row, cell int) float64 { _, sz := tsr.shape.RowCellSize() return BoolToFloat64(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bool) SetFloatRowCell(val float64, row, cell int) { +func (tsr *Bool) SetFloatRow(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values.Set(Float64ToBool(val), row*sz+cell) } -func (tsr *Bool) FloatRow(row int) float64 { - return tsr.FloatRowCell(row, 0) -} - -func (tsr *Bool) SetFloatRow(val float64, row int) { - tsr.SetFloatRowCell(val, row, 0) -} - ///////////////////// Ints func (tsr *Bool) Int(i ...int) int { @@ -237,24 +224,16 @@ func (tsr *Bool) SetInt1D(val int, off int) { tsr.Values.Set(IntToBool(val), off) } -func (tsr *Bool) IntRowCell(row, cell int) int { +func (tsr *Bool) IntRow(row, cell int) int { _, sz := tsr.shape.RowCellSize() return BoolToInt(tsr.Values.Index(row*sz + cell)) } -func (tsr *Bool) SetIntRowCell(val int, row, cell int) { +func (tsr *Bool) SetIntRow(val int, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values.Set(IntToBool(val), row*sz+cell) } -func (tsr *Bool) IntRow(row int) int { - return tsr.IntRowCell(row, 0) -} - -func (tsr *Bool) SetIntRow(val int, row int) { - tsr.SetIntRowCell(val, row, 0) -} - ///////////////////// Bools func (tsr *Bool) Bool(i ...int) bool { diff --git a/tensor/create.go b/tensor/create.go index 0fe4b1c22e..78bee349f7 100644 --- a/tensor/create.go +++ b/tensor/create.go @@ -15,6 +15,12 @@ func NewFloat64Scalar(val float64) *Float64 { return NewNumberFromValues(val) } +// NewFloat32Scalar is a convenience method for a Tensor +// representation of a single float32 scalar value. +func NewFloat32Scalar(val float32) *Float32 { + return NewNumberFromValues(val) +} + // NewIntScalar is a convenience method for a Tensor // representation of a single int scalar value. func NewIntScalar(val int) *Int { @@ -34,6 +40,13 @@ func NewFloat64FromValues(vals ...float64) *Float64 { return NewNumberFromValues(vals...) } +// NewFloat32FromValues returns a new 1-dimensional tensor of given value type +// initialized directly from the given slice values, which are not copied. +// The resulting Tensor thus "wraps" the given values. +func NewFloat32FromValues(vals ...float32) *Float32 { + return NewNumberFromValues(vals...) +} + // NewIntFromValues returns a new 1-dimensional tensor of given value type // initialized directly from the given slice values, which are not copied. // The resulting Tensor thus "wraps" the given values. @@ -171,7 +184,7 @@ func NewFloat64SpacedLinear(start, stop Tensor, num int, endpoint bool) *Float64 } for r := range num { for i := range n { - tsr.SetFloatRowCell(start.Float1D(i)+float64(r)*step.Float1D(i), r, i) + tsr.SetFloatRow(start.Float1D(i)+float64(r)*step.Float1D(i), r, i) } } return tsr diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index ff1b31957b..55784b745c 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -246,7 +246,8 @@ func (fn *FileNode) PlotFile() { return } pl := br.NewTabPlot(ptab, dt) - pl.Options.Title = df + _ = pl + // pl.Options.Title = df // TODO: apply column and plot level options. br.Update() } diff --git a/tensor/datafs/data.go b/tensor/datafs/data.go index 83a53d7417..76cd3a5f10 100644 --- a/tensor/datafs/data.go +++ b/tensor/datafs/data.go @@ -148,7 +148,7 @@ func (d *Data) KnownFileInfo() fileinfo.Known { // This is the actual underlying data, so make a copy if it can be // unintentionally modified or retained more than for immediate use. func (d *Data) Bytes() []byte { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return nil } return d.Data.AsValues().Bytes() @@ -156,7 +156,7 @@ func (d *Data) Bytes() []byte { // AsString returns data as scalar string. func (d *Data) AsString() string { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return "" } return d.Data.String1D(0) @@ -164,7 +164,7 @@ func (d *Data) AsString() string { // SetString sets scalar data value from given string. func (d *Data) SetString(v string) { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return } d.Data.SetString1D(v, 0) @@ -172,7 +172,7 @@ func (d *Data) SetString(v string) { // AsFloat64 returns data as a scalar float64 (first element of tensor). func (d *Data) AsFloat64() float64 { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return 0 } return d.Data.Float1D(0) @@ -180,7 +180,7 @@ func (d *Data) AsFloat64() float64 { // SetFloat64 sets scalar data value from given float64. func (d *Data) SetFloat64(v float64) { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return } d.Data.SetFloat1D(v, 0) @@ -198,7 +198,7 @@ func (d *Data) SetFloat32(v float32) { // AsInt returns data as a scalar int (first element of tensor). func (d *Data) AsInt() int { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return 0 } return d.Data.Int1D(0) @@ -206,7 +206,7 @@ func (d *Data) AsInt() int { // SetInt sets scalar data value from given int. func (d *Data) SetInt(v int) { - if d.Data == nil { + if d.Data == nil || d.Data.NumDims() == 0 || d.Data.Len() == 0 { return } d.Data.SetInt1D(v, 0) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 15604d73e3..9e56fd468d 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -12,7 +12,7 @@ import ( "math/rand/v2" "cogentcore.org/core/core" - "cogentcore.org/core/plot/plotcore" + "cogentcore.org/core/plot" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" "cogentcore.org/core/tensor/datafs" @@ -62,33 +62,28 @@ func (ss *Sim) RunStats(tm Times) { func (ss *Sim) ConfigStats() { ss.Stats, _ = ss.Root.Mkdir("Stats") - mx := ss.Config.Value("Max").(*tensor.Int) ctrs := []Times{Run, Epoch, Trial} for _, ctr := range ctrs { ss.AddStat(func(tm Times) { - name := ctr.String() if tm > ctr { return } + name := ctr.String() ctv := ss.Counters[ctr] - mxi := mx.Value1D(int(tm)) td := ss.Stats.RecycleDir(tm.String()) cd := ss.Stats.RecycleDir("Current") datafs.Scalar[int](cd, name).SetInt1D(ctv, 0) - tv := datafs.Value[int](td, name, mxi) - tv.SetInt1D(ctv, ss.Counters[tm]) - if ps := plotcore.GetPlotStylers(tv.Metadata()); ps == nil { - ps = plotcore.NewPlotStylers() - ps.ColumnStyler(name, func(co *plotcore.ColumnOptions) { - co.Range.FixMin = true - co.On = false + tv := datafs.Value[int](td, name) + tv.AppendRow(tensor.NewIntScalar(ctv)) + if ps := plot.GetStylersFrom(tv); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0) }) - if tm == ctr { - ps.PlotStyler(func(po *plotcore.PlotOptions) { - po.XAxis = name - }) - } - plotcore.SetPlotStylers(tv.Metadata(), ps) + plot.SetStylersTo(tv, ps) + } + if tm > Trial { + subd := ss.Stats.RecycleDir((tm - 1).String()) + subd.Value(name).(tensor.Values).SetNumRows(0) } }) } @@ -98,8 +93,6 @@ func (ss *Sim) ConfigStats() { td := ss.Stats.RecycleDir(tm.String()) switch tm { case Trial: - ctv := ss.Counters[tm] - mxi := mx.Value1D(int(tm)) cd := ss.Stats.RecycleDir("Current") sse := rand.Float32() terr := float32(1) @@ -108,16 +101,16 @@ func (ss *Sim) ConfigStats() { } datafs.Scalar[float32](cd, sseName).SetFloat(float64(sse), 0) datafs.Scalar[float32](cd, errName).SetFloat(float64(terr), 0) - datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) - datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat32Scalar(sse)) + datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat32Scalar(terr)) case Epoch: - ctv := ss.Counters[tm] - mxi := mx.Value1D(int(tm)) trld, _ := ss.Stats.Mkdir(Trial.String()) sse := stats.StatMean.Call(trld.Value(sseName)).Float1D(0) terr := stats.StatMean.Call(trld.Value(errName)).Float1D(0) - datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) - datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat64Scalar(sse)) + datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat64Scalar(terr)) + trld.Value(sseName).(tensor.Values).SetNumRows(0) + trld.Value(errName).(tensor.Values).SetNumRows(0) } }) } diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal index ce75d8bcfa..db2fd5d7d7 100644 --- a/tensor/examples/datafs-sim/sim.goal +++ b/tensor/examples/datafs-sim/sim.goal @@ -10,7 +10,7 @@ import ( "math/rand/v2" "cogentcore.org/core/core" - "cogentcore.org/core/plot/plotcore" + "cogentcore.org/core/plot" "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" "cogentcore.org/core/tensor/datafs" @@ -60,7 +60,6 @@ func (ss *Sim) RunStats(tm Times) { func (ss *Sim) ConfigStats() { ss.Stats, _ = ss.Root.Mkdir("Stats") - mx := ss.Config.Value("Max").(*tensor.Int) ctrs := []Times{Run, Epoch, Trial} for _, ctr := range ctrs { ss.AddStat(func(tm Times) { @@ -69,24 +68,20 @@ func (ss *Sim) ConfigStats() { } name := ctr.String() ctv := ss.Counters[ctr] - mxi := mx.Value1D(int(tm)) td := ss.Stats.RecycleDir(tm.String()) cd := ss.Stats.RecycleDir("Current") datafs.Scalar[int](cd, name).SetInt1D(ctv, 0) - tv := datafs.Value[int](td, name, mxi) - tv.SetInt1D(ctv, ss.Counters[tm]) - if ps := plotcore.GetPlotStylers(tv.Metadata()); ps == nil { - ps = plotcore.NewPlotStylers() - ps.ColumnStyler(name, func(co *plotcore.ColumnOptions) { - co.Range.FixMin = true - co.On = false + tv := datafs.Value[int](td, name) + tv.AppendRow(tensor.NewIntScalar(ctv)) + if ps := plot.GetStylersFrom(tv); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0) }) - if tm == ctr { - ps.PlotStyler(func(po *plotcore.PlotOptions) { - po.XAxis = name - }) - } - plotcore.SetPlotStylers(tv.Metadata(), ps) + plot.SetStylersTo(tv, ps) + } + if tm > Trial { + subd := ss.Stats.RecycleDir((tm - 1).String()) + subd.Value(name).(tensor.Values).SetNumRows(0) } }) } @@ -96,8 +91,6 @@ func (ss *Sim) ConfigStats() { td := ss.Stats.RecycleDir(tm.String()) switch tm { case Trial: - ctv := ss.Counters[tm] - mxi := mx.Value1D(int(tm)) cd := ss.Stats.RecycleDir("Current") sse := rand.Float32() terr := float32(1) @@ -106,16 +99,16 @@ func (ss *Sim) ConfigStats() { } datafs.Scalar[float32](cd, sseName).SetFloat(float64(sse), 0) datafs.Scalar[float32](cd, errName).SetFloat(float64(terr), 0) - datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) - datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat32Scalar(sse)) + datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat32Scalar(terr)) case Epoch: - ctv := ss.Counters[tm] - mxi := mx.Value1D(int(tm)) trld, _ := ss.Stats.Mkdir(Trial.String()) sse := stats.StatMean.Call(trld.Value(sseName)).Float1D(0) terr := stats.StatMean.Call(trld.Value(errName)).Float1D(0) - datafs.Value[float32](td, sseName, mxi).SetFloat1D(float64(sse), ctv) - datafs.Value[float32](td, errName, mxi).SetFloat1D(float64(terr), ctv) + datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat64Scalar(sse)) + datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat64Scalar(terr)) + trld.Value(sseName).(tensor.Values).SetNumRows(0) + trld.Value(errName).(tensor.Values).SetNumRows(0) } }) } diff --git a/tensor/examples/planets/planets.go b/tensor/examples/planets/planets.go index d72cea7f3d..9b511fa92c 100644 --- a/tensor/examples/planets/planets.go +++ b/tensor/examples/planets/planets.go @@ -39,9 +39,9 @@ func AnalyzePlanets(dir *datafs.Data) { decade := Planets.AddFloat64Column("decade") year := Planets.Column("year") for row := range Planets.NumRows() { - yr := year.FloatRow(row) + yr := year.FloatRow(row, 0) dec := math.Floor(yr/10) * 10 - decade.SetFloatRowCell(dec, row, 0) + decade.SetFloatRow(dec, row, 0) } stats.TableGroups(dir, Planets, "method", "decade") diff --git a/tensor/number.go b/tensor/number.go index 0edd5250c7..34f9597211 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -137,29 +137,13 @@ func (tsr Number[T]) SetString1D(val string, i int) { } } -func (tsr *Number[T]) SetStringRowCell(val string, row, cell int) { +func (tsr *Number[T]) SetStringRow(val string, row, cell int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { _, sz := tsr.shape.RowCellSize() tsr.Values[tsr.shape.Header+row*sz+cell] = T(fv) } } -// StringRow returns the value at given row (outermost dimension). -// It is a convenience wrapper for StringRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (tsr *Number[T]) StringRow(row int) string { - return tsr.StringRowCell(row, 0) -} - -// SetStringRow sets the value at given row (outermost dimension). -// It is a convenience wrapper for SetStringRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (tsr *Number[T]) SetStringRow(val string, row int) { - tsr.SetStringRowCell(val, row, 0) -} - ///////////////////// Floats func (tsr *Number[T]) Float(i ...int) float64 { @@ -178,34 +162,17 @@ func (tsr *Number[T]) SetFloat1D(val float64, i int) { tsr.Values[tsr.shape.Header+i] = T(val) } -func (tsr *Number[T]) FloatRowCell(row, cell int) float64 { +func (tsr *Number[T]) FloatRow(row, cell int) float64 { _, sz := tsr.shape.RowCellSize() i := row*sz + cell return float64(tsr.Values[tsr.shape.Header+i]) } -func (tsr *Number[T]) SetFloatRowCell(val float64, row, cell int) { +func (tsr *Number[T]) SetFloatRow(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[tsr.shape.Header+row*sz+cell] = T(val) } -// FloatRow returns the value at given row (outermost dimension). -// It is a convenience wrapper for FloatRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (tsr *Number[T]) FloatRow(row int) float64 { - return tsr.FloatRowCell(row, 0) -} - -// SetFloatRow sets the value at given row (outermost dimension). -// Row is indirected through the [Indexed.Indexes]. -// It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (tsr *Number[T]) SetFloatRow(val float64, row int) { - tsr.SetFloatRowCell(val, row, 0) -} - ///////////////////// Ints func (tsr *Number[T]) Int(i ...int) int { @@ -224,33 +191,17 @@ func (tsr *Number[T]) SetInt1D(val int, i int) { tsr.Values[tsr.shape.Header+i] = T(val) } -func (tsr *Number[T]) IntRowCell(row, cell int) int { +func (tsr *Number[T]) IntRow(row, cell int) int { _, sz := tsr.shape.RowCellSize() i := row*sz + cell return int(tsr.Values[tsr.shape.Header+i]) } -func (tsr *Number[T]) SetIntRowCell(val int, row, cell int) { +func (tsr *Number[T]) SetIntRow(val int, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[tsr.shape.Header+row*sz+cell] = T(val) } -// IntRow returns the value at given row (outermost dimension). -// It is a convenience wrapper for IntRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (tsr *Number[T]) IntRow(row int) int { - return tsr.IntRowCell(row, 0) -} - -// SetIntRow sets the value at given row (outermost dimension). -// It is a convenience wrapper for SetIntRowCell(row, 0), providing robust -// operations on 1D and higher-dimensional data (which nevertheless should -// generally be processed separately in ways that treat it properly). -func (tsr *Number[T]) SetIntRow(val int, row int) { - tsr.SetIntRowCell(val, row, 0) -} - // SetZeros is simple convenience function initialize all values to 0 func (tsr *Number[T]) SetZeros() { n := len(tsr.Values) @@ -340,7 +291,7 @@ func (tsr *Number[T]) SubSpace(offs ...int) Values { return rt } -// RowTensor is a convenience version of [Tensor.SubSpace] to return the +// RowTensor is a convenience version of [RowMajor.SubSpace] to return the // SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. func (tsr *Number[T]) RowTensor(row int) Values { @@ -354,3 +305,13 @@ func (tsr *Number[T]) SetRowTensor(val Values, row int) { mx := min(val.Len(), cells) tsr.CopyCellsFrom(val, st, 0, mx) } + +// AppendRow adds a row and sets values to given values. +func (tsr *Number[T]) AppendRow(val Values) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow := tsr.DimSize(0) + tsr.SetNumRows(nrow + 1) + tsr.SetRowTensor(val, nrow) +} diff --git a/tensor/rowmajor.go b/tensor/rowmajor.go index d700d39999..ab86add1da 100644 --- a/tensor/rowmajor.go +++ b/tensor/rowmajor.go @@ -23,89 +23,48 @@ type RowMajor interface { // extract arbitrary subspaces along ranges of each dimension. SubSpace(offs ...int) Values - // RowTensor is a convenience version of [Tensor.SubSpace] to return the + // RowTensor is a convenience version of [RowMajor.SubSpace] to return the // SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. RowTensor(row int) Values - // SetRowTensor sets the values of the [Tensor.SubSpace] at given row to given values. + // SetRowTensor sets the values of the [RowMajor.SubSpace] at given row to given values. SetRowTensor(val Values, row int) - ///////////////////// Floats + // AppendRow adds a row and sets values to given values. + AppendRow(val Values) - // FloatRowCell returns the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - FloatRowCell(row, cell int) float64 - - // SetFloatRowCell sets the value at given row and cell, where row is the outermost - // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - SetFloatRowCell(val float64, row, cell int) + //////// Floats - // FloatRow returns the value at given row (outermost dimension). - // It is a convenience wrapper for FloatRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - FloatRow(row int) float64 + // FloatRow returns the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions (0 for scalar). + FloatRow(row, cell int) float64 - // SetFloatRow sets the value at given row (outermost dimension). - // It is a convenience wrapper for SetFloatRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - SetFloatRow(val float64, row int) + // SetFloatRow sets the value at given row and cell, where row is the outermost + // dimension, and cell is a 1D index into remaining inner dimensions. + SetFloatRow(val float64, row, cell int) - ///////////////////// Ints + //////// Ints - // IntRowCell returns the value at given row and cell, where row is the outermost + // IntRow returns the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - IntRowCell(row, cell int) int + IntRow(row, cell int) int - // SetIntRowCell sets the value at given row and cell, where row is the outermost + // SetIntRow sets the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. - // [Rows] tensors index along the row, and use this interface extensively. - // This is useful for lists of patterns, and the [table.Table] container. - SetIntRowCell(val int, row, cell int) + SetIntRow(val int, row, cell int) - // IntRow returns the value at given row (outermost dimension). - // It is a convenience wrapper for IntRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - IntRow(row int) int + //////// Strings - // SetIntRow sets the value at given row (outermost dimension). - // It is a convenience wrapper for SetIntRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - SetIntRow(val int, row int) - - ///////////////////// Strings - - // StringRowCell returns the value at given row and cell, where row is the outermost + // StringRow returns the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. - StringRowCell(row, cell int) string + StringRow(row, cell int) string - // SetStringRowCell sets the value at given row and cell, where row is the outermost + // SetStringRow sets the value at given row and cell, where row is the outermost // dimension, and cell is a 1D index into remaining inner dimensions. // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. - SetStringRowCell(val string, row, cell int) - - // StringRow returns the value at given row (outermost dimension). - // It is a convenience wrapper for StringRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - StringRow(row int) string - - // SetStringRow sets the value at given row (outermost dimension). - // It is a convenience wrapper for SetStringRowCell(row, 0), providing robust - // operations on 1D and higher-dimensional data (which nevertheless should - // generally be processed separately in ways that treat it properly). - SetStringRow(val string, row int) + SetStringRow(val string, row, cell int) } diff --git a/tensor/rows.go b/tensor/rows.go index 5f98d0dec8..69c42cfc92 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -187,7 +187,7 @@ func (rw *Rows) ExcludeMissing() { //types:add rw.IndexesNeeded() ni := rw.NumRows() for i := ni - 1; i >= 0; i-- { - if math.IsNaN(rw.Tensor.FloatRowCell(rw.Indexes[i], 0)) { + if math.IsNaN(rw.Tensor.FloatRow(rw.Indexes[i], 0)) { rw.Indexes = append(rw.Indexes[:i], rw.Indexes[i+1:]...) } } @@ -260,11 +260,11 @@ func CompareAscending[T cmp.Ordered](a, b T, ascending bool) int { func (rw *Rows) Sort(ascending bool) { if rw.Tensor.IsString() { rw.SortFunc(func(tsr Values, i, j int) int { - return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) + return CompareAscending(tsr.StringRow(i, 0), tsr.StringRow(j, 0), ascending) }) } else { rw.SortFunc(func(tsr Values, i, j int) int { - return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) + return CompareAscending(tsr.FloatRow(i, 0), tsr.FloatRow(j, 0), ascending) }) } } @@ -288,11 +288,11 @@ func (rw *Rows) SortStableFunc(cmp func(tsr Values, i, j int) int) { func (rw *Rows) SortStable(ascending bool) { if rw.Tensor.IsString() { rw.SortStableFunc(func(tsr Values, i, j int) int { - return CompareAscending(tsr.StringRowCell(i, 0), tsr.StringRowCell(j, 0), ascending) + return CompareAscending(tsr.StringRow(i, 0), tsr.StringRow(j, 0), ascending) }) } else { rw.SortStableFunc(func(tsr Values, i, j int) int { - return CompareAscending(tsr.FloatRowCell(i, 0), tsr.FloatRowCell(j, 0), ascending) + return CompareAscending(tsr.FloatRow(i, 0), tsr.FloatRow(j, 0), ascending) }) } } @@ -340,7 +340,7 @@ type FilterOptions struct { //types:add func (rw *Rows) FilterString(str string, opts FilterOptions) { //types:add lowstr := strings.ToLower(str) rw.Filter(func(tsr Values, row int) bool { - val := tsr.StringRowCell(row, 0) + val := tsr.StringRow(row, 0) has := false switch { case opts.Contains && opts.IgnoreCase: @@ -461,20 +461,20 @@ func (rw *Rows) SetFloat(val float64, i ...int) { rw.Tensor.SetFloat(val, ic...) } -// FloatRowCell returns the value at given row and cell, +// FloatRow returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (rw *Rows) FloatRowCell(row, cell int) float64 { - return rw.Tensor.FloatRowCell(rw.RowIndex(row), cell) +func (rw *Rows) FloatRow(row, cell int) float64 { + return rw.Tensor.FloatRow(rw.RowIndex(row), cell) } -// SetFloatRowCell sets the value at given row and cell, +// SetFloatRow sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (rw *Rows) SetFloatRowCell(val float64, row, cell int) { - rw.Tensor.SetFloatRowCell(val, rw.RowIndex(row), cell) +func (rw *Rows) SetFloatRow(val float64, row, cell int) { + rw.Tensor.SetFloatRow(val, rw.RowIndex(row), cell) } // Float1D is somewhat expensive if indexes are set, because it needs to convert @@ -495,14 +495,6 @@ func (rw *Rows) SetFloat1D(val float64, i int) { rw.SetFloat(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -func (rw *Rows) FloatRow(row int) float64 { - return rw.FloatRowCell(row, 0) -} - -func (rw *Rows) SetFloatRow(val float64, row int) { - rw.SetFloatRowCell(val, row, 0) -} - ///////////////////// Strings // StringValue returns the value of given index as a string. @@ -527,20 +519,20 @@ func (rw *Rows) SetString(val string, i ...int) { rw.Tensor.SetString(val, ic...) } -// StringRowCell returns the value at given row and cell, +// StringRow returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (rw *Rows) StringRowCell(row, cell int) string { - return rw.Tensor.StringRowCell(rw.RowIndex(row), cell) +func (rw *Rows) StringRow(row, cell int) string { + return rw.Tensor.StringRow(rw.RowIndex(row), cell) } -// SetStringRowCell sets the value at given row and cell, +// SetStringRow sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (rw *Rows) SetStringRowCell(val string, row, cell int) { - rw.Tensor.SetStringRowCell(val, rw.RowIndex(row), cell) +func (rw *Rows) SetStringRow(val string, row, cell int) { + rw.Tensor.SetStringRow(val, rw.RowIndex(row), cell) } // String1D is somewhat expensive if indexes are set, because it needs to convert @@ -561,14 +553,6 @@ func (rw *Rows) SetString1D(val string, i int) { rw.SetString(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -func (rw *Rows) StringRow(row int) string { - return rw.StringRowCell(row, 0) -} - -func (rw *Rows) SetStringRow(val string, row int) { - rw.SetStringRowCell(val, row, 0) -} - ///////////////////// Ints // Int returns the value of given index as an int. @@ -594,20 +578,20 @@ func (rw *Rows) SetInt(val int, i ...int) { rw.Tensor.SetInt(val, ic...) } -// IntRowCell returns the value at given row and cell, +// IntRow returns the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (rw *Rows) IntRowCell(row, cell int) int { - return rw.Tensor.IntRowCell(rw.RowIndex(row), cell) +func (rw *Rows) IntRow(row, cell int) int { + return rw.Tensor.IntRow(rw.RowIndex(row), cell) } -// SetIntRowCell sets the value at given row and cell, +// SetIntRow sets the value at given row and cell, // where row is outermost dim, and cell is 1D index into remaining inner dims. // Row is indirected through the [Rows.Indexes]. // This is the preferred interface for all Rows operations. -func (rw *Rows) SetIntRowCell(val int, row, cell int) { - rw.Tensor.SetIntRowCell(val, rw.RowIndex(row), cell) +func (rw *Rows) SetIntRow(val int, row, cell int) { + rw.Tensor.SetIntRow(val, rw.RowIndex(row), cell) } // Int1D is somewhat expensive if indexes are set, because it needs to convert @@ -628,14 +612,6 @@ func (rw *Rows) SetInt1D(val int, i int) { rw.SetInt(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -func (rw *Rows) IntRow(row int) int { - return rw.IntRowCell(row, 0) -} - -func (rw *Rows) SetIntRow(val int, row int) { - rw.SetIntRowCell(val, row, 0) -} - ///////////////////// SubSpaces // SubSpace returns a new tensor with innermost subspace at given @@ -666,5 +642,12 @@ func (rw *Rows) SetRowTensor(val Values, row int) { rw.Tensor.SetRowTensor(val, rw.RowIndex(row)) } +// AppendRow adds a row and sets values to given values. +func (rw *Rows) AppendRow(val Values) { + nrow := rw.Tensor.DimSize(0) + rw.AddRows(1) + rw.Tensor.SetRowTensor(val, nrow) +} + // check for interface impl var _ RowMajor = (*Rows)(nil) diff --git a/tensor/stats/cluster/plot.go b/tensor/stats/cluster/plot.go index 6ab209ef4c..dba9039850 100644 --- a/tensor/stats/cluster/plot.go +++ b/tensor/stats/cluster/plot.go @@ -33,29 +33,29 @@ func (nn *Node) Plot(pt *table.Table, dmat, labels tensor.Tensor) { lbl := pt.ColumnByIndex(2) if nn.IsLeaf() { pt.SetNumRows(row + 1) - xc.SetFloatRow(nn.ParDist, row) - yc.SetFloatRow(nn.Y, row) + xc.SetFloatRow(nn.ParDist, row, 0) + yc.SetFloatRow(nn.Y, row, 0) if labels.Len() > nn.Index { - lbl.SetStringRow(labels.StringValue(nn.Index), row) + lbl.SetStringRow(labels.StringValue(nn.Index), row, 0) } } else { for _, kn := range nn.Kids { pt.SetNumRows(row + 2) - xc.SetFloatRow(nn.ParDist, row) - yc.SetFloatRow(kn.Y, row) + xc.SetFloatRow(nn.ParDist, row, 0) + yc.SetFloatRow(kn.Y, row, 0) row++ - xc.SetFloatRow(nn.ParDist+nn.Dist, row) - yc.SetFloatRow(kn.Y, row) + xc.SetFloatRow(nn.ParDist+nn.Dist, row, 0) + yc.SetFloatRow(kn.Y, row, 0) kn.Plot(pt, dmat, labels) row = pt.NumRows() pt.SetNumRows(row + 1) - xc.SetFloatRow(nn.ParDist, row) - yc.SetFloatRow(kn.Y, row) + xc.SetFloatRow(nn.ParDist, row, 0) + yc.SetFloatRow(kn.Y, row, 0) row++ } pt.SetNumRows(row + 1) - xc.SetFloatRow(nn.ParDist, row) - yc.SetFloatRow(nn.Y, row) + xc.SetFloatRow(nn.ParDist, row, 0) + yc.SetFloatRow(nn.Y, row, 0) } } diff --git a/tensor/stats/glm/glm.go b/tensor/stats/glm/glm.go index 6530864047..51c79d0c06 100644 --- a/tensor/stats/glm/glm.go +++ b/tensor/stats/glm/glm.go @@ -194,24 +194,24 @@ func (glm *GLM) Run() { for di := 0; di < nDv; di++ { pred := 0.0 for ii := 0; ii < nIv; ii++ { - pred += glm.Coeff.Float(di, ii) * iv.FloatRowCell(row, ii) + pred += glm.Coeff.Float(di, ii) * iv.FloatRow(row, ii) } if !glm.ZeroOffset { pred += glm.Coeff.Float(di, nIv) } - targ := dv.FloatRowCell(row, di) + targ := dv.FloatRow(row, di) err := targ - pred sse += err * err for ii := 0; ii < nIv; ii++ { - dc.Values[di*nCi+ii] += err * iv.FloatRowCell(row, ii) + dc.Values[di*nCi+ii] += err * iv.FloatRow(row, ii) } if !glm.ZeroOffset { dc.Values[di*nCi+nIv] += err } if lastItr { - pv.SetFloatRowCell(pred, row, di) + pv.SetFloatRow(pred, row, di) if ev != nil { - ev.SetFloatRowCell(err, row, di) + ev.SetFloatRow(err, row, di) } } } @@ -252,8 +252,8 @@ func (glm *GLM) Run() { for i := 0; i < n; i++ { row := dt.Indexes[i] for di := 0; di < nDv; di++ { - obsMeans[di] += dv.FloatRowCell(row, di) - errMeans[di] += ev.FloatRowCell(row, di) + obsMeans[di] += dv.FloatRow(row, di) + errMeans[di] += ev.FloatRow(row, di) } } for di := 0; di < nDv; di++ { @@ -265,9 +265,9 @@ func (glm *GLM) Run() { for i := 0; i < n; i++ { row := dt.Indexes[i] for di := 0; di < nDv; di++ { - o := dv.FloatRowCell(row, di) - obsMeans[di] + o := dv.FloatRow(row, di) - obsMeans[di] glm.ObsVariance[di] += o * o - e := ev.FloatRowCell(row, di) - errMeans[di] + e := ev.FloatRow(row, di) - errMeans[di] glm.ErrVariance[di] += e * e } } diff --git a/tensor/stats/stats/group.go b/tensor/stats/stats/group.go index 20defc58e3..4ee553d5ad 100644 --- a/tensor/stats/stats/group.go +++ b/tensor/stats/stats/group.go @@ -35,7 +35,7 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) error { n := r - start it := datafs.Value[int](dir, val, n) for j := range n { - it.SetIntRow(srt.Indexes[start+j], j) // key to indirect through sort indexes + it.SetIntRow(srt.Indexes[start+j], j, 0) // key to indirect through sort indexes } } @@ -53,9 +53,9 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) error { srt.SortStable(tensor.Ascending) start := 0 if tsr.IsString() { - lastVal := srt.StringRow(0) + lastVal := srt.StringRow(0, 0) for r := range nr { - v := srt.StringRow(r) + v := srt.StringRow(r, 0) if v != lastVal { makeIdxs(td, srt, lastVal, start, r) start = r @@ -66,9 +66,9 @@ func Groups(dir *datafs.Data, tsrs ...tensor.Tensor) error { makeIdxs(td, srt, lastVal, start, nr) } } else { - lastVal := srt.FloatRow(0) + lastVal := srt.FloatRow(0, 0) for r := range nr { - v := srt.FloatRow(r) + v := srt.FloatRow(r, 0) if v != lastVal { makeIdxs(td, srt, tensor.Float64ToString(lastVal), start, r) start = r @@ -105,7 +105,7 @@ func GroupAll(dir *datafs.Data, tsrs ...tensor.Tensor) error { td, _ := gd.Mkdir("All") it := datafs.Value[int](td, "All", nr) for j := range nr { - it.SetIntRow(tsr.RowIndex(j), j) // key to indirect through any existing indexes + it.SetIntRow(tsr.RowIndex(j), j, 0) // key to indirect through any existing indexes } return nil } @@ -140,7 +140,7 @@ func GroupStats(dir *datafs.Data, stat Stats, tsrs ...tensor.Tensor) error { if gv == nil { gtsr := datafs.Value[string](sgd, gpnm, nv) for i, v := range vals { - gtsr.SetStringRow(v.Metadata().Name(), i) + gtsr.SetStringRow(v.Metadata().Name(), i, 0) } } for _, tsr := range tsrs { @@ -150,7 +150,7 @@ func GroupStats(dir *datafs.Data, stat Stats, tsrs ...tensor.Tensor) error { idx := tensor.AsIntSlice(v) sg := tensor.NewRows(tsr.AsValues(), idx...) stout := stat.Call(sg) - sv.SetFloatRow(stout.Float1D(0), i) + sv.SetFloatRow(stout.Float1D(0), i, 0) } } } diff --git a/tensor/stats/stats/group_test.go b/tensor/stats/stats/group_test.go index e47bbc2666..053655b044 100644 --- a/tensor/stats/stats/group_test.go +++ b/tensor/stats/stats/group_test.go @@ -22,8 +22,8 @@ func TestGroup(t *testing.T) { if i >= 2 { gp = "B" } - dt.Column("Name").SetStringRow(gp, i) - dt.Column("Value").SetFloatRow(float64(i), i) + dt.Column("Name").SetStringRow(gp, i, 0) + dt.Column("Value").SetFloatRow(float64(i), i, 0) } dir, _ := datafs.NewDir("Group") err := TableGroups(dir, dt, "Name") diff --git a/tensor/stats/stats/quantiles.go b/tensor/stats/stats/quantiles.go index 02ea725447..4aeb593f2f 100644 --- a/tensor/stats/stats/quantiles.go +++ b/tensor/stats/stats/quantiles.go @@ -56,13 +56,13 @@ func QuantilesOut(in, qs tensor.Tensor, out tensor.Values) error { lwi := math.Floor(qi) lwii := int(lwi) if lwii >= sz { - val = sin.FloatRow(sz) + val = sin.FloatRow(sz, 0) } else if lwii < 0 { - val = sin.FloatRow(0) + val = sin.FloatRow(0, 0) } else { phi := qi - lwi - lwv := sin.FloatRow(lwii) - hiv := sin.FloatRow(lwii + 1) + lwv := sin.FloatRow(lwii, 0) + hiv := sin.FloatRow(lwii+1, 0) val = (1-phi)*lwv + phi*hiv } out.SetFloat1D(val, i) diff --git a/tensor/stats/stats/stats_test.go b/tensor/stats/stats/stats_test.go index f6ae0066c1..976faf13d9 100644 --- a/tensor/stats/stats/stats_test.go +++ b/tensor/stats/stats/stats_test.go @@ -158,7 +158,7 @@ func TestFuncsCell(t *testing.T) { for i := range 20 { for j := range 10 { - tsr.SetFloatRowCell(vals[j], i, j) + tsr.SetFloatRow(vals[j], i, j) } } @@ -168,15 +168,15 @@ func TestFuncsCell(t *testing.T) { CountOut(ix, out) nsub := out.Len() for i := range nsub { - assert.Equal(t, 20.0, out.FloatRowCell(0, i)) + assert.Equal(t, 20.0, out.FloatRow(0, i)) } MeanOut(ix, out) for i := range nsub { - assert.InDelta(t, vals[i], out.FloatRowCell(0, i), 1.0e-7) // lower tol, using float32 + assert.InDelta(t, vals[i], out.FloatRow(0, i), 1.0e-7) // lower tol, using float32 } VarOut(ix, out) for i := range nsub { - assert.InDelta(t, 0.0, out.FloatRowCell(0, i), 1.0e-7) + assert.InDelta(t, 0.0, out.FloatRow(0, i), 1.0e-7) } } diff --git a/tensor/string.go b/tensor/string.go index 8ee6e1fd85..93a7a3565e 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -69,17 +69,14 @@ func (tsr String) SetString1D(val string, i int) { tsr.Values[tsr.shape.Header+i] = val } -func (tsr *String) SetStringRowCell(val string, row, cell int) { +func (tsr *String) StringRow(row, cell int) string { _, sz := tsr.shape.RowCellSize() - tsr.Values[tsr.shape.Header+row*sz+cell] = val -} - -func (tsr *String) StringRow(row int) string { - return tsr.StringRowCell(row, 0) + return tsr.Values[tsr.shape.Header+row*sz+cell] } -func (tsr *String) SetStringRow(val string, row int) { - tsr.SetStringRowCell(val, row, 0) +func (tsr *String) SetStringRow(val string, row, cell int) { + _, sz := tsr.shape.RowCellSize() + tsr.Values[tsr.shape.Header+row*sz+cell] = val } ///////////////////// Floats @@ -100,24 +97,16 @@ func (tsr *String) SetFloat1D(val float64, i int) { tsr.Values[tsr.shape.Header+i] = Float64ToString(val) } -func (tsr *String) FloatRowCell(row, cell int) float64 { +func (tsr *String) FloatRow(row, cell int) float64 { _, sz := tsr.shape.RowCellSize() return StringToFloat64(tsr.Values[tsr.shape.Header+row*sz+cell]) } -func (tsr *String) SetFloatRowCell(val float64, row, cell int) { +func (tsr *String) SetFloatRow(val float64, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[tsr.shape.Header+row*sz+cell] = Float64ToString(val) } -func (tsr *String) FloatRow(row int) float64 { - return tsr.FloatRowCell(row, 0) -} - -func (tsr *String) SetFloatRow(val float64, row int) { - tsr.SetFloatRowCell(val, row, 0) -} - ///////////////////// Ints func (tsr *String) Int(i ...int) int { @@ -136,24 +125,16 @@ func (tsr *String) SetInt1D(val int, i int) { tsr.Values[tsr.shape.Header+i] = strconv.Itoa(val) } -func (tsr *String) IntRowCell(row, cell int) int { +func (tsr *String) IntRow(row, cell int) int { _, sz := tsr.shape.RowCellSize() return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.Header+row*sz+cell])) } -func (tsr *String) SetIntRowCell(val int, row, cell int) { +func (tsr *String) SetIntRow(val int, row, cell int) { _, sz := tsr.shape.RowCellSize() tsr.Values[tsr.shape.Header+row*sz+cell] = strconv.Itoa(val) } -func (tsr *String) IntRow(row int) int { - return tsr.IntRowCell(row, 0) -} - -func (tsr *String) SetIntRow(val int, row int) { - tsr.SetIntRowCell(val, row, 0) -} - // SetZeros is a simple convenience function initialize all values to the // zero value of the type (empty strings for string type). func (tsr *String) SetZeros() { @@ -241,7 +222,7 @@ func (tsr *String) SubSpace(offs ...int) Values { return rt } -// RowTensor is a convenience version of [Tensor.SubSpace] to return the +// RowTensor is a convenience version of [RowMajor.SubSpace] to return the // SubSpace for the outermost row dimension. [Rows] defines a version // of this that indirects through the row indexes. func (tsr *String) RowTensor(row int) Values { @@ -255,3 +236,13 @@ func (tsr *String) SetRowTensor(val Values, row int) { mx := min(val.Len(), cells) tsr.CopyCellsFrom(val, st, 0, mx) } + +// AppendRow adds a row and sets values to given values. +func (tsr String) AppendRow(val Values) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow := tsr.DimSize(0) + tsr.SetNumRows(nrow + 1) + tsr.SetRowTensor(val, nrow) +} diff --git a/tensor/table/README.md b/tensor/table/README.md index d732f94693..8fffd4b9cf 100644 --- a/tensor/table/README.md +++ b/tensor/table/README.md @@ -43,15 +43,15 @@ dt.Column("Name").SetStringRow(4) To access higher-dimensional "cell" level data using a simple 1D index into the cell patterns: ```Go -// FloatRowCell is a method on the `tensor.Rows` returned from the `Column` method. +// FloatRow is a method on the `tensor.Rows` returned from the `Column` method. // This is the best method to use in general for generic 1D data access, // as it works on any data from 1D on up (although it only samples the first value // from higher dimensional data) . -val := dt.Column("Values").FloatRowCell(3, 2) +val := dt.Column("Values").FloatRow(3, 2) ``` ```Go -dt.Column("Name").SetStringRowCell("Alia", 4, 1) +dt.Column("Name").SetStringRow("Alia", 4, 1) ``` todo: more diff --git a/tensor/table/indexes.go b/tensor/table/indexes.go index de2021a8aa..c01554c8be 100644 --- a/tensor/table/indexes.go +++ b/tensor/table/indexes.go @@ -152,12 +152,12 @@ func (dt *Table) SortColumnIndexes(ascending, stable bool, colIndexes ...int) { for _, ci := range colIndexes { cl := dt.ColumnByIndex(ci).Tensor if cl.IsString() { - v := tensor.CompareAscending(cl.StringRowCell(i, 0), cl.StringRowCell(j, 0), ascending) + v := tensor.CompareAscending(cl.StringRow(i, 0), cl.StringRow(j, 0), ascending) if v != 0 { return v } } else { - v := tensor.CompareAscending(cl.FloatRowCell(i, 0), cl.FloatRowCell(j, 0), ascending) + v := tensor.CompareAscending(cl.FloatRow(i, 0), cl.FloatRow(j, 0), ascending) if v != 0 { return v } diff --git a/tensor/table/slicetable.go b/tensor/table/slicetable.go index cdc2468e8f..61432a39cf 100644 --- a/tensor/table/slicetable.go +++ b/tensor/table/slicetable.go @@ -54,10 +54,10 @@ func UpdateSliceTable(st any, dt *Table) { val := npv.Index(ri).Field(i).Interface() cl := dt.Column(f.Name) if kind == reflect.String { - cl.SetStringRow(val.(string), ri) + cl.SetStringRow(val.(string), ri, 0) } else { fv, _ := reflectx.ToFloat(val) - cl.SetFloatRow(fv, ri) + cl.SetFloatRow(fv, ri, 0) } } } diff --git a/tensor/table/table_test.go b/tensor/table/table_test.go index 744f773ffe..a848e3e31f 100644 --- a/tensor/table/table_test.go +++ b/tensor/table/table_test.go @@ -45,9 +45,9 @@ func NewTestTable() *Table { dt.AddIntColumn("Int") dt.SetNumRows(3) for i := range dt.NumRows() { - dt.Column("Str").SetStringRow(strconv.Itoa(i), i) - dt.Column("Flt64").SetFloatRow(float64(i), i) - dt.Column("Int").SetFloatRow(float64(i), i) + dt.Column("Str").SetStringRow(strconv.Itoa(i), i, 0) + dt.Column("Flt64").SetFloatRow(float64(i), i, 0) + dt.Column("Int").SetFloatRow(float64(i), i, 0) } return dt } @@ -61,16 +61,16 @@ func TestAppendRowsEtc(t *testing.T) { for j := range 3 { for i := range st.NumRows() { sr := j*3 + i - ss := st.Column("Str").StringRow(i) - ds := dt.Column("Str").StringRow(sr) + ss := st.Column("Str").StringRow(i, 0) + ds := dt.Column("Str").StringRow(sr, 0) assert.Equal(t, ss, ds) - sf := st.Column("Flt64").FloatRow(i) - df := dt.Column("Flt64").FloatRow(sr) + sf := st.Column("Flt64").FloatRow(i, 0) + df := dt.Column("Flt64").FloatRow(sr, 0) assert.Equal(t, sf, df) - sf = st.Column("Int").FloatRow(i) - df = dt.Column("Int").FloatRow(sr) + sf = st.Column("Int").FloatRow(i, 0) + df = dt.Column("Int").FloatRow(sr, 0) assert.Equal(t, sf, df) } } @@ -88,7 +88,7 @@ func TestAppendRowsEtc(t *testing.T) { dt.Sequential() dt.Filter(func(dt *Table, row int) bool { - return dt.Column("Flt64").FloatRow(row) > 1 + return dt.Column("Flt64").FloatRow(row, 0) > 1 }) assert.Equal(t, []int{2, 5, 8, 11}, dt.Indexes) } @@ -121,7 +121,7 @@ func TestCells(t *testing.T) { for i := range 10 { vals := make([]float32, 16) for j := range 16 { - vals[j] = float32(in.FloatRowCell(i, j)) + vals[j] = float32(in.FloatRow(i, j)) } // fmt.Println(s) ss := in.Tensor.SubSpace(i).(*tensor.Float32) diff --git a/tensor/tensor.go b/tensor/tensor.go index 425cdf5558..a83e62ddb8 100644 --- a/tensor/tensor.go +++ b/tensor/tensor.go @@ -104,12 +104,12 @@ type Tensor interface { // Float1D returns the value of given 1-dimensional index (0-Len()-1) as a float64. // This can be somewhat expensive in wrapper views ([Rows], [Sliced]), which // convert the flat index back into a full n-dimensional index and use that api. - // [Tensor.FloatRowCell] is preferred. + // [Tensor.FloatRow] is preferred. Float1D(i int) float64 // SetFloat1D sets the value of given 1-dimensional index (0-Len()-1) as a float64. // This can be somewhat expensive in the commonly-used [Rows] view; - // [Tensor.SetFloatRowCell] is preferred. + // [Tensor.SetFloatRow] is preferred. SetFloat1D(val float64, i int) ///////////////////// Strings diff --git a/tensor/tensor_test.go b/tensor/tensor_test.go index 8bee9c7598..2935397245 100644 --- a/tensor/tensor_test.go +++ b/tensor/tensor_test.go @@ -30,9 +30,9 @@ func TestTensorString(t *testing.T) { assert.Equal(t, "testing", tsr.StringValue(2, 1)) assert.Equal(t, "test", tsr.String1D(4)) - assert.Equal(t, "test", tsr.StringRowCell(2, 0)) - assert.Equal(t, "testing", tsr.StringRowCell(2, 1)) - assert.Equal(t, "", tsr.StringRowCell(3, 0)) + assert.Equal(t, "test", tsr.StringRow(2, 0)) + assert.Equal(t, "testing", tsr.StringRow(2, 1)) + assert.Equal(t, "", tsr.StringRow(3, 0)) cln := tsr.Clone() assert.Equal(t, "testing", cln.StringValue(2, 1)) @@ -89,9 +89,9 @@ func TestTensorFloat64(t *testing.T) { assert.Equal(t, 2.17, tsr.Float(2, 1)) assert.Equal(t, 3.14, tsr.Float1D(4)) - assert.Equal(t, 3.14, tsr.FloatRowCell(2, 0)) - assert.Equal(t, 2.17, tsr.FloatRowCell(2, 1)) - assert.Equal(t, 0.0, tsr.FloatRowCell(3, 0)) + assert.Equal(t, 3.14, tsr.FloatRow(2, 0)) + assert.Equal(t, 2.17, tsr.FloatRow(2, 1)) + assert.Equal(t, 0.0, tsr.FloatRow(3, 0)) cln := tsr.Clone() assert.Equal(t, 2.17, cln.Float(2, 1)) @@ -293,7 +293,7 @@ func TestReshaped(t *testing.T) { func TestSortFilter(t *testing.T) { tsr := NewRows(NewFloat64(5)) for i := range 5 { - tsr.SetFloatRowCell(float64(i), i, 0) + tsr.SetFloatRow(float64(i), i, 0) } tsr.Sort(Ascending) assert.Equal(t, []int{0, 1, 2, 3, 4}, tsr.Indexes) diff --git a/tensor/tensorcore/table.go b/tensor/tensorcore/table.go index b1cc5f4596..69408927bf 100644 --- a/tensor/tensorcore/table.go +++ b/tensor/tensorcore/table.go @@ -602,7 +602,7 @@ func (tb *Table) SelectedColumnStrings(colName string) []string { var s []string col := dt.Column(colName) for _, i := range jis { - v := col.StringRow(i) + v := col.StringRow(i, 0) s = append(s, v) } return s diff --git a/tensor/tmath/ops_test.go b/tensor/tmath/ops_test.go index 9f15c4f40d..83bdd18101 100644 --- a/tensor/tmath/ops_test.go +++ b/tensor/tmath/ops_test.go @@ -49,7 +49,7 @@ func TestOps(t *testing.T) { AddOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v+v, cellout.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v+v, cellout.FloatRow(ri, i), 1.0e-6) } } @@ -77,7 +77,7 @@ func TestOps(t *testing.T) { SubOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v-v, cellout.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v-v, cellout.FloatRow(ri, i), 1.0e-6) } } @@ -97,7 +97,7 @@ func TestOps(t *testing.T) { MulOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v*v, cellout.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v*v, cellout.FloatRow(ri, i), 1.0e-6) } } @@ -125,7 +125,7 @@ func TestOps(t *testing.T) { DivOut(cell2d, oned, cellout) for ri := range 5 { for i, v := range vals { - assert.InDelta(t, v/v, cellout.FloatRowCell(ri, i), 1.0e-6) + assert.InDelta(t, v/v, cellout.FloatRow(ri, i), 1.0e-6) } } diff --git a/yaegicore/nogui/cogentcore_org-core-base-reflectx.go b/yaegicore/nogui/cogentcore_org-core-base-reflectx.go index c7ea91a582..e0ec9a756d 100644 --- a/yaegicore/nogui/cogentcore_org-core-base-reflectx.go +++ b/yaegicore/nogui/cogentcore_org-core-base-reflectx.go @@ -13,8 +13,10 @@ func init() { // function, constant and variable definitions "AnyIsNil": reflect.ValueOf(reflectx.AnyIsNil), "CloneToType": reflect.ValueOf(reflectx.CloneToType), + "CopyFields": reflect.ValueOf(reflectx.CopyFields), "CopyMapRobust": reflect.ValueOf(reflectx.CopyMapRobust), "CopySliceRobust": reflect.ValueOf(reflectx.CopySliceRobust), + "FieldValue": reflect.ValueOf(reflectx.FieldValue), "FormatDefault": reflect.ValueOf(reflectx.FormatDefault), "KindIsBasic": reflect.ValueOf(reflectx.KindIsBasic), "KindIsFloat": reflect.ValueOf(reflectx.KindIsFloat), diff --git a/yaegicore/nogui/cogentcore_org-core-tensor.go b/yaegicore/nogui/cogentcore_org-core-tensor.go index 42fb865722..336cfbef08 100644 --- a/yaegicore/nogui/cogentcore_org-core-tensor.go +++ b/yaegicore/nogui/cogentcore_org-core-tensor.go @@ -191,61 +191,55 @@ func init() { // _cogentcore_org_core_tensor_RowMajor is an interface wrapper for RowMajor type type _cogentcore_org_core_tensor_RowMajor struct { - IValue interface{} - WAsValues func() tensor.Values - WDataType func() reflect.Kind - WDimSize func(dim int) int - WFloat func(i ...int) float64 - WFloat1D func(i int) float64 - WFloatRow func(row int) float64 - WFloatRowCell func(row int, cell int) float64 - WInt func(i ...int) int - WInt1D func(i int) int - WIntRow func(row int) int - WIntRowCell func(row int, cell int) int - WIsString func() bool - WLabel func() string - WLen func() int - WMetadata func() *metadata.Data - WNumDims func() int - WRowTensor func(row int) tensor.Values - WSetFloat func(val float64, i ...int) - WSetFloat1D func(val float64, i int) - WSetFloatRow func(val float64, row int) - WSetFloatRowCell func(val float64, row int, cell int) - WSetInt func(val int, i ...int) - WSetInt1D func(val int, i int) - WSetIntRow func(val int, row int) - WSetIntRowCell func(val int, row int, cell int) - WSetRowTensor func(val tensor.Values, row int) - WSetString func(val string, i ...int) - WSetString1D func(val string, i int) - WSetStringRow func(val string, row int) - WSetStringRowCell func(val string, row int, cell int) - WShape func() *tensor.Shape - WShapeSizes func() []int - WString func() string - WString1D func(i int) string - WStringRow func(row int) string - WStringRowCell func(row int, cell int) string - WStringValue func(i ...int) string - WSubSpace func(offs ...int) tensor.Values + IValue interface{} + WAppendRow func(val tensor.Values) + WAsValues func() tensor.Values + WDataType func() reflect.Kind + WDimSize func(dim int) int + WFloat func(i ...int) float64 + WFloat1D func(i int) float64 + WFloatRow func(row int, cell int) float64 + WInt func(i ...int) int + WInt1D func(i int) int + WIntRow func(row int, cell int) int + WIsString func() bool + WLabel func() string + WLen func() int + WMetadata func() *metadata.Data + WNumDims func() int + WRowTensor func(row int) tensor.Values + WSetFloat func(val float64, i ...int) + WSetFloat1D func(val float64, i int) + WSetFloatRow func(val float64, row int, cell int) + WSetInt func(val int, i ...int) + WSetInt1D func(val int, i int) + WSetIntRow func(val int, row int, cell int) + WSetRowTensor func(val tensor.Values, row int) + WSetString func(val string, i ...int) + WSetString1D func(val string, i int) + WSetStringRow func(val string, row int, cell int) + WShape func() *tensor.Shape + WShapeSizes func() []int + WString func() string + WString1D func(i int) string + WStringRow func(row int, cell int) string + WStringValue func(i ...int) string + WSubSpace func(offs ...int) tensor.Values } -func (W _cogentcore_org_core_tensor_RowMajor) AsValues() tensor.Values { return W.WAsValues() } -func (W _cogentcore_org_core_tensor_RowMajor) DataType() reflect.Kind { return W.WDataType() } -func (W _cogentcore_org_core_tensor_RowMajor) DimSize(dim int) int { return W.WDimSize(dim) } -func (W _cogentcore_org_core_tensor_RowMajor) Float(i ...int) float64 { return W.WFloat(i...) } -func (W _cogentcore_org_core_tensor_RowMajor) Float1D(i int) float64 { return W.WFloat1D(i) } -func (W _cogentcore_org_core_tensor_RowMajor) FloatRow(row int) float64 { return W.WFloatRow(row) } -func (W _cogentcore_org_core_tensor_RowMajor) FloatRowCell(row int, cell int) float64 { - return W.WFloatRowCell(row, cell) -} -func (W _cogentcore_org_core_tensor_RowMajor) Int(i ...int) int { return W.WInt(i...) } -func (W _cogentcore_org_core_tensor_RowMajor) Int1D(i int) int { return W.WInt1D(i) } -func (W _cogentcore_org_core_tensor_RowMajor) IntRow(row int) int { return W.WIntRow(row) } -func (W _cogentcore_org_core_tensor_RowMajor) IntRowCell(row int, cell int) int { - return W.WIntRowCell(row, cell) +func (W _cogentcore_org_core_tensor_RowMajor) AppendRow(val tensor.Values) { W.WAppendRow(val) } +func (W _cogentcore_org_core_tensor_RowMajor) AsValues() tensor.Values { return W.WAsValues() } +func (W _cogentcore_org_core_tensor_RowMajor) DataType() reflect.Kind { return W.WDataType() } +func (W _cogentcore_org_core_tensor_RowMajor) DimSize(dim int) int { return W.WDimSize(dim) } +func (W _cogentcore_org_core_tensor_RowMajor) Float(i ...int) float64 { return W.WFloat(i...) } +func (W _cogentcore_org_core_tensor_RowMajor) Float1D(i int) float64 { return W.WFloat1D(i) } +func (W _cogentcore_org_core_tensor_RowMajor) FloatRow(row int, cell int) float64 { + return W.WFloatRow(row, cell) +} +func (W _cogentcore_org_core_tensor_RowMajor) Int(i ...int) int { return W.WInt(i...) } +func (W _cogentcore_org_core_tensor_RowMajor) Int1D(i int) int { return W.WInt1D(i) } +func (W _cogentcore_org_core_tensor_RowMajor) IntRow(row int, cell int) int { + return W.WIntRow(row, cell) } func (W _cogentcore_org_core_tensor_RowMajor) IsString() bool { return W.WIsString() } func (W _cogentcore_org_core_tensor_RowMajor) Label() string { return W.WLabel() } @@ -257,17 +251,13 @@ func (W _cogentcore_org_core_tensor_RowMajor) RowTensor(row int) tensor.Values { } func (W _cogentcore_org_core_tensor_RowMajor) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } func (W _cogentcore_org_core_tensor_RowMajor) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } -func (W _cogentcore_org_core_tensor_RowMajor) SetFloatRow(val float64, row int) { - W.WSetFloatRow(val, row) -} -func (W _cogentcore_org_core_tensor_RowMajor) SetFloatRowCell(val float64, row int, cell int) { - W.WSetFloatRowCell(val, row, cell) +func (W _cogentcore_org_core_tensor_RowMajor) SetFloatRow(val float64, row int, cell int) { + W.WSetFloatRow(val, row, cell) } -func (W _cogentcore_org_core_tensor_RowMajor) SetInt(val int, i ...int) { W.WSetInt(val, i...) } -func (W _cogentcore_org_core_tensor_RowMajor) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } -func (W _cogentcore_org_core_tensor_RowMajor) SetIntRow(val int, row int) { W.WSetIntRow(val, row) } -func (W _cogentcore_org_core_tensor_RowMajor) SetIntRowCell(val int, row int, cell int) { - W.WSetIntRowCell(val, row, cell) +func (W _cogentcore_org_core_tensor_RowMajor) SetInt(val int, i ...int) { W.WSetInt(val, i...) } +func (W _cogentcore_org_core_tensor_RowMajor) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } +func (W _cogentcore_org_core_tensor_RowMajor) SetIntRow(val int, row int, cell int) { + W.WSetIntRow(val, row, cell) } func (W _cogentcore_org_core_tensor_RowMajor) SetRowTensor(val tensor.Values, row int) { W.WSetRowTensor(val, row) @@ -276,11 +266,8 @@ func (W _cogentcore_org_core_tensor_RowMajor) SetString(val string, i ...int) { W.WSetString(val, i...) } func (W _cogentcore_org_core_tensor_RowMajor) SetString1D(val string, i int) { W.WSetString1D(val, i) } -func (W _cogentcore_org_core_tensor_RowMajor) SetStringRow(val string, row int) { - W.WSetStringRow(val, row) -} -func (W _cogentcore_org_core_tensor_RowMajor) SetStringRowCell(val string, row int, cell int) { - W.WSetStringRowCell(val, row, cell) +func (W _cogentcore_org_core_tensor_RowMajor) SetStringRow(val string, row int, cell int) { + W.WSetStringRow(val, row, cell) } func (W _cogentcore_org_core_tensor_RowMajor) Shape() *tensor.Shape { return W.WShape() } func (W _cogentcore_org_core_tensor_RowMajor) ShapeSizes() []int { return W.WShapeSizes() } @@ -290,10 +277,9 @@ func (W _cogentcore_org_core_tensor_RowMajor) String() string { } return W.WString() } -func (W _cogentcore_org_core_tensor_RowMajor) String1D(i int) string { return W.WString1D(i) } -func (W _cogentcore_org_core_tensor_RowMajor) StringRow(row int) string { return W.WStringRow(row) } -func (W _cogentcore_org_core_tensor_RowMajor) StringRowCell(row int, cell int) string { - return W.WStringRowCell(row, cell) +func (W _cogentcore_org_core_tensor_RowMajor) String1D(i int) string { return W.WString1D(i) } +func (W _cogentcore_org_core_tensor_RowMajor) StringRow(row int, cell int) string { + return W.WStringRow(row, cell) } func (W _cogentcore_org_core_tensor_RowMajor) StringValue(i ...int) string { return W.WStringValue(i...) @@ -361,62 +347,58 @@ func (W _cogentcore_org_core_tensor_Tensor) StringValue(i ...int) string { retur // _cogentcore_org_core_tensor_Values is an interface wrapper for Values type type _cogentcore_org_core_tensor_Values struct { - IValue interface{} - WAppendFrom func(from tensor.Values) error - WAsValues func() tensor.Values - WBytes func() []byte - WClone func() tensor.Values - WCopyCellsFrom func(from tensor.Values, to int, start int, n int) - WCopyFrom func(from tensor.Values) - WDataType func() reflect.Kind - WDimSize func(dim int) int - WFloat func(i ...int) float64 - WFloat1D func(i int) float64 - WFloatRow func(row int) float64 - WFloatRowCell func(row int, cell int) float64 - WInt func(i ...int) int - WInt1D func(i int) int - WIntRow func(row int) int - WIntRowCell func(row int, cell int) int - WIsString func() bool - WLabel func() string - WLen func() int - WMetadata func() *metadata.Data - WNumDims func() int - WRowTensor func(row int) tensor.Values - WSetFloat func(val float64, i ...int) - WSetFloat1D func(val float64, i int) - WSetFloatRow func(val float64, row int) - WSetFloatRowCell func(val float64, row int, cell int) - WSetInt func(val int, i ...int) - WSetInt1D func(val int, i int) - WSetIntRow func(val int, row int) - WSetIntRowCell func(val int, row int, cell int) - WSetNumRows func(rows int) - WSetRowTensor func(val tensor.Values, row int) - WSetShapeSizes func(sizes ...int) - WSetString func(val string, i ...int) - WSetString1D func(val string, i int) - WSetStringRow func(val string, row int) - WSetStringRowCell func(val string, row int, cell int) - WSetZeros func() - WShape func() *tensor.Shape - WShapeSizes func() []int - WSizeof func() int64 - WString func() string - WString1D func(i int) string - WStringRow func(row int) string - WStringRowCell func(row int, cell int) string - WStringValue func(i ...int) string - WSubSpace func(offs ...int) tensor.Values + IValue interface{} + WAppendFrom func(from tensor.Values) error + WAppendRow func(val tensor.Values) + WAsValues func() tensor.Values + WBytes func() []byte + WClone func() tensor.Values + WCopyCellsFrom func(from tensor.Values, to int, start int, n int) + WCopyFrom func(from tensor.Values) + WDataType func() reflect.Kind + WDimSize func(dim int) int + WFloat func(i ...int) float64 + WFloat1D func(i int) float64 + WFloatRow func(row int, cell int) float64 + WInt func(i ...int) int + WInt1D func(i int) int + WIntRow func(row int, cell int) int + WIsString func() bool + WLabel func() string + WLen func() int + WMetadata func() *metadata.Data + WNumDims func() int + WRowTensor func(row int) tensor.Values + WSetFloat func(val float64, i ...int) + WSetFloat1D func(val float64, i int) + WSetFloatRow func(val float64, row int, cell int) + WSetInt func(val int, i ...int) + WSetInt1D func(val int, i int) + WSetIntRow func(val int, row int, cell int) + WSetNumRows func(rows int) + WSetRowTensor func(val tensor.Values, row int) + WSetShapeSizes func(sizes ...int) + WSetString func(val string, i ...int) + WSetString1D func(val string, i int) + WSetStringRow func(val string, row int, cell int) + WSetZeros func() + WShape func() *tensor.Shape + WShapeSizes func() []int + WSizeof func() int64 + WString func() string + WString1D func(i int) string + WStringRow func(row int, cell int) string + WStringValue func(i ...int) string + WSubSpace func(offs ...int) tensor.Values } func (W _cogentcore_org_core_tensor_Values) AppendFrom(from tensor.Values) error { return W.WAppendFrom(from) } -func (W _cogentcore_org_core_tensor_Values) AsValues() tensor.Values { return W.WAsValues() } -func (W _cogentcore_org_core_tensor_Values) Bytes() []byte { return W.WBytes() } -func (W _cogentcore_org_core_tensor_Values) Clone() tensor.Values { return W.WClone() } +func (W _cogentcore_org_core_tensor_Values) AppendRow(val tensor.Values) { W.WAppendRow(val) } +func (W _cogentcore_org_core_tensor_Values) AsValues() tensor.Values { return W.WAsValues() } +func (W _cogentcore_org_core_tensor_Values) Bytes() []byte { return W.WBytes() } +func (W _cogentcore_org_core_tensor_Values) Clone() tensor.Values { return W.WClone() } func (W _cogentcore_org_core_tensor_Values) CopyCellsFrom(from tensor.Values, to int, start int, n int) { W.WCopyCellsFrom(from, to, start, n) } @@ -425,15 +407,13 @@ func (W _cogentcore_org_core_tensor_Values) DataType() reflect.Kind { retur func (W _cogentcore_org_core_tensor_Values) DimSize(dim int) int { return W.WDimSize(dim) } func (W _cogentcore_org_core_tensor_Values) Float(i ...int) float64 { return W.WFloat(i...) } func (W _cogentcore_org_core_tensor_Values) Float1D(i int) float64 { return W.WFloat1D(i) } -func (W _cogentcore_org_core_tensor_Values) FloatRow(row int) float64 { return W.WFloatRow(row) } -func (W _cogentcore_org_core_tensor_Values) FloatRowCell(row int, cell int) float64 { - return W.WFloatRowCell(row, cell) +func (W _cogentcore_org_core_tensor_Values) FloatRow(row int, cell int) float64 { + return W.WFloatRow(row, cell) } -func (W _cogentcore_org_core_tensor_Values) Int(i ...int) int { return W.WInt(i...) } -func (W _cogentcore_org_core_tensor_Values) Int1D(i int) int { return W.WInt1D(i) } -func (W _cogentcore_org_core_tensor_Values) IntRow(row int) int { return W.WIntRow(row) } -func (W _cogentcore_org_core_tensor_Values) IntRowCell(row int, cell int) int { - return W.WIntRowCell(row, cell) +func (W _cogentcore_org_core_tensor_Values) Int(i ...int) int { return W.WInt(i...) } +func (W _cogentcore_org_core_tensor_Values) Int1D(i int) int { return W.WInt1D(i) } +func (W _cogentcore_org_core_tensor_Values) IntRow(row int, cell int) int { + return W.WIntRow(row, cell) } func (W _cogentcore_org_core_tensor_Values) IsString() bool { return W.WIsString() } func (W _cogentcore_org_core_tensor_Values) Label() string { return W.WLabel() } @@ -445,17 +425,13 @@ func (W _cogentcore_org_core_tensor_Values) RowTensor(row int) tensor.Values { } func (W _cogentcore_org_core_tensor_Values) SetFloat(val float64, i ...int) { W.WSetFloat(val, i...) } func (W _cogentcore_org_core_tensor_Values) SetFloat1D(val float64, i int) { W.WSetFloat1D(val, i) } -func (W _cogentcore_org_core_tensor_Values) SetFloatRow(val float64, row int) { - W.WSetFloatRow(val, row) +func (W _cogentcore_org_core_tensor_Values) SetFloatRow(val float64, row int, cell int) { + W.WSetFloatRow(val, row, cell) } -func (W _cogentcore_org_core_tensor_Values) SetFloatRowCell(val float64, row int, cell int) { - W.WSetFloatRowCell(val, row, cell) -} -func (W _cogentcore_org_core_tensor_Values) SetInt(val int, i ...int) { W.WSetInt(val, i...) } -func (W _cogentcore_org_core_tensor_Values) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } -func (W _cogentcore_org_core_tensor_Values) SetIntRow(val int, row int) { W.WSetIntRow(val, row) } -func (W _cogentcore_org_core_tensor_Values) SetIntRowCell(val int, row int, cell int) { - W.WSetIntRowCell(val, row, cell) +func (W _cogentcore_org_core_tensor_Values) SetInt(val int, i ...int) { W.WSetInt(val, i...) } +func (W _cogentcore_org_core_tensor_Values) SetInt1D(val int, i int) { W.WSetInt1D(val, i) } +func (W _cogentcore_org_core_tensor_Values) SetIntRow(val int, row int, cell int) { + W.WSetIntRow(val, row, cell) } func (W _cogentcore_org_core_tensor_Values) SetNumRows(rows int) { W.WSetNumRows(rows) } func (W _cogentcore_org_core_tensor_Values) SetRowTensor(val tensor.Values, row int) { @@ -464,11 +440,8 @@ func (W _cogentcore_org_core_tensor_Values) SetRowTensor(val tensor.Values, row func (W _cogentcore_org_core_tensor_Values) SetShapeSizes(sizes ...int) { W.WSetShapeSizes(sizes...) } func (W _cogentcore_org_core_tensor_Values) SetString(val string, i ...int) { W.WSetString(val, i...) } func (W _cogentcore_org_core_tensor_Values) SetString1D(val string, i int) { W.WSetString1D(val, i) } -func (W _cogentcore_org_core_tensor_Values) SetStringRow(val string, row int) { - W.WSetStringRow(val, row) -} -func (W _cogentcore_org_core_tensor_Values) SetStringRowCell(val string, row int, cell int) { - W.WSetStringRowCell(val, row, cell) +func (W _cogentcore_org_core_tensor_Values) SetStringRow(val string, row int, cell int) { + W.WSetStringRow(val, row, cell) } func (W _cogentcore_org_core_tensor_Values) SetZeros() { W.WSetZeros() } func (W _cogentcore_org_core_tensor_Values) Shape() *tensor.Shape { return W.WShape() } @@ -480,10 +453,9 @@ func (W _cogentcore_org_core_tensor_Values) String() string { } return W.WString() } -func (W _cogentcore_org_core_tensor_Values) String1D(i int) string { return W.WString1D(i) } -func (W _cogentcore_org_core_tensor_Values) StringRow(row int) string { return W.WStringRow(row) } -func (W _cogentcore_org_core_tensor_Values) StringRowCell(row int, cell int) string { - return W.WStringRowCell(row, cell) +func (W _cogentcore_org_core_tensor_Values) String1D(i int) string { return W.WString1D(i) } +func (W _cogentcore_org_core_tensor_Values) StringRow(row int, cell int) string { + return W.WStringRow(row, cell) } func (W _cogentcore_org_core_tensor_Values) StringValue(i ...int) string { return W.WStringValue(i...) } func (W _cogentcore_org_core_tensor_Values) SubSpace(offs ...int) tensor.Values { diff --git a/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go b/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go index 61f3260cac..462d579dd8 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go +++ b/yaegicore/symbols/cogentcore_org-core-plot-plotcore.go @@ -10,28 +10,14 @@ import ( func init() { Symbols["cogentcore.org/core/plot/plotcore/plotcore"] = map[string]reflect.Value{ // function, constant and variable definitions - "Bar": reflect.ValueOf(plotcore.Bar), - "FixMax": reflect.ValueOf(plotcore.FixMax), - "FixMin": reflect.ValueOf(plotcore.FixMin), - "FloatMax": reflect.ValueOf(plotcore.FloatMax), - "FloatMin": reflect.ValueOf(plotcore.FloatMin), - "NewPlot": reflect.ValueOf(plotcore.NewPlot), - "NewPlotEditor": reflect.ValueOf(plotcore.NewPlotEditor), - "NewSubPlot": reflect.ValueOf(plotcore.NewSubPlot), - "Off": reflect.ValueOf(plotcore.Off), - "On": reflect.ValueOf(plotcore.On), - "PlotColumnOptions": reflect.ValueOf(plotcore.PlotColumnOptions), - "PlotColumnZeroOne": reflect.ValueOf(plotcore.PlotColumnZeroOne), - "PlotTypesN": reflect.ValueOf(plotcore.PlotTypesN), - "PlotTypesValues": reflect.ValueOf(plotcore.PlotTypesValues), - "SetPlotColumnOptions": reflect.ValueOf(plotcore.SetPlotColumnOptions), - "XY": reflect.ValueOf(plotcore.XY), + "NewPlot": reflect.ValueOf(plotcore.NewPlot), + "NewPlotEditor": reflect.ValueOf(plotcore.NewPlotEditor), + "NewPlotterChooser": reflect.ValueOf(plotcore.NewPlotterChooser), + "NewSubPlot": reflect.ValueOf(plotcore.NewSubPlot), // type definitions - "ColumnOptions": reflect.ValueOf((*plotcore.ColumnOptions)(nil)), - "Plot": reflect.ValueOf((*plotcore.Plot)(nil)), - "PlotEditor": reflect.ValueOf((*plotcore.PlotEditor)(nil)), - "PlotOptions": reflect.ValueOf((*plotcore.PlotOptions)(nil)), - "PlotTypes": reflect.ValueOf((*plotcore.PlotTypes)(nil)), + "Plot": reflect.ValueOf((*plotcore.Plot)(nil)), + "PlotEditor": reflect.ValueOf((*plotcore.PlotEditor)(nil)), + "PlotterChooser": reflect.ValueOf((*plotcore.PlotterChooser)(nil)), } } diff --git a/yaegicore/symbols/cogentcore_org-core-plot-plots.go b/yaegicore/symbols/cogentcore_org-core-plot-plots.go index 570f75d20f..32f2179e08 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot-plots.go +++ b/yaegicore/symbols/cogentcore_org-core-plot-plots.go @@ -4,122 +4,32 @@ package symbols import ( "cogentcore.org/core/plot/plots" + "go/constant" + "go/token" "reflect" ) func init() { Symbols["cogentcore.org/core/plot/plots/plots"] = map[string]reflect.Value{ // function, constant and variable definitions - "AddTableLine": reflect.ValueOf(plots.AddTableLine), - "AddTableLinePoints": reflect.ValueOf(plots.AddTableLinePoints), - "Box": reflect.ValueOf(plots.Box), - "Circle": reflect.ValueOf(plots.Circle), - "Cross": reflect.ValueOf(plots.Cross), - "DrawBox": reflect.ValueOf(plots.DrawBox), - "DrawCircle": reflect.ValueOf(plots.DrawCircle), - "DrawCross": reflect.ValueOf(plots.DrawCross), - "DrawPlus": reflect.ValueOf(plots.DrawPlus), - "DrawPyramid": reflect.ValueOf(plots.DrawPyramid), - "DrawRing": reflect.ValueOf(plots.DrawRing), - "DrawShape": reflect.ValueOf(plots.DrawShape), - "DrawSquare": reflect.ValueOf(plots.DrawSquare), - "DrawTriangle": reflect.ValueOf(plots.DrawTriangle), - "MidStep": reflect.ValueOf(plots.MidStep), - "NewBarChart": reflect.ValueOf(plots.NewBarChart), - "NewBarChartTensor": reflect.ValueOf(plots.NewBarChartTensor), - "NewLabels": reflect.ValueOf(plots.NewLabels), - "NewLine": reflect.ValueOf(plots.NewLine), - "NewLinePoints": reflect.ValueOf(plots.NewLinePoints), - "NewLineTensor": reflect.ValueOf(plots.NewLineTensor), - "NewScatter": reflect.ValueOf(plots.NewScatter), - "NewTableXYer": reflect.ValueOf(plots.NewTableXYer), - "NewXErrorBars": reflect.ValueOf(plots.NewXErrorBars), - "NewYErrorBars": reflect.ValueOf(plots.NewYErrorBars), - "NoStep": reflect.ValueOf(plots.NoStep), - "Plus": reflect.ValueOf(plots.Plus), - "PostStep": reflect.ValueOf(plots.PostStep), - "PreStep": reflect.ValueOf(plots.PreStep), - "Pyramid": reflect.ValueOf(plots.Pyramid), - "Ring": reflect.ValueOf(plots.Ring), - "ShapesN": reflect.ValueOf(plots.ShapesN), - "ShapesValues": reflect.ValueOf(plots.ShapesValues), - "Square": reflect.ValueOf(plots.Square), - "StepKindN": reflect.ValueOf(plots.StepKindN), - "StepKindValues": reflect.ValueOf(plots.StepKindValues), - "TableColumnByIndex": reflect.ValueOf(plots.TableColumnByIndex), - "Triangle": reflect.ValueOf(plots.Triangle), + "BarType": reflect.ValueOf(constant.MakeFromLiteral("\"Bar\"", token.STRING, 0)), + "LabelsType": reflect.ValueOf(constant.MakeFromLiteral("\"Labels\"", token.STRING, 0)), + "NewBar": reflect.ValueOf(plots.NewBar), + "NewLabels": reflect.ValueOf(plots.NewLabels), + "NewLine": reflect.ValueOf(plots.NewLine), + "NewScatter": reflect.ValueOf(plots.NewScatter), + "NewXErrorBars": reflect.ValueOf(plots.NewXErrorBars), + "NewXY": reflect.ValueOf(plots.NewXY), + "NewYErrorBars": reflect.ValueOf(plots.NewYErrorBars), + "XErrorBarsType": reflect.ValueOf(constant.MakeFromLiteral("\"XErrorBars\"", token.STRING, 0)), + "XYType": reflect.ValueOf(constant.MakeFromLiteral("\"XY\"", token.STRING, 0)), + "YErrorBarsType": reflect.ValueOf(constant.MakeFromLiteral("\"YErrorBars\"", token.STRING, 0)), // type definitions - "BarChart": reflect.ValueOf((*plots.BarChart)(nil)), - "Errors": reflect.ValueOf((*plots.Errors)(nil)), + "Bar": reflect.ValueOf((*plots.Bar)(nil)), "Labels": reflect.ValueOf((*plots.Labels)(nil)), - "Line": reflect.ValueOf((*plots.Line)(nil)), - "Scatter": reflect.ValueOf((*plots.Scatter)(nil)), - "Shapes": reflect.ValueOf((*plots.Shapes)(nil)), - "StepKind": reflect.ValueOf((*plots.StepKind)(nil)), - "Table": reflect.ValueOf((*plots.Table)(nil)), - "TableXYer": reflect.ValueOf((*plots.TableXYer)(nil)), "XErrorBars": reflect.ValueOf((*plots.XErrorBars)(nil)), - "XErrorer": reflect.ValueOf((*plots.XErrorer)(nil)), - "XErrors": reflect.ValueOf((*plots.XErrors)(nil)), - "XYLabeler": reflect.ValueOf((*plots.XYLabeler)(nil)), - "XYLabels": reflect.ValueOf((*plots.XYLabels)(nil)), + "XY": reflect.ValueOf((*plots.XY)(nil)), "YErrorBars": reflect.ValueOf((*plots.YErrorBars)(nil)), - "YErrorer": reflect.ValueOf((*plots.YErrorer)(nil)), - "YErrors": reflect.ValueOf((*plots.YErrors)(nil)), - - // interface wrapper definitions - "_Table": reflect.ValueOf((*_cogentcore_org_core_plot_plots_Table)(nil)), - "_XErrorer": reflect.ValueOf((*_cogentcore_org_core_plot_plots_XErrorer)(nil)), - "_XYLabeler": reflect.ValueOf((*_cogentcore_org_core_plot_plots_XYLabeler)(nil)), - "_YErrorer": reflect.ValueOf((*_cogentcore_org_core_plot_plots_YErrorer)(nil)), } } - -// _cogentcore_org_core_plot_plots_Table is an interface wrapper for Table type -type _cogentcore_org_core_plot_plots_Table struct { - IValue interface{} - WColumnName func(i int) string - WNumColumns func() int - WNumRows func() int - WPlotData func(column int, row int) float32 -} - -func (W _cogentcore_org_core_plot_plots_Table) ColumnName(i int) string { return W.WColumnName(i) } -func (W _cogentcore_org_core_plot_plots_Table) NumColumns() int { return W.WNumColumns() } -func (W _cogentcore_org_core_plot_plots_Table) NumRows() int { return W.WNumRows() } -func (W _cogentcore_org_core_plot_plots_Table) PlotData(column int, row int) float32 { - return W.WPlotData(column, row) -} - -// _cogentcore_org_core_plot_plots_XErrorer is an interface wrapper for XErrorer type -type _cogentcore_org_core_plot_plots_XErrorer struct { - IValue interface{} - WXError func(i int) (low float32, high float32) -} - -func (W _cogentcore_org_core_plot_plots_XErrorer) XError(i int) (low float32, high float32) { - return W.WXError(i) -} - -// _cogentcore_org_core_plot_plots_XYLabeler is an interface wrapper for XYLabeler type -type _cogentcore_org_core_plot_plots_XYLabeler struct { - IValue interface{} - WLabel func(i int) string - WLen func() int - WXY func(i int) (x float32, y float32) -} - -func (W _cogentcore_org_core_plot_plots_XYLabeler) Label(i int) string { return W.WLabel(i) } -func (W _cogentcore_org_core_plot_plots_XYLabeler) Len() int { return W.WLen() } -func (W _cogentcore_org_core_plot_plots_XYLabeler) XY(i int) (x float32, y float32) { return W.WXY(i) } - -// _cogentcore_org_core_plot_plots_YErrorer is an interface wrapper for YErrorer type -type _cogentcore_org_core_plot_plots_YErrorer struct { - IValue interface{} - WYError func(i int) (float32, float32) -} - -func (W _cogentcore_org_core_plot_plots_YErrorer) YError(i int) (float32, float32) { - return W.WYError(i) -} diff --git a/yaegicore/symbols/cogentcore_org-core-plot.go b/yaegicore/symbols/cogentcore_org-core-plot.go index 73c7dfb7e3..32d636ac6c 100644 --- a/yaegicore/symbols/cogentcore_org-core-plot.go +++ b/yaegicore/symbols/cogentcore_org-core-plot.go @@ -3,6 +3,7 @@ package symbols import ( + "cogentcore.org/core/math32/minmax" "cogentcore.org/core/plot" "reflect" ) @@ -10,112 +11,164 @@ import ( func init() { Symbols["cogentcore.org/core/plot/plot"] = map[string]reflect.Value{ // function, constant and variable definitions - "CheckFloats": reflect.ValueOf(plot.CheckFloats), - "CheckNaNs": reflect.ValueOf(plot.CheckNaNs), - "CopyValues": reflect.ValueOf(plot.CopyValues), - "CopyXYZs": reflect.ValueOf(plot.CopyXYZs), - "CopyXYs": reflect.ValueOf(plot.CopyXYs), - "DefaultFontFamily": reflect.ValueOf(&plot.DefaultFontFamily).Elem(), - "ErrInfinity": reflect.ValueOf(&plot.ErrInfinity).Elem(), - "ErrNoData": reflect.ValueOf(&plot.ErrNoData).Elem(), - "New": reflect.ValueOf(plot.New), - "PlotXYs": reflect.ValueOf(plot.PlotXYs), - "Range": reflect.ValueOf(plot.Range), - "UTCUnixTime": reflect.ValueOf(&plot.UTCUnixTime).Elem(), - "UnixTimeIn": reflect.ValueOf(plot.UnixTimeIn), - "XYRange": reflect.ValueOf(plot.XYRange), + "AddStylerTo": reflect.ValueOf(plot.AddStylerTo), + "AxisScalesN": reflect.ValueOf(plot.AxisScalesN), + "AxisScalesValues": reflect.ValueOf(plot.AxisScalesValues), + "Box": reflect.ValueOf(plot.Box), + "CheckFloats": reflect.ValueOf(plot.CheckFloats), + "CheckNaNs": reflect.ValueOf(plot.CheckNaNs), + "Circle": reflect.ValueOf(plot.Circle), + "Color": reflect.ValueOf(plot.Color), + "CopyRole": reflect.ValueOf(plot.CopyRole), + "CopyValues": reflect.ValueOf(plot.CopyValues), + "Cross": reflect.ValueOf(plot.Cross), + "Default": reflect.ValueOf(plot.Default), + "DefaultFontFamily": reflect.ValueOf(&plot.DefaultFontFamily).Elem(), + "DefaultOffOnN": reflect.ValueOf(plot.DefaultOffOnN), + "DefaultOffOnValues": reflect.ValueOf(plot.DefaultOffOnValues), + "DrawBox": reflect.ValueOf(plot.DrawBox), + "DrawCircle": reflect.ValueOf(plot.DrawCircle), + "DrawCross": reflect.ValueOf(plot.DrawCross), + "DrawPlus": reflect.ValueOf(plot.DrawPlus), + "DrawPyramid": reflect.ValueOf(plot.DrawPyramid), + "DrawRing": reflect.ValueOf(plot.DrawRing), + "DrawSquare": reflect.ValueOf(plot.DrawSquare), + "DrawTriangle": reflect.ValueOf(plot.DrawTriangle), + "ErrInfinity": reflect.ValueOf(&plot.ErrInfinity).Elem(), + "ErrNoData": reflect.ValueOf(&plot.ErrNoData).Elem(), + "GetStylers": reflect.ValueOf(plot.GetStylers), + "GetStylersFrom": reflect.ValueOf(plot.GetStylersFrom), + "GetStylersFromData": reflect.ValueOf(plot.GetStylersFromData), + "High": reflect.ValueOf(plot.High), + "InverseLinear": reflect.ValueOf(plot.InverseLinear), + "InverseLog": reflect.ValueOf(plot.InverseLog), + "Label": reflect.ValueOf(plot.Label), + "Linear": reflect.ValueOf(plot.Linear), + "Log": reflect.ValueOf(plot.Log), + "Low": reflect.ValueOf(plot.Low), + "MidStep": reflect.ValueOf(plot.MidStep), + "MustCopyRole": reflect.ValueOf(plot.MustCopyRole), + "New": reflect.ValueOf(plot.New), + "NewPlotter": reflect.ValueOf(plot.NewPlotter), + "NewStyle": reflect.ValueOf(plot.NewStyle), + "NewTablePlot": reflect.ValueOf(plot.NewTablePlot), + "NoRole": reflect.ValueOf(plot.NoRole), + "NoStep": reflect.ValueOf(plot.NoStep), + "Off": reflect.ValueOf(plot.Off), + "On": reflect.ValueOf(plot.On), + "PlotX": reflect.ValueOf(plot.PlotX), + "PlotY": reflect.ValueOf(plot.PlotY), + "PlotterByType": reflect.ValueOf(plot.PlotterByType), + "Plotters": reflect.ValueOf(&plot.Plotters).Elem(), + "Plus": reflect.ValueOf(plot.Plus), + "PostStep": reflect.ValueOf(plot.PostStep), + "PreStep": reflect.ValueOf(plot.PreStep), + "Pyramid": reflect.ValueOf(plot.Pyramid), + "Range": reflect.ValueOf(plot.Range), + "RangeClamp": reflect.ValueOf(plot.RangeClamp), + "RegisterPlotter": reflect.ValueOf(plot.RegisterPlotter), + "Ring": reflect.ValueOf(plot.Ring), + "RolesN": reflect.ValueOf(plot.RolesN), + "RolesValues": reflect.ValueOf(plot.RolesValues), + "SetStylers": reflect.ValueOf(plot.SetStylers), + "SetStylersTo": reflect.ValueOf(plot.SetStylersTo), + "ShapesN": reflect.ValueOf(plot.ShapesN), + "ShapesValues": reflect.ValueOf(plot.ShapesValues), + "Size": reflect.ValueOf(plot.Size), + "Square": reflect.ValueOf(plot.Square), + "StepKindN": reflect.ValueOf(plot.StepKindN), + "StepKindValues": reflect.ValueOf(plot.StepKindValues), + "Triangle": reflect.ValueOf(plot.Triangle), + "U": reflect.ValueOf(plot.U), + "UTCUnixTime": reflect.ValueOf(&plot.UTCUnixTime).Elem(), + "UnixTimeIn": reflect.ValueOf(plot.UnixTimeIn), + "V": reflect.ValueOf(plot.V), + "W": reflect.ValueOf(plot.W), + "X": reflect.ValueOf(plot.X), + "Y": reflect.ValueOf(plot.Y), + "Z": reflect.ValueOf(plot.Z), // type definitions "Axis": reflect.ValueOf((*plot.Axis)(nil)), + "AxisScales": reflect.ValueOf((*plot.AxisScales)(nil)), + "AxisStyle": reflect.ValueOf((*plot.AxisStyle)(nil)), "ConstantTicks": reflect.ValueOf((*plot.ConstantTicks)(nil)), - "DataRanger": reflect.ValueOf((*plot.DataRanger)(nil)), + "Data": reflect.ValueOf((*plot.Data)(nil)), + "DefaultOffOn": reflect.ValueOf((*plot.DefaultOffOn)(nil)), "DefaultTicks": reflect.ValueOf((*plot.DefaultTicks)(nil)), "InvertedScale": reflect.ValueOf((*plot.InvertedScale)(nil)), - "Labeler": reflect.ValueOf((*plot.Labeler)(nil)), + "Labels": reflect.ValueOf((*plot.Labels)(nil)), "Legend": reflect.ValueOf((*plot.Legend)(nil)), "LegendEntry": reflect.ValueOf((*plot.LegendEntry)(nil)), "LegendPosition": reflect.ValueOf((*plot.LegendPosition)(nil)), + "LegendStyle": reflect.ValueOf((*plot.LegendStyle)(nil)), "LineStyle": reflect.ValueOf((*plot.LineStyle)(nil)), "LinearScale": reflect.ValueOf((*plot.LinearScale)(nil)), "LogScale": reflect.ValueOf((*plot.LogScale)(nil)), "LogTicks": reflect.ValueOf((*plot.LogTicks)(nil)), "Normalizer": reflect.ValueOf((*plot.Normalizer)(nil)), + "PanZoom": reflect.ValueOf((*plot.PanZoom)(nil)), "Plot": reflect.ValueOf((*plot.Plot)(nil)), + "PlotStyle": reflect.ValueOf((*plot.PlotStyle)(nil)), "Plotter": reflect.ValueOf((*plot.Plotter)(nil)), + "PlotterName": reflect.ValueOf((*plot.PlotterName)(nil)), + "PlotterType": reflect.ValueOf((*plot.PlotterType)(nil)), + "PointStyle": reflect.ValueOf((*plot.PointStyle)(nil)), + "Roles": reflect.ValueOf((*plot.Roles)(nil)), + "Shapes": reflect.ValueOf((*plot.Shapes)(nil)), + "StepKind": reflect.ValueOf((*plot.StepKind)(nil)), + "Style": reflect.ValueOf((*plot.Style)(nil)), "Stylers": reflect.ValueOf((*plot.Stylers)(nil)), - "TensorValues": reflect.ValueOf((*plot.TensorValues)(nil)), - "TensorXYs": reflect.ValueOf((*plot.TensorXYs)(nil)), "Text": reflect.ValueOf((*plot.Text)(nil)), "TextStyle": reflect.ValueOf((*plot.TextStyle)(nil)), "Thumbnailer": reflect.ValueOf((*plot.Thumbnailer)(nil)), "Tick": reflect.ValueOf((*plot.Tick)(nil)), "Ticker": reflect.ValueOf((*plot.Ticker)(nil)), - "TickerFunc": reflect.ValueOf((*plot.TickerFunc)(nil)), "TimeTicks": reflect.ValueOf((*plot.TimeTicks)(nil)), "Valuer": reflect.ValueOf((*plot.Valuer)(nil)), "Values": reflect.ValueOf((*plot.Values)(nil)), - "XValues": reflect.ValueOf((*plot.XValues)(nil)), - "XYValues": reflect.ValueOf((*plot.XYValues)(nil)), - "XYZ": reflect.ValueOf((*plot.XYZ)(nil)), - "XYZer": reflect.ValueOf((*plot.XYZer)(nil)), - "XYZs": reflect.ValueOf((*plot.XYZs)(nil)), - "XYer": reflect.ValueOf((*plot.XYer)(nil)), - "XYs": reflect.ValueOf((*plot.XYs)(nil)), - "YValues": reflect.ValueOf((*plot.YValues)(nil)), + "WidthStyle": reflect.ValueOf((*plot.WidthStyle)(nil)), + "XAxisStyle": reflect.ValueOf((*plot.XAxisStyle)(nil)), // interface wrapper definitions - "_DataRanger": reflect.ValueOf((*_cogentcore_org_core_plot_DataRanger)(nil)), - "_Labeler": reflect.ValueOf((*_cogentcore_org_core_plot_Labeler)(nil)), "_Normalizer": reflect.ValueOf((*_cogentcore_org_core_plot_Normalizer)(nil)), "_Plotter": reflect.ValueOf((*_cogentcore_org_core_plot_Plotter)(nil)), "_Thumbnailer": reflect.ValueOf((*_cogentcore_org_core_plot_Thumbnailer)(nil)), "_Ticker": reflect.ValueOf((*_cogentcore_org_core_plot_Ticker)(nil)), "_Valuer": reflect.ValueOf((*_cogentcore_org_core_plot_Valuer)(nil)), - "_XYZer": reflect.ValueOf((*_cogentcore_org_core_plot_XYZer)(nil)), - "_XYer": reflect.ValueOf((*_cogentcore_org_core_plot_XYer)(nil)), } } -// _cogentcore_org_core_plot_DataRanger is an interface wrapper for DataRanger type -type _cogentcore_org_core_plot_DataRanger struct { - IValue interface{} - WDataRange func(pt *plot.Plot) (xmin float32, xmax float32, ymin float32, ymax float32) -} - -func (W _cogentcore_org_core_plot_DataRanger) DataRange(pt *plot.Plot) (xmin float32, xmax float32, ymin float32, ymax float32) { - return W.WDataRange(pt) -} - -// _cogentcore_org_core_plot_Labeler is an interface wrapper for Labeler type -type _cogentcore_org_core_plot_Labeler struct { - IValue interface{} - WLabel func(i int) string -} - -func (W _cogentcore_org_core_plot_Labeler) Label(i int) string { return W.WLabel(i) } - // _cogentcore_org_core_plot_Normalizer is an interface wrapper for Normalizer type type _cogentcore_org_core_plot_Normalizer struct { IValue interface{} - WNormalize func(min float32, max float32, x float32) float32 + WNormalize func(min float64, max float64, x float64) float64 } -func (W _cogentcore_org_core_plot_Normalizer) Normalize(min float32, max float32, x float32) float32 { +func (W _cogentcore_org_core_plot_Normalizer) Normalize(min float64, max float64, x float64) float64 { return W.WNormalize(min, max, x) } // _cogentcore_org_core_plot_Plotter is an interface wrapper for Plotter type type _cogentcore_org_core_plot_Plotter struct { - IValue interface{} - WApplyStyle func() - WPlot func(pt *plot.Plot) - WXYData func() (data plot.XYer, pixels plot.XYer) + IValue interface{} + WApplyStyle func(plotStyle *plot.PlotStyle) + WData func() (data plot.Data, pixX []float32, pixY []float32) + WPlot func(pt *plot.Plot) + WStylers func() *plot.Stylers + WUpdateRange func(plt *plot.Plot, xr *minmax.F64, yr *minmax.F64, zr *minmax.F64) } -func (W _cogentcore_org_core_plot_Plotter) ApplyStyle() { W.WApplyStyle() } -func (W _cogentcore_org_core_plot_Plotter) Plot(pt *plot.Plot) { W.WPlot(pt) } -func (W _cogentcore_org_core_plot_Plotter) XYData() (data plot.XYer, pixels plot.XYer) { - return W.WXYData() +func (W _cogentcore_org_core_plot_Plotter) ApplyStyle(plotStyle *plot.PlotStyle) { + W.WApplyStyle(plotStyle) +} +func (W _cogentcore_org_core_plot_Plotter) Data() (data plot.Data, pixX []float32, pixY []float32) { + return W.WData() +} +func (W _cogentcore_org_core_plot_Plotter) Plot(pt *plot.Plot) { W.WPlot(pt) } +func (W _cogentcore_org_core_plot_Plotter) Stylers() *plot.Stylers { return W.WStylers() } +func (W _cogentcore_org_core_plot_Plotter) UpdateRange(plt *plot.Plot, xr *minmax.F64, yr *minmax.F64, zr *minmax.F64) { + W.WUpdateRange(plt, xr, yr, zr) } // _cogentcore_org_core_plot_Thumbnailer is an interface wrapper for Thumbnailer type @@ -129,41 +182,21 @@ func (W _cogentcore_org_core_plot_Thumbnailer) Thumbnail(pt *plot.Plot) { W.WThu // _cogentcore_org_core_plot_Ticker is an interface wrapper for Ticker type type _cogentcore_org_core_plot_Ticker struct { IValue interface{} - WTicks func(min float32, max float32) []plot.Tick + WTicks func(min float64, max float64, nticks int) []plot.Tick } -func (W _cogentcore_org_core_plot_Ticker) Ticks(min float32, max float32) []plot.Tick { - return W.WTicks(min, max) +func (W _cogentcore_org_core_plot_Ticker) Ticks(min float64, max float64, nticks int) []plot.Tick { + return W.WTicks(min, max, nticks) } // _cogentcore_org_core_plot_Valuer is an interface wrapper for Valuer type type _cogentcore_org_core_plot_Valuer struct { - IValue interface{} - WLen func() int - WValue func(i int) float32 -} - -func (W _cogentcore_org_core_plot_Valuer) Len() int { return W.WLen() } -func (W _cogentcore_org_core_plot_Valuer) Value(i int) float32 { return W.WValue(i) } - -// _cogentcore_org_core_plot_XYZer is an interface wrapper for XYZer type -type _cogentcore_org_core_plot_XYZer struct { - IValue interface{} - WLen func() int - WXY func(i int) (float32, float32) - WXYZ func(i int) (float32, float32, float32) -} - -func (W _cogentcore_org_core_plot_XYZer) Len() int { return W.WLen() } -func (W _cogentcore_org_core_plot_XYZer) XY(i int) (float32, float32) { return W.WXY(i) } -func (W _cogentcore_org_core_plot_XYZer) XYZ(i int) (float32, float32, float32) { return W.WXYZ(i) } - -// _cogentcore_org_core_plot_XYer is an interface wrapper for XYer type -type _cogentcore_org_core_plot_XYer struct { - IValue interface{} - WLen func() int - WXY func(i int) (x float32, y float32) + IValue interface{} + WFloat1D func(i int) float64 + WLen func() int + WString1D func(i int) string } -func (W _cogentcore_org_core_plot_XYer) Len() int { return W.WLen() } -func (W _cogentcore_org_core_plot_XYer) XY(i int) (x float32, y float32) { return W.WXY(i) } +func (W _cogentcore_org_core_plot_Valuer) Float1D(i int) float64 { return W.WFloat1D(i) } +func (W _cogentcore_org_core_plot_Valuer) Len() int { return W.WLen() } +func (W _cogentcore_org_core_plot_Valuer) String1D(i int) string { return W.WString1D(i) } From d25dbd2bba924c19fdfbade090b75d728a9593c2 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 01:21:01 -0800 Subject: [PATCH 297/311] sim stats working much better: need loop phase input --- tensor/base.go | 3 + tensor/examples/datafs-sim/sim.go | 283 ++++++++++++---------------- tensor/examples/datafs-sim/sim.goal | 283 ++++++++++++---------------- tensor/shape.go | 3 + 4 files changed, 248 insertions(+), 324 deletions(-) diff --git a/tensor/base.go b/tensor/base.go index 38903ed56e..fe778339d2 100644 --- a/tensor/base.go +++ b/tensor/base.go @@ -85,6 +85,9 @@ func (tsr *Base[T]) Set1D(val T, i int) { tsr.Values[tsr.shape.Header+i] = val } // it is best to first set the anticipated full size, which allocates the // full amount of memory, and then set to 0 and grow incrementally. func (tsr *Base[T]) SetNumRows(rows int) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } _, cells := tsr.shape.RowCellSize() nln := tsr.shape.Header + rows*cells tsr.shape.Sizes[0] = rows diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 9e56fd468d..f77f018744 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -27,12 +27,40 @@ const ( Run ) +// LoopPhase is the phase of loop processing for given time. +type LoopPhase int32 //enums:enum + +const ( + // Start is the start of the loop. + Start LoopPhase = iota + + // Step is at each iteration of the loop. + Step + + // End is at the end of the loop, after all iterations. + // This is only called for the outer-most loop, because all others + // are synonymous with Step at the next higher level. + End +) + type Sim struct { - Root *datafs.Data - Config *datafs.Data - Stats *datafs.Data - StatFuncs []func(tm Times) - Counters [TimesN]int + // Root is the root data dir. + Root *datafs.Data + + // Config has config data. + Config *datafs.Data + + // Stats has all stats data. + Stats *datafs.Data + + // Current has current value of all stats + Current *datafs.Data + + // StatFuncs are statistics functions, per stat, handles everything. + StatFuncs []func(tm Times, lp LoopPhase) + + // Counters are current values of counters: normally in looper. + Counters [TimesN]int } // ConfigAll configures the sim @@ -50,188 +78,138 @@ func (ss *Sim) ConfigAll() { ss.ConfigStats() } -func (ss *Sim) AddStat(f func(tm Times)) { +func (ss *Sim) AddStat(f func(tm Times, lp LoopPhase)) { ss.StatFuncs = append(ss.StatFuncs, f) } -func (ss *Sim) RunStats(tm Times) { +func (ss *Sim) RunStats(tm Times, lp LoopPhase) { for _, sf := range ss.StatFuncs { - sf(tm) + sf(tm, lp) } } func (ss *Sim) ConfigStats() { ss.Stats, _ = ss.Root.Mkdir("Stats") + ss.Current, _ = ss.Stats.Mkdir("Current") ctrs := []Times{Run, Epoch, Trial} for _, ctr := range ctrs { - ss.AddStat(func(tm Times) { - if tm > ctr { + ss.AddStat(func(tm Times, lp LoopPhase) { + if lp == End || tm > ctr { // don't record counter for time above it return } - name := ctr.String() - ctv := ss.Counters[ctr] - td := ss.Stats.RecycleDir(tm.String()) - cd := ss.Stats.RecycleDir("Current") - datafs.Scalar[int](cd, name).SetInt1D(ctv, 0) + name := ctr.String() // name of stat = counter + td := ss.Stats.RecycleDir(tm.String()) // log for tm time tv := datafs.Value[int](td, name) + if lp == Start { + tv.SetNumRows(0) + if ps := plot.GetStylersFrom(tv); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0) + }) + plot.SetStylersTo(tv, ps) + } + return + } + ctv := ss.Counters[ctr] + datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) tv.AppendRow(tensor.NewIntScalar(ctv)) + }) + } + // note: it is essential to only have 1 per func + // so generic names can be used for everything. + ss.AddStat(func(tm Times, lp LoopPhase) { + if lp == End { // only called for Run; we ignore + return + } + name := "SSE" + td := ss.Stats.RecycleDir(tm.String()) + tv := datafs.Value[float64](td, name) + if lp == Start { + tv.SetNumRows(0) if ps := plot.GetStylersFrom(tv); ps == nil { ps.Add(func(s *plot.Style) { - s.Range.SetMin(0) + s.Range.SetMin(0).SetMax(1) + s.On = true }) plot.SetStylersTo(tv, ps) } - if tm > Trial { - subd := ss.Stats.RecycleDir((tm - 1).String()) - subd.Value(name).(tensor.Values).SetNumRows(0) - } - }) - } - ss.AddStat(func(tm Times) { - sseName := "SSE" - errName := "Err" + return + } + switch tm { + case Trial: + sv := rand.Float64() + datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) + tv.AppendRow(tensor.NewFloat64Scalar(sv)) + case Epoch: + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) + case Run: + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) + } + }) + ss.AddStat(func(tm Times, lp LoopPhase) { + if lp == End { // only called for Run; we ignore + return + } + name := "Err" td := ss.Stats.RecycleDir(tm.String()) + tv := datafs.Value[float64](td, name) + if lp == Start { + tv.SetNumRows(0) + if ps := plot.GetStylersFrom(tv); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0).SetMax(1) + s.On = true + }) + plot.SetStylersTo(tv, ps) + } + return + } switch tm { case Trial: - cd := ss.Stats.RecycleDir("Current") - sse := rand.Float32() - terr := float32(1) + sse := ss.Current.Item("SSE").AsFloat64() + sv := 1.0 if sse < 0.5 { - terr = 0 + sv = 0 } - datafs.Scalar[float32](cd, sseName).SetFloat(float64(sse), 0) - datafs.Scalar[float32](cd, errName).SetFloat(float64(terr), 0) - datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat32Scalar(sse)) - datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat32Scalar(terr)) + datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) + tv.AppendRow(tensor.NewFloat64Scalar(sv)) case Epoch: - trld, _ := ss.Stats.Mkdir(Trial.String()) - sse := stats.StatMean.Call(trld.Value(sseName)).Float1D(0) - terr := stats.StatMean.Call(trld.Value(errName)).Float1D(0) - datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat64Scalar(sse)) - datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat64Scalar(terr)) - trld.Value(sseName).(tensor.Values).SetNumRows(0) - trld.Value(errName).(tensor.Values).SetNumRows(0) + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)).Float1D(0) + tv.AppendRow(tensor.NewFloat64Scalar(sv)) + case Run: + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)).Float1D(0) + tv.AppendRow(tensor.NewFloat64Scalar(sv)) } }) } -// // ConfigStats adds basic stats that we record for our simulation. -// func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { -// stats, _ := dir.Mkdir("Stats") -// datafs.NewScalar[int](stats, "Run", "Epoch", "Trial") // counters -// datafs.NewScalar[string](stats, "TrialName") -// datafs.NewScalar[float32](stats, "SSE", "AvgSSE", "TrlErr") -// z1, key := plotcore.PlotColumnZeroOne() -// stats.SetMetaItems(key, z1, "AvgErr", "TrlErr") -// zmax, _ := plotcore.PlotColumnZeroOne() -// zmax.Range.FixMax = false -// stats.SetMetaItems(key, z1, "SSE") -// return stats -// } -// -// // ConfigLogs adds first-level logging of stats into tensors -// func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { -// logd, _ := dir.Mkdir("Log") -// trial := ss.ConfigTrialLog(logd) -// ss.ConfigAggLog(logd, "Epoch", trial, stats.StatMean, stats.StatSem, stats.StatMin) -// return logd -// } -// -// // ConfigTrialLog adds first-level logging of stats into tensors -// func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { -// logd, _ := dir.Mkdir("Trial") -// ntrial := ss.Config.Item("NTrial").AsInt() -// sitems := ss.Stats.ValuesFunc(nil) -// for _, st := range sitems { -// nm := st.Metadata().Name() -// lt := logd.NewOfType(nm, st.DataType(), ntrial) -// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source -// tensor.SetCalcFunc(lt, func() error { -// trl := ss.Stats.Item("Trial").AsInt() -// if st.IsString() { -// lt.SetStringRow(st.String1D(0), trl) -// } else { -// lt.SetFloatRow(st.Float1D(0), trl) -// } -// return nil -// }) -// } -// alllogd, _ := dir.Mkdir("AllTrials") -// for _, st := range sitems { -// nm := st.Metadata().Name() -// // allocate full size -// lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) -// lt.SetShapeSizes(0) // then truncate to 0 -// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source -// tensor.SetCalcFunc(lt, func() error { -// row := lt.DimSize(0) -// lt.SetShapeSizes(row + 1) -// if st.IsString() { -// lt.SetStringRow(st.String1D(0), row) -// } else { -// lt.SetFloatRow(st.Float1D(0), row) -// } -// return nil -// }) -// } -// return logd -// } -// -// // ConfigAggLog adds a higher-level logging of lower-level into higher-level tensors -// func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, aggs ...stats.Stats) *datafs.Data { -// logd, _ := dir.Mkdir(level) -// sitems := ss.Stats.ValuesFunc(nil) -// nctr := ss.Config.Item("N" + level).AsInt() -// for _, st := range sitems { -// if st.IsString() { -// continue -// } -// nm := st.Metadata().Name() -// src := from.Value(nm) -// if st.DataType() >= reflect.Float32 { -// // todo: pct correct etc -// dd, _ := logd.Mkdir(nm) -// for _, ag := range aggs { // key advantage of dir structure: multiple stats per item -// lt := dd.NewOfType(ag.String(), st.DataType(), nctr) -// lt.Metadata().Copy(*st.Metadata()) -// tensor.SetCalcFunc(lt, func() error { -// stout := ag.Call(src) -// ctr := ss.Stats.Item(level).AsInt() -// lt.SetFloatRow(stout.FloatRow(0), ctr) -// return nil -// }) -// } -// } else { -// lt := logd.NewOfType(nm, st.DataType(), nctr) -// lt.Metadata().Copy(*st.Metadata()) -// tensor.SetCalcFunc(lt, func() error { -// v := st.Float1D(0) -// ctr := ss.Stats.Item(level).AsInt() -// lt.SetFloatRow(v, ctr) -// return nil -// }) -// } -// } -// return logd -// } - func (ss *Sim) Run() { mx := ss.Config.Value("Max").(*tensor.Int) nrun := mx.Value1D(int(Run)) nepc := mx.Value1D(int(Epoch)) ntrl := mx.Value1D(int(Trial)) + ss.RunStats(Run, Start) for run := range nrun { ss.Counters[Run] = run + ss.RunStats(Epoch, Start) for epc := range nepc { ss.Counters[Epoch] = epc + ss.RunStats(Trial, Start) for trl := range ntrl { ss.Counters[Trial] = trl - ss.RunStats(Trial) + ss.RunStats(Trial, Step) } - ss.RunStats(Epoch) + ss.RunStats(Epoch, Step) } - ss.RunStats(Run) + ss.RunStats(Run, Step) } + ss.RunStats(Run, End) // alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) // dir, _ := ss.Logs.Mkdir("Stats") // stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") @@ -240,25 +218,6 @@ func (ss *Sim) Run() { // stats.TableGroupStats(dir, stats.StatSem, alldt, sts...) } -// func (ss *Sim) RunTrial(trl int) { -// ss.Stats.Item("TrialName").SetString("Trial_" + strconv.Itoa(trl)) -// sse := rand.Float32() -// avgSSE := rand.Float32() -// ss.Stats.Item("SSE").SetFloat32(sse) -// ss.Stats.Item("AvgSSE").SetFloat32(avgSSE) -// trlErr := float32(1) -// if sse < 0.5 { -// trlErr = 0 -// } -// ss.Stats.Item("TrlErr").SetFloat32(trlErr) -// ss.Logs.Item("Trial").CalcAll() -// ss.Logs.Item("AllTrials").CalcAll() -// } - -// func (ss *Sim) EpochDone() { -// ss.Logs.Item("Epoch").CalcAll() -// } - func main() { ss := &Sim{} ss.ConfigAll() diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal index db2fd5d7d7..042622eded 100644 --- a/tensor/examples/datafs-sim/sim.goal +++ b/tensor/examples/datafs-sim/sim.goal @@ -25,12 +25,40 @@ const ( Run ) +// LoopPhase is the phase of loop processing for given time. +type LoopPhase int32 //enums:enum + +const ( + // Start is the start of the loop. + Start LoopPhase = iota + + // Step is at each iteration of the loop. + Step + + // End is at the end of the loop, after all iterations. + // This is only called for the outer-most loop, because all others + // are synonymous with Step at the next higher level. + End +) + type Sim struct { - Root *datafs.Data - Config *datafs.Data - Stats *datafs.Data - StatFuncs []func(tm Times) - Counters [TimesN]int + // Root is the root data dir. + Root *datafs.Data + + // Config has config data. + Config *datafs.Data + + // Stats has all stats data. + Stats *datafs.Data + + // Current has current value of all stats + Current *datafs.Data + + // StatFuncs are statistics functions, per stat, handles everything. + StatFuncs []func(tm Times, lp LoopPhase) + + // Counters are current values of counters: normally in looper. + Counters [TimesN]int } // ConfigAll configures the sim @@ -48,188 +76,138 @@ func (ss *Sim) ConfigAll() { ss.ConfigStats() } -func (ss *Sim) AddStat(f func(tm Times)) { +func (ss *Sim) AddStat(f func(tm Times, lp LoopPhase)) { ss.StatFuncs = append(ss.StatFuncs, f) } -func (ss *Sim) RunStats(tm Times) { +func (ss *Sim) RunStats(tm Times, lp LoopPhase) { for _, sf := range ss.StatFuncs { - sf(tm) + sf(tm, lp) } } func (ss *Sim) ConfigStats() { ss.Stats, _ = ss.Root.Mkdir("Stats") + ss.Current, _ = ss.Stats.Mkdir("Current") ctrs := []Times{Run, Epoch, Trial} for _, ctr := range ctrs { - ss.AddStat(func(tm Times) { - if tm > ctr { + ss.AddStat(func(tm Times, lp LoopPhase) { + if lp == End || tm > ctr { // don't record counter for time above it return } - name := ctr.String() - ctv := ss.Counters[ctr] - td := ss.Stats.RecycleDir(tm.String()) - cd := ss.Stats.RecycleDir("Current") - datafs.Scalar[int](cd, name).SetInt1D(ctv, 0) + name := ctr.String() // name of stat = counter + td := ss.Stats.RecycleDir(tm.String()) // log for tm time tv := datafs.Value[int](td, name) + if lp == Start { + tv.SetNumRows(0) + if ps := plot.GetStylersFrom(tv); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0) + }) + plot.SetStylersTo(tv, ps) + } + return + } + ctv := ss.Counters[ctr] + datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) tv.AppendRow(tensor.NewIntScalar(ctv)) + }) + } + // note: it is essential to only have 1 per func + // so generic names can be used for everything. + ss.AddStat(func(tm Times, lp LoopPhase) { + if lp == End { // only called for Run; we ignore + return + } + name := "SSE" + td := ss.Stats.RecycleDir(tm.String()) + tv := datafs.Value[float64](td, name) + if lp == Start { + tv.SetNumRows(0) if ps := plot.GetStylersFrom(tv); ps == nil { ps.Add(func(s *plot.Style) { - s.Range.SetMin(0) + s.Range.SetMin(0).SetMax(1) + s.On = true }) plot.SetStylersTo(tv, ps) } - if tm > Trial { - subd := ss.Stats.RecycleDir((tm - 1).String()) - subd.Value(name).(tensor.Values).SetNumRows(0) - } - }) - } - ss.AddStat(func(tm Times) { - sseName := "SSE" - errName := "Err" + return + } + switch tm { + case Trial: + sv := rand.Float64() + datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) + tv.AppendRow(tensor.NewFloat64Scalar(sv)) + case Epoch: + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) + case Run: + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) + } + }) + ss.AddStat(func(tm Times, lp LoopPhase) { + if lp == End { // only called for Run; we ignore + return + } + name := "Err" td := ss.Stats.RecycleDir(tm.String()) + tv := datafs.Value[float64](td, name) + if lp == Start { + tv.SetNumRows(0) + if ps := plot.GetStylersFrom(tv); ps == nil { + ps.Add(func(s *plot.Style) { + s.Range.SetMin(0).SetMax(1) + s.On = true + }) + plot.SetStylersTo(tv, ps) + } + return + } switch tm { case Trial: - cd := ss.Stats.RecycleDir("Current") - sse := rand.Float32() - terr := float32(1) + sse := ss.Current.Item("SSE").AsFloat64() + sv := 1.0 if sse < 0.5 { - terr = 0 + sv = 0 } - datafs.Scalar[float32](cd, sseName).SetFloat(float64(sse), 0) - datafs.Scalar[float32](cd, errName).SetFloat(float64(terr), 0) - datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat32Scalar(sse)) - datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat32Scalar(terr)) + datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) + tv.AppendRow(tensor.NewFloat64Scalar(sv)) case Epoch: - trld, _ := ss.Stats.Mkdir(Trial.String()) - sse := stats.StatMean.Call(trld.Value(sseName)).Float1D(0) - terr := stats.StatMean.Call(trld.Value(errName)).Float1D(0) - datafs.Value[float32](td, sseName).AppendRow(tensor.NewFloat64Scalar(sse)) - datafs.Value[float32](td, errName).AppendRow(tensor.NewFloat64Scalar(terr)) - trld.Value(sseName).(tensor.Values).SetNumRows(0) - trld.Value(errName).(tensor.Values).SetNumRows(0) + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) + case Run: + subd := ss.Stats.RecycleDir((tm - 1).String()) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) } }) } -// // ConfigStats adds basic stats that we record for our simulation. -// func (ss *Sim) ConfigStats(dir *datafs.Data) *datafs.Data { -// stats, _ := dir.Mkdir("Stats") -// datafs.NewScalar[int](stats, "Run", "Epoch", "Trial") // counters -// datafs.NewScalar[string](stats, "TrialName") -// datafs.NewScalar[float32](stats, "SSE", "AvgSSE", "TrlErr") -// z1, key := plotcore.PlotColumnZeroOne() -// stats.SetMetaItems(key, z1, "AvgErr", "TrlErr") -// zmax, _ := plotcore.PlotColumnZeroOne() -// zmax.Range.FixMax = false -// stats.SetMetaItems(key, z1, "SSE") -// return stats -// } -// -// // ConfigLogs adds first-level logging of stats into tensors -// func (ss *Sim) ConfigLogs(dir *datafs.Data) *datafs.Data { -// logd, _ := dir.Mkdir("Log") -// trial := ss.ConfigTrialLog(logd) -// ss.ConfigAggLog(logd, "Epoch", trial, stats.StatMean, stats.StatSem, stats.StatMin) -// return logd -// } -// -// // ConfigTrialLog adds first-level logging of stats into tensors -// func (ss *Sim) ConfigTrialLog(dir *datafs.Data) *datafs.Data { -// logd, _ := dir.Mkdir("Trial") -// ntrial := ss.Config.Item("NTrial").AsInt() -// sitems := ss.Stats.ValuesFunc(nil) -// for _, st := range sitems { -// nm := st.Metadata().Name() -// lt := logd.NewOfType(nm, st.DataType(), ntrial) -// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source -// tensor.SetCalcFunc(lt, func() error { -// trl := ss.Stats.Item("Trial").AsInt() -// if st.IsString() { -// lt.SetStringRow(st.String1D(0), trl) -// } else { -// lt.SetFloatRow(st.Float1D(0), trl) -// } -// return nil -// }) -// } -// alllogd, _ := dir.Mkdir("AllTrials") -// for _, st := range sitems { -// nm := st.Metadata().Name() -// // allocate full size -// lt := alllogd.NewOfType(nm, st.DataType(), ntrial*ss.Config.Item("NEpoch").AsInt()*ss.Config.Item("NRun").AsInt()) -// lt.SetShapeSizes(0) // then truncate to 0 -// lt.Metadata().Copy(*st.Metadata()) // key affordance: we get meta data from source -// tensor.SetCalcFunc(lt, func() error { -// row := lt.DimSize(0) -// lt.SetShapeSizes(row + 1) -// if st.IsString() { -// lt.SetStringRow(st.String1D(0), row) -// } else { -// lt.SetFloatRow(st.Float1D(0), row) -// } -// return nil -// }) -// } -// return logd -// } -// -// // ConfigAggLog adds a higher-level logging of lower-level into higher-level tensors -// func (ss *Sim) ConfigAggLog(dir *datafs.Data, level string, from *datafs.Data, aggs ...stats.Stats) *datafs.Data { -// logd, _ := dir.Mkdir(level) -// sitems := ss.Stats.ValuesFunc(nil) -// nctr := ss.Config.Item("N" + level).AsInt() -// for _, st := range sitems { -// if st.IsString() { -// continue -// } -// nm := st.Metadata().Name() -// src := from.Value(nm) -// if st.DataType() >= reflect.Float32 { -// // todo: pct correct etc -// dd, _ := logd.Mkdir(nm) -// for _, ag := range aggs { // key advantage of dir structure: multiple stats per item -// lt := dd.NewOfType(ag.String(), st.DataType(), nctr) -// lt.Metadata().Copy(*st.Metadata()) -// tensor.SetCalcFunc(lt, func() error { -// stout := ag.Call(src) -// ctr := ss.Stats.Item(level).AsInt() -// lt.SetFloatRow(stout.FloatRow(0), ctr) -// return nil -// }) -// } -// } else { -// lt := logd.NewOfType(nm, st.DataType(), nctr) -// lt.Metadata().Copy(*st.Metadata()) -// tensor.SetCalcFunc(lt, func() error { -// v := st.Float1D(0) -// ctr := ss.Stats.Item(level).AsInt() -// lt.SetFloatRow(v, ctr) -// return nil -// }) -// } -// } -// return logd -// } - func (ss *Sim) Run() { mx := ss.Config.Value("Max").(*tensor.Int) nrun := mx.Value1D(int(Run)) nepc := mx.Value1D(int(Epoch)) ntrl := mx.Value1D(int(Trial)) + ss.RunStats(Run, Start) for run := range nrun { ss.Counters[Run] = run + ss.RunStats(Epoch, Start) for epc := range nepc { ss.Counters[Epoch] = epc + ss.RunStats(Trial, Start) for trl := range ntrl { ss.Counters[Trial] = trl - ss.RunStats(Trial) + ss.RunStats(Trial, Step) } - ss.RunStats(Epoch) + ss.RunStats(Epoch, Step) } - ss.RunStats(Run) + ss.RunStats(Run, Step) } + ss.RunStats(Run, End) // alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) // dir, _ := ss.Logs.Mkdir("Stats") // stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") @@ -238,25 +216,6 @@ func (ss *Sim) Run() { // stats.TableGroupStats(dir, stats.StatSem, alldt, sts...) } -// func (ss *Sim) RunTrial(trl int) { -// ss.Stats.Item("TrialName").SetString("Trial_" + strconv.Itoa(trl)) -// sse := rand.Float32() -// avgSSE := rand.Float32() -// ss.Stats.Item("SSE").SetFloat32(sse) -// ss.Stats.Item("AvgSSE").SetFloat32(avgSSE) -// trlErr := float32(1) -// if sse < 0.5 { -// trlErr = 0 -// } -// ss.Stats.Item("TrlErr").SetFloat32(trlErr) -// ss.Logs.Item("Trial").CalcAll() -// ss.Logs.Item("AllTrials").CalcAll() -// } - -// func (ss *Sim) EpochDone() { -// ss.Logs.Item("Epoch").CalcAll() -// } - func main() { ss := &Sim{} ss.ConfigAll() diff --git a/tensor/shape.go b/tensor/shape.go index c2d311a7cb..e5caa93319 100644 --- a/tensor/shape.go +++ b/tensor/shape.go @@ -111,6 +111,9 @@ func (sh *Shape) IsEqual(oth *Shape) bool { // and the size of all the remaining inner dimensions (the "cell" size). // Used for Tensors that are columns in a data table. func (sh *Shape) RowCellSize() (rows, cells int) { + if len(sh.Sizes) == 0 { + return 0, 1 + } rows = sh.Sizes[0] if len(sh.Sizes) == 1 { cells = 1 From 90d672fe14f4343ba8d197e067a44ad1a30017c8 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 01:42:23 -0800 Subject: [PATCH 298/311] add AppendRowFloat, Int, String -- much cleaner for those cases --- tensor/bool.go | 19 ++++++++---- tensor/examples/datafs-sim/sim.go | 14 ++++----- tensor/examples/datafs-sim/sim.goal | 6 ++-- tensor/number.go | 45 ++++++++++++++++++++++++++-- tensor/rowmajor.go | 9 ++++++ tensor/rows.go | 46 +++++++++++++++++++++-------- tensor/string.go | 45 ++++++++++++++++++++++++++-- 7 files changed, 151 insertions(+), 33 deletions(-) diff --git a/tensor/bool.go b/tensor/bool.go index db98b71795..7aa0dc9db8 100644 --- a/tensor/bool.go +++ b/tensor/bool.go @@ -130,7 +130,7 @@ func (tsr *Bool) SetRowTensor(val Values, row int) {} // AppendRow not possible with Bool. func (tsr *Bool) AppendRow(val Values) {} -///////////////////// Bool +/////// Bool func (tsr *Bool) Value(i ...int) bool { return tsr.Values.Index(tsr.shape.IndexTo1D(i...)) @@ -144,7 +144,7 @@ func (tsr *Bool) Value1D(i int) bool { return tsr.Values.Index(i) } func (tsr *Bool) Set1D(val bool, i int) { tsr.Values.Set(val, i) } -///////////////////// Strings +/////// Strings func (tsr *Bool) String1D(off int) string { return reflectx.ToString(tsr.Values.Index(off)) @@ -178,7 +178,10 @@ func (tsr *Bool) SetStringRow(val string, row, cell int) { } } -///////////////////// Floats +// AppendRowString not possible with Bool. +func (tsr *Bool) AppendRowString(val ...string) {} + +/////// Floats func (tsr *Bool) Float(i ...int) float64 { return BoolToFloat64(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) @@ -206,7 +209,10 @@ func (tsr *Bool) SetFloatRow(val float64, row, cell int) { tsr.Values.Set(Float64ToBool(val), row*sz+cell) } -///////////////////// Ints +// AppendRowFloat not possible with Bool. +func (tsr *Bool) AppendRowFloat(val ...float64) {} + +/////// Ints func (tsr *Bool) Int(i ...int) int { return BoolToInt(tsr.Values.Index(tsr.shape.IndexTo1D(i...))) @@ -234,7 +240,10 @@ func (tsr *Bool) SetIntRow(val int, row, cell int) { tsr.Values.Set(IntToBool(val), row*sz+cell) } -///////////////////// Bools +// AppendRowInt not possible with Bool. +func (tsr *Bool) AppendRowInt(val ...int) {} + +/////// Bools func (tsr *Bool) Bool(i ...int) bool { return tsr.Values.Index(tsr.shape.IndexTo1D(i...)) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index f77f018744..9c8b642ced 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -112,7 +112,7 @@ func (ss *Sim) ConfigStats() { } ctv := ss.Counters[ctr] datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) - tv.AppendRow(tensor.NewIntScalar(ctv)) + tv.AppendRowInt(ctv) }) } // note: it is essential to only have 1 per func @@ -139,7 +139,7 @@ func (ss *Sim) ConfigStats() { case Trial: sv := rand.Float64() datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRow(tensor.NewFloat64Scalar(sv)) + tv.AppendRowFloat(sv) case Epoch: subd := ss.Stats.RecycleDir((tm - 1).String()) sv := stats.StatMean.Call(subd.Value(name)) @@ -176,15 +176,15 @@ func (ss *Sim) ConfigStats() { sv = 0 } datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRow(tensor.NewFloat64Scalar(sv)) + tv.AppendRowFloat(sv) case Epoch: subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)).Float1D(0) - tv.AppendRow(tensor.NewFloat64Scalar(sv)) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) case Run: subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)).Float1D(0) - tv.AppendRow(tensor.NewFloat64Scalar(sv)) + sv := stats.StatMean.Call(subd.Value(name)) + tv.AppendRow(sv) } }) } diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal index 042622eded..c20574e7c3 100644 --- a/tensor/examples/datafs-sim/sim.goal +++ b/tensor/examples/datafs-sim/sim.goal @@ -110,7 +110,7 @@ func (ss *Sim) ConfigStats() { } ctv := ss.Counters[ctr] datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) - tv.AppendRow(tensor.NewIntScalar(ctv)) + tv.AppendRowInt(ctv) }) } // note: it is essential to only have 1 per func @@ -137,7 +137,7 @@ func (ss *Sim) ConfigStats() { case Trial: sv := rand.Float64() datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRow(tensor.NewFloat64Scalar(sv)) + tv.AppendRowFloat(sv) case Epoch: subd := ss.Stats.RecycleDir((tm - 1).String()) sv := stats.StatMean.Call(subd.Value(name)) @@ -174,7 +174,7 @@ func (ss *Sim) ConfigStats() { sv = 0 } datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRow(tensor.NewFloat64Scalar(sv)) + tv.AppendRowFloat(sv) case Epoch: subd := ss.Stats.RecycleDir((tm - 1).String()) sv := stats.StatMean.Call(subd.Value(name)) diff --git a/tensor/number.go b/tensor/number.go index 34f9597211..37adb775de 100644 --- a/tensor/number.go +++ b/tensor/number.go @@ -123,7 +123,7 @@ func (tsr *Number[T]) SetDiv(val T, i ...int) { tsr.Values[tsr.shape.IndexTo1D(i...)] /= val } -///////////////////// Strings +/////// Strings func (tsr *Number[T]) SetString(val string, i ...int) { if fv, err := strconv.ParseFloat(val, 64); err == nil { @@ -144,7 +144,20 @@ func (tsr *Number[T]) SetStringRow(val string, row, cell int) { } } -///////////////////// Floats +// AppendRowString adds a row and sets string value(s), up to number of cells. +func (tsr *Number[T]) AppendRowString(val ...string) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow, sz := tsr.shape.RowCellSize() + tsr.SetNumRows(nrow + 1) + mx := min(sz, len(val)) + for i := range mx { + tsr.SetStringRow(val[i], nrow, i) + } +} + +/////// Floats func (tsr *Number[T]) Float(i ...int) float64 { return float64(tsr.Values[tsr.shape.IndexTo1D(i...)]) @@ -173,7 +186,20 @@ func (tsr *Number[T]) SetFloatRow(val float64, row, cell int) { tsr.Values[tsr.shape.Header+row*sz+cell] = T(val) } -///////////////////// Ints +// AppendRowFloat adds a row and sets float value(s), up to number of cells. +func (tsr *Number[T]) AppendRowFloat(val ...float64) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow, sz := tsr.shape.RowCellSize() + tsr.SetNumRows(nrow + 1) + mx := min(sz, len(val)) + for i := range mx { + tsr.SetFloatRow(val[i], nrow, i) + } +} + +/////// Ints func (tsr *Number[T]) Int(i ...int) int { return int(tsr.Values[tsr.shape.IndexTo1D(i...)]) @@ -202,6 +228,19 @@ func (tsr *Number[T]) SetIntRow(val int, row, cell int) { tsr.Values[tsr.shape.Header+row*sz+cell] = T(val) } +// AppendRowInt adds a row and sets int value(s), up to number of cells. +func (tsr *Number[T]) AppendRowInt(val ...int) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow, sz := tsr.shape.RowCellSize() + tsr.SetNumRows(nrow + 1) + mx := min(sz, len(val)) + for i := range mx { + tsr.SetIntRow(val[i], nrow, i) + } +} + // SetZeros is simple convenience function initialize all values to 0 func (tsr *Number[T]) SetZeros() { n := len(tsr.Values) diff --git a/tensor/rowmajor.go b/tensor/rowmajor.go index ab86add1da..3bce95bc8a 100644 --- a/tensor/rowmajor.go +++ b/tensor/rowmajor.go @@ -44,6 +44,9 @@ type RowMajor interface { // dimension, and cell is a 1D index into remaining inner dimensions. SetFloatRow(val float64, row, cell int) + // AppendRowFloat adds a row and sets float value(s), up to number of cells. + AppendRowFloat(val ...float64) + //////// Ints // IntRow returns the value at given row and cell, where row is the outermost @@ -54,6 +57,9 @@ type RowMajor interface { // dimension, and cell is a 1D index into remaining inner dimensions. SetIntRow(val int, row, cell int) + // AppendRowInt adds a row and sets int value(s), up to number of cells. + AppendRowInt(val ...int) + //////// Strings // StringRow returns the value at given row and cell, where row is the outermost @@ -67,4 +73,7 @@ type RowMajor interface { // [Rows] tensors index along the row, and use this interface extensively. // This is useful for lists of patterns, and the [table.Table] container. SetStringRow(val string, row, cell int) + + // AppendRowString adds a row and sets string value(s), up to number of cells. + AppendRowString(val ...string) } diff --git a/tensor/rows.go b/tensor/rows.go index 69c42cfc92..7f22263cde 100644 --- a/tensor/rows.go +++ b/tensor/rows.go @@ -395,15 +395,22 @@ func (rw *Rows) CopyIndexes(oix *Rows) { } } +// addRowsIndexes adds n rows to indexes starting at end of current tensor size +func (rw *Rows) addRowsIndexes(n int) { //types:add + if rw.Indexes == nil { + return + } + stidx := rw.Tensor.DimSize(0) + for i := stidx; i < stidx+n; i++ { + rw.Indexes = append(rw.Indexes, i) + } +} + // AddRows adds n rows to end of underlying Tensor, and to the indexes in this view func (rw *Rows) AddRows(n int) { //types:add stidx := rw.Tensor.DimSize(0) + rw.addRowsIndexes(n) rw.Tensor.SetNumRows(stidx + n) - if rw.Indexes != nil { - for i := stidx; i < stidx+n; i++ { - rw.Indexes = append(rw.Indexes, i) - } - } } // InsertRows adds n rows to end of underlying Tensor, and to the indexes starting at @@ -433,10 +440,7 @@ func (rw *Rows) Swap(i, j int) { rw.Indexes[i], rw.Indexes[j] = rw.Indexes[j], rw.Indexes[i] } -/////////////////////////////////////////////// -// Rows access - -///////////////////// Floats +/////// Floats // Float returns the value of given index as a float64. // The first index value is indirected through the indexes. @@ -495,7 +499,7 @@ func (rw *Rows) SetFloat1D(val float64, i int) { rw.SetFloat(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -///////////////////// Strings +/////// Strings // StringValue returns the value of given index as a string. // The first index value is indirected through the indexes. @@ -535,6 +539,12 @@ func (rw *Rows) SetStringRow(val string, row, cell int) { rw.Tensor.SetStringRow(val, rw.RowIndex(row), cell) } +// AppendRowFloat adds a row and sets float value(s), up to number of cells. +func (rw *Rows) AppendRowFloat(val ...float64) { + rw.addRowsIndexes(1) + rw.Tensor.AppendRowFloat(val...) +} + // String1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (rw *Rows) String1D(i int) string { @@ -553,7 +563,13 @@ func (rw *Rows) SetString1D(val string, i int) { rw.SetString(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -///////////////////// Ints +// AppendRowString adds a row and sets string value(s), up to number of cells. +func (rw *Rows) AppendRowString(val ...string) { + rw.addRowsIndexes(1) + rw.Tensor.AppendRowString(val...) +} + +/////// Ints // Int returns the value of given index as an int. // The first index value is indirected through the indexes. @@ -594,6 +610,12 @@ func (rw *Rows) SetIntRow(val int, row, cell int) { rw.Tensor.SetIntRow(val, rw.RowIndex(row), cell) } +// AppendRowInt adds a row and sets int value(s), up to number of cells. +func (rw *Rows) AppendRowInt(val ...int) { + rw.addRowsIndexes(1) + rw.Tensor.AppendRowInt(val...) +} + // Int1D is somewhat expensive if indexes are set, because it needs to convert // the flat index back into a full n-dimensional index and then use that api. func (rw *Rows) Int1D(i int) int { @@ -612,7 +634,7 @@ func (rw *Rows) SetInt1D(val int, i int) { rw.SetInt(val, rw.Tensor.Shape().IndexFrom1D(i)...) } -///////////////////// SubSpaces +/////// SubSpaces // SubSpace returns a new tensor with innermost subspace at given // offset(s) in outermost dimension(s) (len(offs) < NumDims). diff --git a/tensor/string.go b/tensor/string.go index 93a7a3565e..3322c9d28a 100644 --- a/tensor/string.go +++ b/tensor/string.go @@ -59,7 +59,7 @@ func (tsr *String) IsString() bool { func (tsr *String) AsValues() Values { return tsr } -///////////////////// Strings +/////// Strings func (tsr *String) SetString(val string, i ...int) { tsr.Values[tsr.shape.IndexTo1D(i...)] = val @@ -79,7 +79,20 @@ func (tsr *String) SetStringRow(val string, row, cell int) { tsr.Values[tsr.shape.Header+row*sz+cell] = val } -///////////////////// Floats +// AppendRowString adds a row and sets string value(s), up to number of cells. +func (tsr *String) AppendRowString(val ...string) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow, sz := tsr.shape.RowCellSize() + tsr.SetNumRows(nrow + 1) + mx := min(sz, len(val)) + for i := range mx { + tsr.SetStringRow(val[i], nrow, i) + } +} + +/////// Floats func (tsr *String) Float(i ...int) float64 { return StringToFloat64(tsr.Values[tsr.shape.IndexTo1D(i...)]) @@ -107,7 +120,20 @@ func (tsr *String) SetFloatRow(val float64, row, cell int) { tsr.Values[tsr.shape.Header+row*sz+cell] = Float64ToString(val) } -///////////////////// Ints +// AppendRowFloat adds a row and sets float value(s), up to number of cells. +func (tsr *String) AppendRowFloat(val ...float64) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow, sz := tsr.shape.RowCellSize() + tsr.SetNumRows(nrow + 1) + mx := min(sz, len(val)) + for i := range mx { + tsr.SetFloatRow(val[i], nrow, i) + } +} + +/////// Ints func (tsr *String) Int(i ...int) int { return errors.Ignore1(strconv.Atoi(tsr.Values[tsr.shape.IndexTo1D(i...)])) @@ -135,6 +161,19 @@ func (tsr *String) SetIntRow(val int, row, cell int) { tsr.Values[tsr.shape.Header+row*sz+cell] = strconv.Itoa(val) } +// AppendRowInt adds a row and sets int value(s), up to number of cells. +func (tsr *String) AppendRowInt(val ...int) { + if tsr.NumDims() == 0 { + tsr.SetShapeSizes(0) + } + nrow, sz := tsr.shape.RowCellSize() + tsr.SetNumRows(nrow + 1) + mx := min(sz, len(val)) + for i := range mx { + tsr.SetIntRow(val[i], nrow, i) + } +} + // SetZeros is a simple convenience function initialize all values to the // zero value of the type (empty strings for string type). func (tsr *String) SetZeros() { From 79dc4c565da3e99f9768e00b0fc2d4658b99bc1e Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 09:50:24 -0800 Subject: [PATCH 299/311] better names and cleanup of datafs-sim --- tensor/examples/datafs-sim/sim.go | 113 +++++++++++++--------------- tensor/examples/datafs-sim/sim.goal | 113 +++++++++++++--------------- 2 files changed, 102 insertions(+), 124 deletions(-) diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 9c8b642ced..bc4876596d 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -31,16 +31,11 @@ const ( type LoopPhase int32 //enums:enum const ( - // Start is the start of the loop. + // Start is the start of the loop. Start does initialization, and should be called at Init. Start LoopPhase = iota // Step is at each iteration of the loop. Step - - // End is at the end of the loop, after all iterations. - // This is only called for the outer-most loop, because all others - // are synonymous with Step at the next higher level. - End ) type Sim struct { @@ -57,7 +52,7 @@ type Sim struct { Current *datafs.Data // StatFuncs are statistics functions, per stat, handles everything. - StatFuncs []func(tm Times, lp LoopPhase) + StatFuncs []func(ltime Times, lphase LoopPhase) // Counters are current values of counters: normally in looper. Counters [TimesN]int @@ -78,13 +73,13 @@ func (ss *Sim) ConfigAll() { ss.ConfigStats() } -func (ss *Sim) AddStat(f func(tm Times, lp LoopPhase)) { +func (ss *Sim) AddStat(f func(ltime Times, lphase LoopPhase)) { ss.StatFuncs = append(ss.StatFuncs, f) } -func (ss *Sim) RunStats(tm Times, lp LoopPhase) { +func (ss *Sim) RunStats(ltime Times, lphase LoopPhase) { for _, sf := range ss.StatFuncs { - sf(tm, lp) + sf(ltime, lphase) } } @@ -93,98 +88,92 @@ func (ss *Sim) ConfigStats() { ss.Current, _ = ss.Stats.Mkdir("Current") ctrs := []Times{Run, Epoch, Trial} for _, ctr := range ctrs { - ss.AddStat(func(tm Times, lp LoopPhase) { - if lp == End || tm > ctr { // don't record counter for time above it + ss.AddStat(func(ltime Times, lphase LoopPhase) { + if ltime > ctr { // don't record counter for time above it return } - name := ctr.String() // name of stat = counter - td := ss.Stats.RecycleDir(tm.String()) // log for tm time - tv := datafs.Value[int](td, name) - if lp == Start { - tv.SetNumRows(0) - if ps := plot.GetStylersFrom(tv); ps == nil { + name := ctr.String() // name of stat = counter + timeDir := ss.Stats.RecycleDir(ltime.String()) + tsr := datafs.Value[int](timeDir, name) + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { s.Range.SetMin(0) }) - plot.SetStylersTo(tv, ps) + plot.SetStylersTo(tsr, ps) } return } ctv := ss.Counters[ctr] datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) - tv.AppendRowInt(ctv) + tsr.AppendRowInt(ctv) }) } // note: it is essential to only have 1 per func // so generic names can be used for everything. - ss.AddStat(func(tm Times, lp LoopPhase) { - if lp == End { // only called for Run; we ignore - return - } + ss.AddStat(func(ltime Times, lphase LoopPhase) { name := "SSE" - td := ss.Stats.RecycleDir(tm.String()) - tv := datafs.Value[float64](td, name) - if lp == Start { - tv.SetNumRows(0) - if ps := plot.GetStylersFrom(tv); ps == nil { + timeDir := ss.Stats.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { s.Range.SetMin(0).SetMax(1) s.On = true }) - plot.SetStylersTo(tv, ps) + plot.SetStylersTo(tsr, ps) } return } - switch tm { + switch ltime { case Trial: - sv := rand.Float64() - datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRowFloat(sv) + stat := rand.Float64() + datafs.Scalar[float64](ss.Current, name).SetFloat(stat, 0) + tsr.AppendRowFloat(stat) case Epoch: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) case Run: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) } }) - ss.AddStat(func(tm Times, lp LoopPhase) { - if lp == End { // only called for Run; we ignore - return - } + ss.AddStat(func(ltime Times, lphase LoopPhase) { name := "Err" - td := ss.Stats.RecycleDir(tm.String()) - tv := datafs.Value[float64](td, name) - if lp == Start { - tv.SetNumRows(0) - if ps := plot.GetStylersFrom(tv); ps == nil { + timeDir := ss.Stats.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { s.Range.SetMin(0).SetMax(1) s.On = true }) - plot.SetStylersTo(tv, ps) + plot.SetStylersTo(tsr, ps) } return } - switch tm { + switch ltime { case Trial: sse := ss.Current.Item("SSE").AsFloat64() - sv := 1.0 + stat := 1.0 if sse < 0.5 { - sv = 0 + stat = 0 } - datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRowFloat(sv) + datafs.Scalar[float64](ss.Current, name).SetFloat(stat, 0) + tsr.AppendRowFloat(stat) case Epoch: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) case Run: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) } }) } @@ -209,7 +198,7 @@ func (ss *Sim) Run() { } ss.RunStats(Run, Step) } - ss.RunStats(Run, End) + // todo: could do final analysis here // alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) // dir, _ := ss.Logs.Mkdir("Stats") // stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal index c20574e7c3..f0b3f72d08 100644 --- a/tensor/examples/datafs-sim/sim.goal +++ b/tensor/examples/datafs-sim/sim.goal @@ -29,16 +29,11 @@ const ( type LoopPhase int32 //enums:enum const ( - // Start is the start of the loop. + // Start is the start of the loop. Start does initialization, and should be called at Init. Start LoopPhase = iota // Step is at each iteration of the loop. Step - - // End is at the end of the loop, after all iterations. - // This is only called for the outer-most loop, because all others - // are synonymous with Step at the next higher level. - End ) type Sim struct { @@ -55,7 +50,7 @@ type Sim struct { Current *datafs.Data // StatFuncs are statistics functions, per stat, handles everything. - StatFuncs []func(tm Times, lp LoopPhase) + StatFuncs []func(ltime Times, lphase LoopPhase) // Counters are current values of counters: normally in looper. Counters [TimesN]int @@ -76,13 +71,13 @@ func (ss *Sim) ConfigAll() { ss.ConfigStats() } -func (ss *Sim) AddStat(f func(tm Times, lp LoopPhase)) { +func (ss *Sim) AddStat(f func(ltime Times, lphase LoopPhase)) { ss.StatFuncs = append(ss.StatFuncs, f) } -func (ss *Sim) RunStats(tm Times, lp LoopPhase) { +func (ss *Sim) RunStats(ltime Times, lphase LoopPhase) { for _, sf := range ss.StatFuncs { - sf(tm, lp) + sf(ltime, lphase) } } @@ -91,98 +86,92 @@ func (ss *Sim) ConfigStats() { ss.Current, _ = ss.Stats.Mkdir("Current") ctrs := []Times{Run, Epoch, Trial} for _, ctr := range ctrs { - ss.AddStat(func(tm Times, lp LoopPhase) { - if lp == End || tm > ctr { // don't record counter for time above it + ss.AddStat(func(ltime Times, lphase LoopPhase) { + if ltime > ctr { // don't record counter for time above it return } - name := ctr.String() // name of stat = counter - td := ss.Stats.RecycleDir(tm.String()) // log for tm time - tv := datafs.Value[int](td, name) - if lp == Start { - tv.SetNumRows(0) - if ps := plot.GetStylersFrom(tv); ps == nil { + name := ctr.String() // name of stat = counter + timeDir := ss.Stats.RecycleDir(ltime.String()) + tsr := datafs.Value[int](timeDir, name) + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { s.Range.SetMin(0) }) - plot.SetStylersTo(tv, ps) + plot.SetStylersTo(tsr, ps) } return } ctv := ss.Counters[ctr] datafs.Scalar[int](ss.Current, name).SetInt1D(ctv, 0) - tv.AppendRowInt(ctv) + tsr.AppendRowInt(ctv) }) } // note: it is essential to only have 1 per func // so generic names can be used for everything. - ss.AddStat(func(tm Times, lp LoopPhase) { - if lp == End { // only called for Run; we ignore - return - } + ss.AddStat(func(ltime Times, lphase LoopPhase) { name := "SSE" - td := ss.Stats.RecycleDir(tm.String()) - tv := datafs.Value[float64](td, name) - if lp == Start { - tv.SetNumRows(0) - if ps := plot.GetStylersFrom(tv); ps == nil { + timeDir := ss.Stats.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { s.Range.SetMin(0).SetMax(1) s.On = true }) - plot.SetStylersTo(tv, ps) + plot.SetStylersTo(tsr, ps) } return } - switch tm { + switch ltime { case Trial: - sv := rand.Float64() - datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRowFloat(sv) + stat := rand.Float64() + datafs.Scalar[float64](ss.Current, name).SetFloat(stat, 0) + tsr.AppendRowFloat(stat) case Epoch: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) case Run: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) } }) - ss.AddStat(func(tm Times, lp LoopPhase) { - if lp == End { // only called for Run; we ignore - return - } + ss.AddStat(func(ltime Times, lphase LoopPhase) { name := "Err" - td := ss.Stats.RecycleDir(tm.String()) - tv := datafs.Value[float64](td, name) - if lp == Start { - tv.SetNumRows(0) - if ps := plot.GetStylersFrom(tv); ps == nil { + timeDir := ss.Stats.RecycleDir(ltime.String()) + tsr := datafs.Value[float64](timeDir, name) + if lphase == Start { + tsr.SetNumRows(0) + if ps := plot.GetStylersFrom(tsr); ps == nil { ps.Add(func(s *plot.Style) { s.Range.SetMin(0).SetMax(1) s.On = true }) - plot.SetStylersTo(tv, ps) + plot.SetStylersTo(tsr, ps) } return } - switch tm { + switch ltime { case Trial: sse := ss.Current.Item("SSE").AsFloat64() - sv := 1.0 + stat := 1.0 if sse < 0.5 { - sv = 0 + stat = 0 } - datafs.Scalar[float64](ss.Current, name).SetFloat(sv, 0) - tv.AppendRowFloat(sv) + datafs.Scalar[float64](ss.Current, name).SetFloat(stat, 0) + tsr.AppendRowFloat(stat) case Epoch: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) case Run: - subd := ss.Stats.RecycleDir((tm - 1).String()) - sv := stats.StatMean.Call(subd.Value(name)) - tv.AppendRow(sv) + subd := ss.Stats.RecycleDir((ltime - 1).String()) + stat := stats.StatMean.Call(subd.Value(name)) + tsr.AppendRow(stat) } }) } @@ -207,7 +196,7 @@ func (ss *Sim) Run() { } ss.RunStats(Run, Step) } - ss.RunStats(Run, End) + // todo: could do final analysis here // alldt := ss.Logs.Item("AllTrials").GetDirTable(nil) // dir, _ := ss.Logs.Mkdir("Stats") // stats.TableGroups(dir, alldt, "Run", "Epoch", "Trial") From 9c3259449660c4c35148e2fb1655b40b9965cea8 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 09:54:56 -0800 Subject: [PATCH 300/311] add readme --- tensor/examples/datafs-sim/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 tensor/examples/datafs-sim/README.md diff --git a/tensor/examples/datafs-sim/README.md b/tensor/examples/datafs-sim/README.md new file mode 100644 index 0000000000..125e55c12c --- /dev/null +++ b/tensor/examples/datafs-sim/README.md @@ -0,0 +1,5 @@ +# datafs sim + +This is a prototype for neural network simulation statistics computation using the [datafs](../datafs) framework, now implemented in the [emergent](https://github.com/emer) framework. + + From fa11baa693cef7187f86f7f9a5e3a89eef327121 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 13:37:54 -0800 Subject: [PATCH 301/311] core.tree and filetree: major update: Root is exported and is type Treer so that different types of roots are automatically supported; filetree uses new Root for FileRoot() method (instead of redundant field). Also made external files only show when present, fixes #1232 databrowser now has a more modular design with Tabber interface for tabbing element -- so it can be used in emer sims. --- core/layout.go | 5 +- core/tree.go | 112 ++++++++++-------- core/treesync.go | 10 +- filetree/copypaste.go | 8 +- filetree/file.go | 30 ++--- filetree/find.go | 2 +- filetree/menu.go | 2 +- filetree/node.go | 58 ++++----- filetree/tree.go | 25 +++- filetree/vcs.go | 8 +- tensor/databrowser/README.md | 14 +++ tensor/databrowser/browser.go | 24 ++-- tensor/databrowser/datatab.go | 123 ------------------- tensor/databrowser/filetree.go | 85 +++++++++---- tensor/databrowser/tabs.go | 177 ++++++++++++++++++++++++++++ tensor/databrowser/typegen.go | 21 +++- tensor/examples/datafs-sim/sim.go | 5 +- tensor/examples/datafs-sim/sim.goal | 5 +- 18 files changed, 434 insertions(+), 280 deletions(-) create mode 100644 tensor/databrowser/README.md delete mode 100644 tensor/databrowser/datatab.go create mode 100644 tensor/databrowser/tabs.go diff --git a/core/layout.go b/core/layout.go index 7a0f533da7..45f6f55348 100644 --- a/core/layout.go +++ b/core/layout.go @@ -140,8 +140,8 @@ type Layouter interface { SetScrollParams(d math32.Dims, sb *Slider) } -// AsFrame returns the given value as a value of type [Frame] if the type -// of the given value embeds [Frame], or nil otherwise. +// AsFrame returns the given value as a [Frame] if it has +// an AsFrame() method, or nil otherwise. func AsFrame(n tree.Node) *Frame { if t, ok := n.(Layouter); ok { return t.AsFrame() @@ -149,7 +149,6 @@ func AsFrame(n tree.Node) *Frame { return nil } -// AsFrame satisfies the [Layouter] interface. func (t *Frame) AsFrame() *Frame { return t } diff --git a/core/tree.go b/core/tree.go index 175183a9c6..e8c184c7c1 100644 --- a/core/tree.go +++ b/core/tree.go @@ -60,8 +60,8 @@ type Treer interface { //types:add DropDeleteSource(e events.Event) } -// AsTree returns the given value as a value of type [Tree] if the type -// of the given value embeds [Tree], or nil otherwise. +// AsTree returns the given value as a [Tree] if it has +// an AsCoreTree() method, or nil otherwise. func AsTree(n tree.Node) *Tree { if t, ok := n.(Treer); ok { return t.AsCoreTree() @@ -123,7 +123,7 @@ type Tree struct { // with each child tree node when it is initialized. It is only // called with the root node itself in [Tree.SetTreeInit], so you // should typically call that instead of setting this directly. - TreeInit func(tr *Tree) `set:"-"` + TreeInit func(tr *Tree) `set:"-" json:"-" xml:"-"` // Indent is the amount to indent children relative to this node. // It should be set in a Styler like all other style properties. @@ -151,8 +151,8 @@ type Tree struct { // our alloc includes all of our children, but we only draw us. widgetSize math32.Vector2 - // root is the cached root of the tree. It is automatically set. - root *Tree + // Root is the cached root of the tree. It is automatically set. + Root Treer `copier:"-" json:"-" xml:"-" edit:"-" set:"-"` // SelectedNodes holds the currently selected nodes. // It is only set on the root node. See [Tree.GetSelectedNodes] @@ -186,7 +186,9 @@ func (tr *Tree) rootSetViewIndex() int { tvn := AsTree(cw) if tvn != nil { tvn.viewIndex = idx - tvn.root = tr + if tvn.Root == nil { + tvn.Root = tr + } idx++ } return tree.Continue @@ -480,15 +482,18 @@ func (tr *Tree) OnAdd() { tr.WidgetBase.OnAdd() tr.Text = tr.Name if ptv := AsTree(tr.Parent); ptv != nil { - tr.root = ptv.root + tr.Root = ptv.Root tr.IconOpen = ptv.IconOpen tr.IconClosed = ptv.IconClosed tr.IconLeaf = ptv.IconLeaf } else { - tr.root = tr + if tr.Root == nil { + tr.Root = tr + } } - if tr.root.TreeInit != nil { - tr.root.TreeInit(tr) + troot := tr.Root.AsCoreTree() + if troot.TreeInit != nil { + troot.TreeInit(tr) } } @@ -507,10 +512,10 @@ func (tr *Tree) SetTreeInit(v func(tr *Tree)) *Tree { // which is what controls the functional inactivity of the tree // if individual nodes are ReadOnly that only affects display typically. func (tr *Tree) rootIsReadOnly() bool { - if tr.root == nil { + if tr.Root == nil { return true } - return tr.root.IsReadOnly() + return tr.Root.AsCoreTree().IsReadOnly() } func (tr *Tree) Style() { @@ -549,8 +554,8 @@ func (tr *Tree) SizeUp() { tr.widgetSize = tr.Geom.Size.Actual.Total h := tr.widgetSize.Y w := tr.widgetSize.X - if tr.root.This == tr.This { // do it every time on root - tr.root.rootSetViewIndex() + if tr.IsRoot() { // do it every time on root + tr.rootSetViewIndex() } if !tr.Closed { @@ -583,11 +588,11 @@ func (tr *Tree) SizeDown(iter int) bool { } func (tr *Tree) Position() { - rn := tr.root - if rn == nil { + if tr.Root == nil { slog.Error("core.Tree: RootView is nil", "in node:", tr) return } + rn := tr.Root.AsCoreTree() tr.setBranchState() sz := &tr.Geom.Size sz.Actual.Total.X = rn.Geom.Size.Actual.Total.X - (tr.Geom.Pos.Total.X - rn.Geom.Pos.Total.X) @@ -665,19 +670,20 @@ func (tr *Tree) RenderWidget() { // Trees within the entire tree, using a list maintained // by the root node. func (tr *Tree) GetSelectedNodes() []Treer { - if tr.root == nil { + if tr.Root == nil { return nil } - if len(tr.root.SelectedNodes) == 0 { - return tr.root.SelectedNodes + rn := tr.Root.AsCoreTree() + if len(rn.SelectedNodes) == 0 { + return rn.SelectedNodes } - return tr.root.SelectedNodes + return rn.SelectedNodes } // SetSelectedNodes updates the selected nodes on the root node to the given list. func (tr *Tree) SetSelectedNodes(sl []Treer) { - if tr.root != nil { - tr.root.SelectedNodes = sl + if tr.Root != nil { + tr.Root.AsCoreTree().SelectedNodes = sl } } @@ -743,7 +749,7 @@ func (tr *Tree) SelectAll() { return } tr.UnselectAll() - nn := tr.root + nn := tr.Root.AsCoreTree() nn.Select() for nn != nil { nn = nn.moveDown(events.SelectQuiet) @@ -830,26 +836,27 @@ func (tr *Tree) selectUpdate(mode events.SelectModes) bool { // sendSelectEvent sends an [events.Select] event on both this node and the root node. func (tr *Tree) sendSelectEvent(original ...events.Event) { - if tr.This != tr.root.This { + if !tr.IsRoot() { tr.Send(events.Select, original...) } - tr.root.Send(events.Select, original...) + tr.Root.AsCoreTree().Send(events.Select, original...) } // sendChangeEvent sends an [events.Change] event on both this node and the root node. func (tr *Tree) sendChangeEvent(original ...events.Event) { - if tr.This != tr.root.This { + if !tr.IsRoot() { tr.SendChange(original...) } - tr.root.SendChange(original...) + tr.Root.AsCoreTree().SendChange(original...) } // sendChangeEventReSync sends an [events.Change] event on the RootView node. // If SyncNode != nil, it also does a re-sync from root. func (tr *Tree) sendChangeEventReSync(original ...events.Event) { tr.sendChangeEvent(original...) - if tr.root.SyncNode != nil { - tr.root.Resync() + rn := tr.Root.AsCoreTree() + if rn.SyncNode != nil { + rn.Resync() } } @@ -916,7 +923,7 @@ func (tr *Tree) moveDownSibling(selMode events.SelectModes) *Tree { if tr.Parent == nil { return nil } - if tr == tr.root { + if tr == tr.Root { return nil } myidx := tr.IndexInParent() @@ -936,7 +943,7 @@ func (tr *Tree) moveDownSibling(selMode events.SelectModes) *Tree { // using given select mode (from keyboard modifiers). // Returns newly selected node func (tr *Tree) moveUp(selMode events.SelectModes) *Tree { - if tr.Parent == nil || tr == tr.root { + if tr.Parent == nil || tr == tr.Root { return nil } myidx := tr.IndexInParent() @@ -1038,7 +1045,7 @@ func (tr *Tree) movePageDownEvent(selMode events.SelectModes) *Tree { // moveToLastChild moves to the last child under me, using given select mode // (from keyboard modifiers) func (tr *Tree) moveToLastChild(selMode events.SelectModes) *Tree { - if tr.Parent == nil || tr == tr.root { + if tr.Parent == nil || tr == tr.Root { return nil } if !tr.Closed && tr.HasChildren() { @@ -1054,11 +1061,12 @@ func (tr *Tree) moveToLastChild(selMode events.SelectModes) *Tree { // using given select mode (from keyboard modifiers) // and emits select event for newly selected item func (tr *Tree) moveHomeEvent(selMode events.SelectModes) *Tree { - tr.root.selectUpdate(selMode) - tr.root.SetFocus() - tr.root.ScrollToThis() - tr.root.sendSelectEvent() - return tr.root + rn := tr.Root.AsCoreTree() + rn.selectUpdate(selMode) + rn.SetFocus() + rn.ScrollToThis() + rn.sendSelectEvent() + return rn } // moveEndEvent moves the selection to the very last node in the tree, @@ -1249,10 +1257,10 @@ func (tr *Tree) contextMenu(m *Scene) { // IsRoot returns true if given node is the root of the tree, // creating an error snackbar if it is and action is non-empty. -func (tr *Tree) IsRoot(action string) bool { - if tr.This == tr.root.This { - if action != "" { - MessageSnackbar(tr, fmt.Sprintf("Cannot %v the root of the tree", action)) +func (tr *Tree) IsRoot(action ...string) bool { + if tr.This == tr.Root.AsCoreTree().This { + if len(action) == 1 { + MessageSnackbar(tr, fmt.Sprintf("Cannot %v the root of the tree", action[0])) } return true } @@ -1268,7 +1276,7 @@ func (tr *Tree) MimeData(md *mimedata.Mimes) { tr.mimeDataSync(md) return } - *md = append(*md, mimedata.NewTextData(tr.PathFrom(tr.root))) + *md = append(*md, mimedata.NewTextData(tr.PathFrom(tr.Root.AsCoreTree()))) var buf bytes.Buffer err := jsonx.Write(tr.This, &buf) if err == nil { @@ -1326,13 +1334,13 @@ func (tr *Tree) Cut() { //types:add } tr.Copy() sels := tr.GetSelectedNodes() - root := tr.root + rn := tr.Root.AsCoreTree() tr.UnselectAll() for _, sn := range sels { sn.AsTree().Delete() } - root.Update() - root.sendChangeEvent() + rn.Update() + rn.sendChangeEvent() } // Paste pastes clipboard at given node. @@ -1370,7 +1378,7 @@ func (tr *Tree) makePasteMenu(m *Scene, md mimedata.Mimes, fun func()) { fun() } }) - if !tr.IsRoot("") && tr.root.This != tr.This { + if !tr.IsRoot() { NewButton(m).SetText("Insert Before").OnClick(func(e events.Event) { tr.pasteBefore(md, events.DropCopy) if fun != nil { @@ -1459,10 +1467,10 @@ func (tr *Tree) pasteAt(md mimedata.Mimes, mod events.DropMods, rel int, actNm s parent.InsertChild(ns, myidx+i) nwb := AsWidget(ns) ntv := AsTree(ns) - ntv.root = tr.root + ntv.Root = tr.Root nwb.setScene(tr.Scene) nwb.Update() // incl children - npath := ns.AsTree().PathFrom(tr.root) + npath := ns.AsTree().PathFrom(tr.Root) if mod == events.DropMove && npath == orgpath { // we will be nuked immediately after drag ns.AsTree().SetName(ns.AsTree().Name + treeTempMovedTag) // special keyword :) } @@ -1490,7 +1498,7 @@ func (tr *Tree) pasteChildren(md mimedata.Mimes, mod events.DropMods) { tr.AddChild(ns) nwb := AsWidget(ns) ntv := AsTree(ns) - ntv.root = tr.root + ntv.Root = tr.Root nwb.setScene(tr.Scene) } tr.Update() @@ -1568,17 +1576,17 @@ func (tr *Tree) DropDeleteSource(e events.Event) { return } md := de.Data.(mimedata.Mimes) - root := tr.root + rn := tr.Root.AsCoreTree() for _, d := range md { if d.Type != fileinfo.TextPlain { // link continue } path := string(d.Data) - sn := root.FindPath(path) + sn := rn.FindPath(path) if sn != nil { sn.AsTree().Delete() } - sn = root.FindPath(path + treeTempMovedTag) + sn = rn.FindPath(path + treeTempMovedTag) if sn != nil { psplt := strings.Split(path, "/") orgnm := psplt[len(psplt)-1] diff --git a/core/treesync.go b/core/treesync.go index 1cda830ea7..6eebb22df1 100644 --- a/core/treesync.go +++ b/core/treesync.go @@ -62,13 +62,13 @@ func (tr *Tree) Resync() { func (tr *Tree) syncToSrc(tvIndex *int, init bool, depth int) { sn := tr.SyncNode // root must keep the same name for continuity with surrounding context - if tr != tr.root { + if tr != tr.Root { nm := "tv_" + sn.AsTree().Name tr.SetName(nm) } tr.viewIndex = *tvIndex *tvIndex++ - if init && depth >= tr.root.OpenDepth { + if init && depth >= tr.Root.AsCoreTree().OpenDepth { tr.SetClosed(true) } skids := sn.AsTree().Children @@ -377,7 +377,7 @@ func (tr *Tree) inspectNode() { //types:add // mimeDataSync adds mimedata for this node: a text/plain of the Path, // and an application/json of the sync node. func (tr *Tree) mimeDataSync(md *mimedata.Mimes) { - sroot := tr.root.SyncNode + sroot := tr.Root.AsCoreTree().SyncNode src := tr.SyncNode *md = append(*md, mimedata.NewTextData(src.AsTree().PathFrom(sroot))) var buf bytes.Buffer @@ -435,7 +435,7 @@ func (tr *Tree) pasteAtSync(md mimedata.Mimes, mod events.DropMods, rel int, act return } myidx += rel - sroot := tr.root.SyncNode + sroot := tr.Root.AsCoreTree().SyncNode sz := len(sl) var seln tree.Node for i, ns := range sl { @@ -488,7 +488,7 @@ func (tr *Tree) cutSync() { // dropDeleteSourceSync handles delete source event for DropMove case, for Sync func (tr *Tree) dropDeleteSourceSync(de *events.DragDrop) { md := de.Data.(mimedata.Mimes) - sroot := tr.root.SyncNode + sroot := tr.Root.AsCoreTree().SyncNode for _, d := range md { if d.Type != fileinfo.TextPlain { // link continue diff --git a/filetree/copypaste.go b/filetree/copypaste.go index f75dd7ae6d..9c7244a74c 100644 --- a/filetree/copypaste.go +++ b/filetree/copypaste.go @@ -24,7 +24,7 @@ import ( // MimeData adds mimedata for this node: a text/plain of the Path, // text/plain of filename, and text/ func (fn *Node) MimeData(md *mimedata.Mimes) { - froot := fn.FileRoot + froot := fn.FileRoot() path := string(fn.Filepath) punq := fn.PathFrom(froot) // note: tree paths have . escaped -> \, *md = append(*md, mimedata.NewTextData(punq)) @@ -77,7 +77,7 @@ func (fn *Node) DragDrop(e events.Event) { // that is non-nil (otherwise just uses absolute path), and returns list of existing // and node for last one if exists. func (fn *Node) pasteCheckExisting(tfn *Node, md mimedata.Mimes, externalDrop bool) ([]string, *Node) { - froot := fn.FileRoot + froot := fn.FileRoot() tpath := "" if tfn != nil { tpath = string(tfn.Filepath) @@ -118,7 +118,7 @@ func (fn *Node) pasteCheckExisting(tfn *Node, md mimedata.Mimes, externalDrop bo // pasteCopyFiles copies files in given data into given target directory func (fn *Node) pasteCopyFiles(tdir *Node, md mimedata.Mimes, externalDrop bool) { - froot := fn.FileRoot + froot := fn.FileRoot() nf := len(md) if !externalDrop { nf /= 3 @@ -283,7 +283,7 @@ func (fn *Node) pasteFiles(md mimedata.Mimes, externalDrop bool, dropFinal func( // satisfies core.DragNDropper interface and can be overridden by subtypes func (fn *Node) DropDeleteSource(e events.Event) { de := e.(*events.DragDrop) - froot := fn.FileRoot + froot := fn.FileRoot() if froot == nil || fn.isExternal() { return } diff --git a/filetree/file.go b/filetree/file.go index 813c731b36..dc6082888a 100644 --- a/filetree/file.go +++ b/filetree/file.go @@ -59,7 +59,7 @@ func (fn *Node) OpenFileDefault() error { // duplicateFiles makes a copy of selected files func (fn *Node) duplicateFiles() { //types:add - fn.FileRoot.NeedsLayout() + fn.FileRoot().NeedsLayout() fn.SelectedFunc(func(sn *Node) { sn.duplicateFile() }) @@ -92,7 +92,7 @@ func (fn *Node) deleteFiles() { //types:add // deleteFilesImpl does the actual deletion, no prompts func (fn *Node) deleteFilesImpl() { - fn.FileRoot.NeedsLayout() + fn.FileRoot().NeedsLayout() fn.SelectedFunc(func(sn *Node) { if !sn.Info.IsDir() { sn.deleteFile() @@ -100,7 +100,7 @@ func (fn *Node) deleteFilesImpl() { } var fns []string sn.Info.Filenames(&fns) - ft := sn.FileRoot + ft := sn.FileRoot() for _, filename := range fns { sn, ok := ft.FindFile(filename) if !ok { @@ -145,7 +145,7 @@ func (fn *Node) deleteFile() error { // renames any selected files func (fn *Node) RenameFiles() { //types:add - fn.FileRoot.NeedsLayout() + fn.FileRoot().NeedsLayout() fn.SelectedFunc(func(sn *Node) { fb := core.NewSoloFuncButton(sn).SetFunc(sn.RenameFile) fb.Args[0].SetValue(sn.Name) @@ -158,7 +158,7 @@ func (fn *Node) RenameFile(newpath string) error { //types:add if fn.isExternal() { return nil } - root := fn.FileRoot + root := fn.FileRoot() var err error fn.closeBuf() // invalid after this point orgpath := fn.Filepath @@ -167,8 +167,8 @@ func (fn *Node) RenameFile(newpath string) error { //types:add return err } if fn.IsDir() { - if fn.FileRoot.isDirOpen(orgpath) { - fn.FileRoot.setDirOpen(core.Filename(newpath)) + if fn.FileRoot().isDirOpen(orgpath) { + fn.FileRoot().setDirOpen(core.Filename(newpath)) } } repo, _ := fn.Repo() @@ -230,15 +230,15 @@ func (fn *Node) newFile(filename string, addToVCS bool) { //types:add return } if addToVCS { - nfn, ok := fn.FileRoot.FindFile(np) - if ok && nfn.This != fn.FileRoot.This && string(nfn.Filepath) == np { + nfn, ok := fn.FileRoot().FindFile(np) + if ok && !nfn.IsRoot() && string(nfn.Filepath) == np { // todo: this is where it is erroneously adding too many files to vcs! fmt.Println("Adding new file to VCS:", nfn.Filepath) core.MessageSnackbar(fn, "Adding new file to VCS: "+fsx.DirAndFile(string(nfn.Filepath))) nfn.AddToVCS() } } - fn.FileRoot.UpdatePath(np) + fn.FileRoot().UpdatePath(np) } // makes a new folder in the given selected directory @@ -267,7 +267,7 @@ func (fn *Node) newFolder(foldername string) { //types:add core.ErrorSnackbar(fn, err) return } - fn.FileRoot.UpdatePath(ppath) + fn.FileRoot().UpdatePath(ppath) } // copyFileToDir copies given file path into node that is a directory. @@ -280,11 +280,11 @@ func (fn *Node) copyFileToDir(filename string, perm os.FileMode) { sfn := filepath.Base(filename) tpath := filepath.Join(ppath, sfn) fileinfo.CopyFile(tpath, filename, perm) - fn.FileRoot.UpdatePath(ppath) - ofn, ok := fn.FileRoot.FindFile(filename) + fn.FileRoot().UpdatePath(ppath) + ofn, ok := fn.FileRoot().FindFile(filename) if ok && ofn.Info.VCS >= vcs.Stored { - nfn, ok := fn.FileRoot.FindFile(tpath) - if ok && nfn.This != fn.FileRoot.This { + nfn, ok := fn.FileRoot().FindFile(tpath) + if ok && !nfn.IsRoot() { if string(nfn.Filepath) != tpath { fmt.Printf("error: nfn.FPath != tpath; %q != %q, see bug #453\n", nfn.Filepath, tpath) } else { diff --git a/filetree/find.go b/filetree/find.go index b48b8a912f..7ecac4c304 100644 --- a/filetree/find.go +++ b/filetree/find.go @@ -62,7 +62,7 @@ func (fn *Node) FindFile(fnm string) (*Node, bool) { } } - if efn, err := fn.FileRoot.externalNodeByPath(fnm); err == nil { + if efn, err := fn.FileRoot().externalNodeByPath(fnm); err == nil { return efn, true } diff --git a/filetree/menu.go b/filetree/menu.go index 0f30d373b1..f3d5dd3a4d 100644 --- a/filetree/menu.go +++ b/filetree/menu.go @@ -26,7 +26,7 @@ func vcsLabelFunc(fn *Node, label string) string { } func (fn *Node) VCSContextMenu(m *core.Scene) { - if fn.FileRoot.FS != nil { + if fn.FileRoot().FS != nil { return } core.NewFuncButton(m).SetFunc(fn.addToVCSSelected).SetText(vcsLabelFunc(fn, "Add to VCS")).SetIcon(icons.Add). diff --git a/filetree/node.go b/filetree/node.go index f164b474be..5807d95999 100644 --- a/filetree/node.go +++ b/filetree/node.go @@ -53,9 +53,6 @@ type Node struct { //core:embedder // Buffer is the file buffer for editing this file. Buffer *texteditor.Buffer `edit:"-" set:"-" json:"-" xml:"-" copier:"-"` - // FileRoot is the root [Tree] of the tree, which has global state. - FileRoot *Tree `edit:"-" set:"-" json:"-" xml:"-" copier:"-"` - // DirRepo is the version control system repository for this directory, // only non-nil if this is the highest-level directory in the tree under vcs control. DirRepo vcs.Repo `edit:"-" set:"-" json:"-" xml:"-" copier:"-"` @@ -70,6 +67,11 @@ func (fn *Node) AsFileNode() *Node { return fn } +// FileRoot returns the Root node as a [Tree]. +func (fn *Node) FileRoot() *Tree { + return AsTree(fn.Root) +} + func (fn *Node) Init() { fn.Tree.Init() fn.IconOpen = icons.FolderOpen @@ -192,14 +194,14 @@ func (fn *Node) Init() { return } if fn.Name == externalFilesName { - files := fn.FileRoot.externalFiles + files := fn.FileRoot().externalFiles for _, fi := range files { tree.AddNew(p, fi, func() Filer { - return tree.NewOfType(fn.FileRoot.FileNodeType).(Filer) + return tree.NewOfType(fn.FileRoot().FileNodeType).(Filer) }, func(wf Filer) { w := wf.AsFileNode() + w.Root = fn.Root w.NeedsLayout() - w.FileRoot = fn.FileRoot w.Filepath = core.Filename(fi) w.Info.Mode = os.ModeIrregular w.Info.VCS = vcs.Stored @@ -210,25 +212,25 @@ func (fn *Node) Init() { if !fn.IsDir() || fn.IsIrregular() { return } - if !((fn.FileRoot.inOpenAll && !fn.Info.IsHidden()) || fn.FileRoot.isDirOpen(fn.Filepath)) { + if !((fn.FileRoot().inOpenAll && !fn.Info.IsHidden()) || fn.FileRoot().isDirOpen(fn.Filepath)) { return } repo, _ := fn.Repo() files := fn.dirFileList() for _, fi := range files { fpath := filepath.Join(string(fn.Filepath), fi.Name()) - if fn.FileRoot.FilterFunc != nil && !fn.FileRoot.FilterFunc(fpath, fi) { + if fn.FileRoot().FilterFunc != nil && !fn.FileRoot().FilterFunc(fpath, fi) { continue } tree.AddNew(p, fi.Name(), func() Filer { - return tree.NewOfType(fn.FileRoot.FileNodeType).(Filer) + return tree.NewOfType(fn.FileRoot().FileNodeType).(Filer) }, func(wf Filer) { w := wf.AsFileNode() + w.Root = fn.Root w.NeedsLayout() - w.FileRoot = fn.FileRoot w.Filepath = core.Filename(fpath) w.This.(Filer).GetFileInfo() - if w.FileRoot.FS == nil { + if w.FileRoot().FS == nil { if w.IsDir() && repo == nil { w.detectVCSRepo(true) // update files } @@ -287,10 +289,10 @@ func (fn *Node) isAutoSave() bool { // RelativePath returns the relative path from root for this node func (fn *Node) RelativePath() string { - if fn.IsIrregular() || fn.FileRoot == nil { + if fn.IsIrregular() || fn.FileRoot() == nil { return fn.Name } - return fsx.RelativeFilePath(string(fn.Filepath), string(fn.FileRoot.Filepath)) + return fsx.RelativeFilePath(string(fn.Filepath), string(fn.FileRoot().Filepath)) } // dirFileList returns the list of files in this directory, @@ -301,15 +303,15 @@ func (fn *Node) dirFileList() []fs.FileInfo { var dirs []fs.FileInfo // for DirsOnTop mode var di []fs.DirEntry isFS := false - if fn.FileRoot.FS == nil { + if fn.FileRoot().FS == nil { di = errors.Log1(os.ReadDir(path)) } else { isFS = true - di = errors.Log1(fs.ReadDir(fn.FileRoot.FS, path)) + di = errors.Log1(fs.ReadDir(fn.FileRoot().FS, path)) } for _, d := range di { info := errors.Log1(d.Info()) - if fn.FileRoot.DirsOnTop { + if fn.FileRoot().DirsOnTop { if d.IsDir() { dirs = append(dirs, info) } else { @@ -319,14 +321,14 @@ func (fn *Node) dirFileList() []fs.FileInfo { files = append(files, info) } } - doModSort := fn.FileRoot.SortByModTime + doModSort := fn.FileRoot().SortByModTime if doModSort { - doModSort = !fn.FileRoot.dirSortByName(core.Filename(path)) + doModSort = !fn.FileRoot().dirSortByName(core.Filename(path)) } else { - doModSort = fn.FileRoot.dirSortByModTime(core.Filename(path)) + doModSort = fn.FileRoot().dirSortByModTime(core.Filename(path)) } - if fn.FileRoot.DirsOnTop { + if fn.FileRoot().DirsOnTop { if doModSort { sortByModTime(dirs, isFS) // note: FS = ascending, otherwise descending sortByModTime(files, isFS) @@ -381,7 +383,7 @@ func (fn *Node) InitFileInfo() error { return nil } var err error - if fn.FileRoot.FS == nil { // deal with symlinks + if fn.FileRoot().FS == nil { // deal with symlinks ls, err := os.Lstat(string(fn.Filepath)) if errors.Log(err) != nil { return err @@ -397,7 +399,7 @@ func (fn *Node) InitFileInfo() error { } err = fn.Info.InitFile(string(fn.Filepath)) } else { - err = fn.Info.InitFileFS(fn.FileRoot.FS, string(fn.Filepath)) + err = fn.Info.InitFileFS(fn.FileRoot().FS, string(fn.Filepath)) } if err != nil { emsg := fmt.Errorf("filetree.Node InitFileInfo Path %q: Error: %v", fn.Filepath, err) @@ -441,7 +443,7 @@ func (fn *Node) OnClose() { if !fn.IsDir() { return } - fn.FileRoot.setDirClosed(fn.Filepath) + fn.FileRoot().setDirClosed(fn.Filepath) } func (fn *Node) CanOpen() bool { @@ -453,7 +455,7 @@ func (fn *Node) openDir() { if !fn.IsDir() { return } - fn.FileRoot.setDirOpen(fn.Filepath) + fn.FileRoot().setDirOpen(fn.Filepath) fn.Update() } @@ -468,15 +470,15 @@ func (fn *Node) sortBys(modTime bool) { //types:add // sortBy determines how to sort the files in the directory -- default is alpha by name, // optionally can be sorted by modification time. func (fn *Node) sortBy(modTime bool) { - fn.FileRoot.setDirSortBy(fn.Filepath, modTime) + fn.FileRoot().setDirSortBy(fn.Filepath, modTime) fn.Update() } // openAll opens all directories under this one func (fn *Node) openAll() { //types:add - fn.FileRoot.inOpenAll = true // causes chaining of opening + fn.FileRoot().inOpenAll = true // causes chaining of opening fn.Tree.OpenAll() - fn.FileRoot.inOpenAll = false + fn.FileRoot().inOpenAll = false } // OpenBuf opens the file in its buffer if it is not already open. @@ -509,7 +511,7 @@ func (fn *Node) removeFromExterns() { //types:add if !sn.isExternal() { return } - sn.FileRoot.removeExternalFile(string(sn.Filepath)) + sn.FileRoot().removeExternalFile(string(sn.Filepath)) sn.closeBuf() sn.Delete() }) diff --git a/filetree/tree.go b/filetree/tree.go index f0d561782c..83e7356256 100644 --- a/filetree/tree.go +++ b/filetree/tree.go @@ -26,6 +26,20 @@ const ( externalFilesName = "[external files]" ) +// Treer is an interface for getting the Root node as a Tree struct. +type Treer interface { + AsFileTree() *Tree +} + +// AsTree returns the given value as a [Tree] if it has +// an AsFileTree() method, or nil otherwise. +func AsTree(n tree.Node) *Tree { + if t, ok := n.(Treer); ok { + return t.AsFileTree() + } + return nil +} + // Tree is the root widget of a file tree representing files in a given directory // (and subdirectories thereof), and has some overall management state for how to // view things. @@ -80,16 +94,19 @@ type Tree struct { func (ft *Tree) Init() { ft.Node.Init() - ft.FileRoot = ft + ft.Root = ft ft.FileNodeType = types.For[Node]() ft.OpenDepth = 4 ft.DirsOnTop = true ft.FirstMaker(func(p *tree.Plan) { + if len(ft.externalFiles) == 0 { + return + } tree.AddNew(p, externalFilesName, func() Filer { return tree.NewOfType(ft.FileNodeType).(Filer) }, func(wf Filer) { w := wf.AsFileNode() - w.FileRoot = ft + w.Root = ft.Root w.Filepath = externalFilesName w.Info.Mode = os.ModeDir w.Info.VCS = vcs.Stored @@ -110,6 +127,10 @@ func (fv *Tree) Destroy() { fv.Tree.Destroy() } +func (ft *Tree) AsFileTree() *Tree { + return ft +} + // OpenPath opens the filetree at the given os file system directory path. // It reads all the files at the given path into this tree. // Only paths listed in [Tree.Dirs] will be opened. diff --git a/filetree/vcs.go b/filetree/vcs.go index 6e9b1fc5cb..db9a72e938 100644 --- a/filetree/vcs.go +++ b/filetree/vcs.go @@ -22,7 +22,7 @@ import ( // FirstVCS returns the first VCS repository starting from this node and going down. // also returns the node having that repository func (fn *Node) FirstVCS() (vcs.Repo, *Node) { - if fn.FileRoot.FS != nil { + if fn.FileRoot().FS != nil { return nil, nil } var repo vcs.Repo @@ -73,7 +73,11 @@ func (fn *Node) detectVCSRepo(updateFiles bool) bool { // and the node for the directory where the repo is based. // Goes up the tree until a repository is found. func (fn *Node) Repo() (vcs.Repo, *Node) { - if fn.isExternal() || fn.FileRoot.FS != nil { + fr := fn.FileRoot() + if fr == nil { + return nil, nil + } + if fn.isExternal() || fr == nil || fr.FS != nil { return nil, nil } if fn.DirRepo != nil { diff --git a/tensor/databrowser/README.md b/tensor/databrowser/README.md new file mode 100644 index 0000000000..dfcf6ccb6a --- /dev/null +++ b/tensor/databrowser/README.md @@ -0,0 +1,14 @@ +# databrowser + +The databrowser package provides GUI elements for data exploration and visualization, and a simple `Browser` implementation that combines these elements. + +* `FileTree` (with `FileNode` elements), implementing a [filetree](https://github.com/cogentcore/tree/main/filetree) that has support for a [datafs](../datafs) filesystem, and data files in an actual filesystem. It has a `Tabber` pointer that handles the viewing actions on `datafs` elements (showing a Plot, etc). + +* `Tabber` interface and `Tabs` base implementation provides methods for showing data plots and editors in tabs. + +* `Terminal` running a `goal` shell that supports interactive commands operating on the `datafs` data etc. TODO! + +The basic `Browser` puts the `FileTree` in a left `Splits` and the `Tabs` in the right, and supports interactive exploration and visualization of data. + +In the [emergent](https://github.com/emer) framework, these elements are combined with other GUI elements to provide a full neural network simulation environment on top of the databrowser foundation. + diff --git a/tensor/databrowser/browser.go b/tensor/databrowser/browser.go index a1103eefc3..2cef08aaa1 100644 --- a/tensor/databrowser/browser.go +++ b/tensor/databrowser/browser.go @@ -16,7 +16,6 @@ import ( "cogentcore.org/core/base/fsx" "cogentcore.org/core/core" "cogentcore.org/core/events" - "cogentcore.org/core/filetree" "cogentcore.org/core/goal/interpreter" "cogentcore.org/core/icons" "cogentcore.org/core/styles" @@ -39,7 +38,7 @@ var TheBrowser *Browser type Browser struct { core.Frame - // FS is the filesystem, if browsing an FS + // FS is the filesystem, if browsing an FS. FS fs.FS // DataRoot is the path to the root of the data to browse. @@ -60,8 +59,8 @@ type Browser struct { toolbar *core.Toolbar splits *core.Splits - files *filetree.Tree - tabs *core.Tabs + files *DataTree + tabs *Tabs } // Init initializes with the data and script directories @@ -85,23 +84,22 @@ func (br *Browser) Init() { s.Overflow.Set(styles.OverflowAuto) s.Grow.Set(1, 1) }) - tree.AddChildAt(w, "filetree", func(w *filetree.Tree) { + tree.AddChildAt(w, "filetree", func(w *DataTree) { br.files = w w.FileNodeType = types.For[FileNode]() - // w.OnSelect(func(e events.Event) { - // e.SetHandled() - // sels := w.SelectedViews() - // if sels != nil { - // br.FileNodeSelected(sn) - // } - // }) }) }) - tree.AddChildAt(w, "tabs", func(w *core.Tabs) { + tree.AddChildAt(w, "tabs", func(w *Tabs) { br.tabs = w w.Type = core.FunctionalTabs }) }) + br.Updater(func() { + if br.files != nil { + br.files.Tabber = br.tabs + } + }) + } // NewBrowserWindow opens a new data Browser for given diff --git a/tensor/databrowser/datatab.go b/tensor/databrowser/datatab.go deleted file mode 100644 index 53b430b9b8..0000000000 --- a/tensor/databrowser/datatab.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (c) 2024, Cogent Core. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package databrowser - -import ( - "cogentcore.org/core/core" - "cogentcore.org/core/plot/plotcore" - "cogentcore.org/core/styles" - "cogentcore.org/core/tensor" - "cogentcore.org/core/tensor/table" - "cogentcore.org/core/tensor/tensorcore" - "cogentcore.org/core/texteditor" -) - -// NewTab creates a tab with given label, or returns the existing one -// with given type of widget within it. mkfun function is called to create -// and configure a new widget if not already existing. -func NewTab[T any](br *Browser, label string, mkfun func(tab *core.Frame) T) T { - tab := br.tabs.RecycleTab(label) - if tab.HasChildren() { - return tab.Child(1).(T) - } - w := mkfun(tab) - return w -} - -// NewTabTensorTable creates a tab with a tensorcore.Table widget -// to view given table.Table, using its own table.Table as tv.Table. -// Use tv.Table.Table to get the underlying *table.Table -// Use tv.Table.Sequential to update the Indexed to view -// all of the rows when done updating the Table, and then call br.Update() -func (br *Browser) NewTabTensorTable(label string, dt *table.Table) *tensorcore.Table { - tv := NewTab[*tensorcore.Table](br, label, func(tab *core.Frame) *tensorcore.Table { - tb := core.NewToolbar(tab) - tv := tensorcore.NewTable(tab) - tb.Maker(tv.MakeToolbar) - return tv - }) - tv.SetTable(dt) - br.Update() - return tv -} - -// NewTabTensorEditor creates a tab with a tensorcore.TensorEditor widget -// to view given Tensor. -func (br *Browser) NewTabTensorEditor(label string, tsr tensor.Tensor) *tensorcore.TensorEditor { - tv := NewTab[*tensorcore.TensorEditor](br, label, func(tab *core.Frame) *tensorcore.TensorEditor { - tb := core.NewToolbar(tab) - tv := tensorcore.NewTensorEditor(tab) - tb.Maker(tv.MakeToolbar) - return tv - }) - tv.SetTensor(tsr) - br.Update() - return tv -} - -// NewTabTensorGrid creates a tab with a tensorcore.TensorGrid widget -// to view given Tensor. -func (br *Browser) NewTabTensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGrid { - tv := NewTab[*tensorcore.TensorGrid](br, label, func(tab *core.Frame) *tensorcore.TensorGrid { - // tb := core.NewToolbar(tab) - tv := tensorcore.NewTensorGrid(tab) - // tb.Maker(tv.MakeToolbar) - return tv - }) - tv.SetTensor(tsr) - br.Update() - return tv -} - -// NewTabPlot creates a tab with a Plot of given table.Table. -func (br *Browser) NewTabPlot(label string, dt *table.Table) *plotcore.PlotEditor { - pl := NewTab[*plotcore.PlotEditor](br, label, func(tab *core.Frame) *plotcore.PlotEditor { - return plotcore.NewSubPlot(tab) - }) - pl.SetTable(dt) - br.Update() - return pl -} - -// NewTabSliceTable creates a tab with a core.Table widget -// to view the given slice of structs. -func (br *Browser) NewTabSliceTable(label string, slc any) *core.Table { - tv := NewTab[*core.Table](br, label, func(tab *core.Frame) *core.Table { - return core.NewTable(tab) - }) - tv.SetSlice(slc) - br.Update() - return tv -} - -// NewTabEditor opens a texteditor.Editor tab, displaying given string. -func (br *Browser) NewTabEditor(label, content string) *texteditor.Editor { - ed := NewTab[*texteditor.Editor](br, label, func(tab *core.Frame) *texteditor.Editor { - ed := texteditor.NewEditor(tab) - ed.Styler(func(s *styles.Style) { - s.Grow.Set(1, 1) - }) - return ed - }) - if content != "" { - ed.Buffer.SetText([]byte(content)) - } - br.Update() - return ed -} - -// NewTabEditorFile opens an editor tab for given file -func (br *Browser) NewTabEditorFile(label, filename string) *texteditor.Editor { - ed := NewTab[*texteditor.Editor](br, label, func(tab *core.Frame) *texteditor.Editor { - ed := texteditor.NewEditor(tab) - ed.Styler(func(s *styles.Style) { - s.Grow.Set(1, 1) - }) - return ed - }) - ed.Buffer.Open(core.Filename(filename)) - br.Update() - return ed -} diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index 55784b745c..b3f017e2f2 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -21,8 +21,41 @@ import ( "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" "cogentcore.org/core/texteditor/diffbrowser" + "cogentcore.org/core/tree" ) +// Treer is an interface for getting the Root node as a DataTree struct. +type Treer interface { + AsDataTree() *DataTree +} + +// AsDataTree returns the given value as a [DataTree] if it has +// an AsDataTree() method, or nil otherwise. +func AsDataTree(n tree.Node) *DataTree { + if t, ok := n.(Treer); ok { + return t.AsDataTree() + } + return nil +} + +// DataTree is the databrowser version of [filetree.Tree], +// which provides the Tabber to show data editors. +type DataTree struct { + filetree.Tree + + // Tabber is the [Tabber] for this tree. + Tabber Tabber +} + +func (ft *DataTree) AsDataTree() *DataTree { + return ft +} + +func (ft *DataTree) Init() { + ft.Tree.Init() + ft.Root = ft +} + // FileNode is databrowser version of FileNode for FileTree type FileNode struct { filetree.Node @@ -33,6 +66,15 @@ func (fn *FileNode) Init() { fn.AddContextMenu(fn.ContextMenu) } +// Tabber returns the [Tabber] for this filenode, from root tree. +func (fn *FileNode) Tabber() Tabber { + fr := AsDataTree(fn.Root) + if fr != nil { + return fr.Tabber + } + return nil +} + func (fn *FileNode) WidgetTooltip(pos image.Point) (string, image.Point) { res := fn.Tooltip if fn.Info.Cat == fileinfo.Data { @@ -53,7 +95,7 @@ func (fn *FileNode) WidgetTooltip(pos image.Point) (string, image.Point) { // DataFS returns the datafs representation of this item. // returns nil if not a dataFS item. func DataFS(fn *filetree.Node) *datafs.Data { - dfs, ok := fn.FileRoot.FS.(*datafs.Data) + dfs, ok := fn.FileRoot().FS.(*datafs.Data) if !ok { return nil } @@ -66,7 +108,7 @@ func DataFS(fn *filetree.Node) *datafs.Data { func (fn *FileNode) GetFileInfo() error { err := fn.InitFileInfo() - if fn.FileRoot.FS == nil { + if fn.FileRoot().FS == nil { return err } d := DataFS(fn.AsNode()) @@ -91,8 +133,8 @@ func (fn *FileNode) GetFileInfo() error { func (fn *FileNode) OpenFile() error { ofn := fn.AsNode() - br := ParentBrowser(fn.This) - if br == nil { + ts := fn.Tabber() + if ts == nil { return nil } df := fsx.DirAndFile(string(fn.Filepath)) @@ -100,18 +142,12 @@ func (fn *FileNode) OpenFile() error { case fn.IsDir(): d := DataFS(ofn) dt := d.GetDirTable(nil) - br.NewTabTensorTable(df, dt) - br.Update() + ts.TensorTable(df, dt) case fn.Info.Cat == fileinfo.Data: switch fn.Info.Known { case fileinfo.Tensor: d := DataFS(ofn) - br.NewTabTensorEditor(df, d.Data) - // case fileinfo.Table: - // d := DataFS(ofn) - // dt := d.AsTable() - // br.NewTabTensorTable(df, dt) - // br.Update() + ts.TensorEditor(df, d.Data) case fileinfo.Number: dv := DataFS(ofn) v := dv.AsFloat32() @@ -124,7 +160,7 @@ func (fn *FileNode) OpenFile() error { dv.SetFloat32(sp.Value) }) }) - d.RunDialog(br) + d.RunDialog(fn) case fileinfo.String: dv := DataFS(ofn) v := dv.AsString() @@ -137,15 +173,15 @@ func (fn *FileNode) OpenFile() error { dv.SetString(tf.Text()) }) }) - d.RunDialog(br) + d.RunDialog(fn) default: dt := table.New() err := dt.OpenCSV(fsx.Filename(fn.Filepath), tensor.Tab) // todo: need more flexible data handling mode if err != nil { - core.ErrorSnackbar(br, err) + core.ErrorSnackbar(fn, err) } else { - br.NewTabTensorTable(df, dt) + ts.TensorTable(df, dt) } } case fn.IsExec(): // todo: use exec? @@ -165,7 +201,7 @@ func (fn *FileNode) OpenFile() error { case fn.Info.Cat == fileinfo.Archive || fn.Info.Cat == fileinfo.Backup: // don't edit fn.OpenFilesDefault() default: - br.NewTabEditor(df, string(fn.Filepath)) + ts.EditorString(df, string(fn.Filepath)) } return nil } @@ -183,8 +219,8 @@ func (fn *FileNode) EditFile() { fn.OpenFile() return } - br := ParentBrowser(fn.This) - if br == nil { + ts := fn.Tabber() + if ts == nil { return } if fn.Info.Cat == fileinfo.Data { @@ -192,7 +228,7 @@ func (fn *FileNode) EditFile() { return } df := fsx.DirAndFile(string(fn.Filepath)) - br.NewTabEditor(df, string(fn.Filepath)) + ts.EditorString(df, string(fn.Filepath)) } // PlotFiles calls PlotFile on selected files @@ -206,8 +242,8 @@ func (fn *FileNode) PlotFiles() { //types:add // PlotFile pulls up this file in a texteditor. func (fn *FileNode) PlotFile() { - br := ParentBrowser(fn.This) - if br == nil { + ts := fn.Tabber() + if ts == nil { return } d := DataFS(fn.AsNode()) @@ -237,7 +273,7 @@ func (fn *FileNode) PlotFile() { dt = table.New(df) err := dt.OpenCSV(fsx.Filename(fn.Filepath), tensor.Tab) // todo: need more flexible data handling mode if err != nil { - core.ErrorSnackbar(br, err) + core.ErrorSnackbar(fn, err) dt = nil } } @@ -245,11 +281,10 @@ func (fn *FileNode) PlotFile() { if dt == nil { return } - pl := br.NewTabPlot(ptab, dt) + pl := ts.PlotTable(ptab, dt) _ = pl // pl.Options.Title = df // TODO: apply column and plot level options. - br.Update() } // DiffDirs displays a browser with differences between two selected directories diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go new file mode 100644 index 0000000000..4d8d77a92e --- /dev/null +++ b/tensor/databrowser/tabs.go @@ -0,0 +1,177 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package databrowser + +import ( + "cogentcore.org/core/core" + "cogentcore.org/core/plot/plotcore" + "cogentcore.org/core/styles" + "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/table" + "cogentcore.org/core/tensor/tensorcore" + "cogentcore.org/core/texteditor" +) + +// Tabber is a [core.Tabs] based widget that has support for opening +// tabs for [plotcore.PlotEditor] and [tensorcore.Table] editors, +// among others. +type Tabber interface { + + // AsTabs returns the underlying [Tabs] widget. + AsTabs() *Tabs + + // RecycleTab returns a tab with the given name, first by looking for an existing one, + // and if not found, making a new one. It returns the frame for the tab. + RecycleTab(name string) *core.Frame + + // TensorTable recycles a tab with a [tensorcore.Table] widget + // to view given [table.Table], using its own table.Table. + TensorTable(label string, dt *table.Table) *tensorcore.Table + + // TensorEditor recycles a tab with a [tensorcore.TensorEditor] widget + // to view given Tensor. + TensorEditor(label string, tsr tensor.Tensor) *tensorcore.TensorEditor + + // TensorGrid recycles a tab with a [tensorcore.TensorGrid] widget + // to view given Tensor. + TensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGrid + + // PlotTable recycles a tab with a Plot of given [table.Table]. + PlotTable(label string, dt *table.Table) *plotcore.PlotEditor + + // todo: PlotData of plot.Data + + // SliceTable recycles a tab with a [core.Table] widget + // to view the given slice of structs. + SliceTable(label string, slc any) *core.Table + + // EditorString recycles a [texteditor.Editor] tab, displaying given string. + EditorString(label, content string) *texteditor.Editor + + // EditorFile opens an editor tab for given file. + EditorFile(label, filename string) *texteditor.Editor +} + +// NewTab recycles a tab with given label, or returns the existing one +// with given type of widget within it. mkfun function is called to create +// and configure a new widget if not already existing. +func NewTab[T any](tb Tabber, label string, mkfun func(tab *core.Frame) T) T { + tab := tb.RecycleTab(label) + var zv T + if tab.HasChildren() { + if tt, ok := tab.Child(1).(T); ok { + return tt + } + core.MessageSnackbar(tb.AsTabs(), "Name / Type conflict: tab "+label+" does not have the expected type of content") + return zv + } + w := mkfun(tab) + return w +} + +// Tabs implements the [Tabber] interface. +type Tabs struct { + core.Tabs +} + +func (ts *Tabs) AsTabs() *Tabs { + return ts +} + +// TensorTable recycles a tab with a tensorcore.Table widget +// to view given table.Table, using its own table.Table as tv.Table. +// Use tv.Table.Table to get the underlying *table.Table +// Use tv.Table.Sequential to update the Indexed to view +// all of the rows when done updating the Table, and then call br.Update() +func (ts *Tabs) TensorTable(label string, dt *table.Table) *tensorcore.Table { + tv := NewTab[*tensorcore.Table](ts, label, func(tab *core.Frame) *tensorcore.Table { + tb := core.NewToolbar(tab) + tv := tensorcore.NewTable(tab) + tb.Maker(tv.MakeToolbar) + return tv + }) + tv.SetTable(dt) + ts.Update() + return tv +} + +// TensorEditor recycles a tab with a tensorcore.TensorEditor widget +// to view given Tensor. +func (ts *Tabs) TensorEditor(label string, tsr tensor.Tensor) *tensorcore.TensorEditor { + tv := NewTab[*tensorcore.TensorEditor](ts, label, func(tab *core.Frame) *tensorcore.TensorEditor { + tb := core.NewToolbar(tab) + tv := tensorcore.NewTensorEditor(tab) + tb.Maker(tv.MakeToolbar) + return tv + }) + tv.SetTensor(tsr) + ts.Update() + return tv +} + +// TensorGrid recycles a tab with a tensorcore.TensorGrid widget +// to view given Tensor. +func (ts *Tabs) TensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGrid { + tv := NewTab[*tensorcore.TensorGrid](ts, label, func(tab *core.Frame) *tensorcore.TensorGrid { + // tb := core.NewToolbar(tab) + tv := tensorcore.NewTensorGrid(tab) + // tb.Maker(tv.MakeToolbar) + return tv + }) + tv.SetTensor(tsr) + ts.Update() + return tv +} + +// PlotTable recycles a tab with a Plot of given table.Table. +func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { + pl := NewTab[*plotcore.PlotEditor](ts, label, func(tab *core.Frame) *plotcore.PlotEditor { + return plotcore.NewSubPlot(tab) + }) + pl.SetTable(dt) + ts.Update() + return pl +} + +// SliceTable recycles a tab with a core.Table widget +// to view the given slice of structs. +func (ts *Tabs) SliceTable(label string, slc any) *core.Table { + tv := NewTab[*core.Table](ts, label, func(tab *core.Frame) *core.Table { + return core.NewTable(tab) + }) + tv.SetSlice(slc) + ts.Update() + return tv +} + +// EditorString recycles a [texteditor.Editor] tab, displaying given string. +func (ts *Tabs) EditorString(label, content string) *texteditor.Editor { + ed := NewTab[*texteditor.Editor](ts, label, func(tab *core.Frame) *texteditor.Editor { + ed := texteditor.NewEditor(tab) + ed.Styler(func(s *styles.Style) { + s.Grow.Set(1, 1) + }) + return ed + }) + if content != "" { + ed.Buffer.SetText([]byte(content)) + } + ts.Update() + return ed +} + +// EditorFile opens an editor tab for given file. +func (ts *Tabs) EditorFile(label, filename string) *texteditor.Editor { + ed := NewTab[*texteditor.Editor](ts, label, func(tab *core.Frame) *texteditor.Editor { + ed := texteditor.NewEditor(tab) + ed.Styler(func(s *styles.Style) { + s.Grow.Set(1, 1) + }) + return ed + }) + ed.Buffer.Open(core.Filename(filename)) + ts.Update() + return ed +} diff --git a/tensor/databrowser/typegen.go b/tensor/databrowser/typegen.go index b376948271..822525f714 100644 --- a/tensor/databrowser/typegen.go +++ b/tensor/databrowser/typegen.go @@ -9,7 +9,7 @@ import ( "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Browser", IDName: "browser", Doc: "Browser is a data browser, for browsing data either on an OS filesystem\nor as a datafs virtual data filesystem.\nIt supports the automatic loading of [goal] scripts as toolbar actions to\nperform pre-programmed tasks on the data, to create app-like functionality.\nScripts are ordered alphabetically and any leading #- prefix is automatically\nremoved from the label, so you can use numbers to specify a custom order.", Methods: []types.Method{{Name: "UpdateFiles", Doc: "UpdateFiles Updates the files list.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "UpdateScripts", Doc: "UpdateScripts updates the Scripts and updates the toolbar.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FS", Doc: "FS is the filesystem, if browsing an FS"}, {Name: "DataRoot", Doc: "DataRoot is the path to the root of the data to browse."}, {Name: "StartDir", Doc: "StartDir is the starting directory, where the app was originally started."}, {Name: "ScriptsDir", Doc: "ScriptsDir is the directory containing scripts for toolbar actions.\nIt defaults to DataRoot/dbscripts"}, {Name: "Scripts", Doc: "Scripts"}, {Name: "Interpreter", Doc: "Interpreter is the interpreter to use for running Browser scripts"}, {Name: "toolbar"}, {Name: "splits"}, {Name: "files"}, {Name: "tabs"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Browser", IDName: "browser", Doc: "Browser is a data browser, for browsing data either on an OS filesystem\nor as a datafs virtual data filesystem.\nIt supports the automatic loading of [goal] scripts as toolbar actions to\nperform pre-programmed tasks on the data, to create app-like functionality.\nScripts are ordered alphabetically and any leading #- prefix is automatically\nremoved from the label, so you can use numbers to specify a custom order.", Methods: []types.Method{{Name: "UpdateFiles", Doc: "UpdateFiles Updates the files list.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "UpdateScripts", Doc: "UpdateScripts updates the Scripts and updates the toolbar.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FS", Doc: "FS is the filesystem, if browsing an FS."}, {Name: "DataRoot", Doc: "DataRoot is the path to the root of the data to browse."}, {Name: "StartDir", Doc: "StartDir is the starting directory, where the app was originally started."}, {Name: "ScriptsDir", Doc: "ScriptsDir is the directory containing scripts for toolbar actions.\nIt defaults to DataRoot/dbscripts"}, {Name: "Scripts", Doc: "Scripts"}, {Name: "Interpreter", Doc: "Interpreter is the interpreter to use for running Browser scripts"}, {Name: "toolbar"}, {Name: "splits"}, {Name: "files"}, {Name: "tabs"}}}) // NewBrowser returns a new [Browser] with the given optional parent: // Browser is a data browser, for browsing data either on an OS filesystem @@ -21,7 +21,7 @@ var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser. func NewBrowser(parent ...tree.Node) *Browser { return tree.New[Browser](parent...) } // SetFS sets the [Browser.FS]: -// FS is the filesystem, if browsing an FS +// FS is the filesystem, if browsing an FS. func (t *Browser) SetFS(v fs.FS) *Browser { t.FS = v; return t } // SetDataRoot sets the [Browser.DataRoot]: @@ -37,8 +37,25 @@ func (t *Browser) SetStartDir(v string) *Browser { t.StartDir = v; return t } // It defaults to DataRoot/dbscripts func (t *Browser) SetScriptsDir(v string) *Browser { t.ScriptsDir = v; return t } +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.DataTree", IDName: "data-tree", Doc: "DataTree is the databrowser version of [filetree.Tree],\nwhich provides the Tabber to show data editors.", Embeds: []types.Field{{Name: "Tree"}}, Fields: []types.Field{{Name: "Tabber", Doc: "Tabber is the [Tabber] for this tree."}}}) + +// NewDataTree returns a new [DataTree] with the given optional parent: +// DataTree is the databrowser version of [filetree.Tree], +// which provides the Tabber to show data editors. +func NewDataTree(parent ...tree.Node) *DataTree { return tree.New[DataTree](parent...) } + +// SetTabber sets the [DataTree.Tabber]: +// Tabber is the [Tabber] for this tree. +func (t *DataTree) SetTabber(v Tabber) *DataTree { t.Tabber = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.FileNode", IDName: "file-node", Doc: "FileNode is databrowser version of FileNode for FileTree", Methods: []types.Method{{Name: "EditFiles", Doc: "EditFiles calls EditFile on selected files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "PlotFiles", Doc: "PlotFiles calls PlotFile on selected files", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "DiffDirs", Doc: "DiffDirs displays a browser with differences between two selected directories", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Node"}}}) // NewFileNode returns a new [FileNode] with the given optional parent: // FileNode is databrowser version of FileNode for FileTree func NewFileNode(parent ...tree.Node) *FileNode { return tree.New[FileNode](parent...) } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Tabs", IDName: "tabs", Doc: "Tabs implements the [Tabber] interface.", Embeds: []types.Field{{Name: "Tabs"}}}) + +// NewTabs returns a new [Tabs] with the given optional parent: +// Tabs implements the [Tabber] interface. +func NewTabs(parent ...tree.Node) *Tabs { return tree.New[Tabs](parent...) } diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index bc4876596d..153b5f7d7d 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -19,6 +19,7 @@ import ( "cogentcore.org/core/tensor/stats/stats" ) +// Times are the looping time levels for running and statistics. type Times int32 //enums:enum const ( @@ -31,10 +32,10 @@ const ( type LoopPhase int32 //enums:enum const ( - // Start is the start of the loop. Start does initialization, and should be called at Init. + // Start is the start of the loop: resets accumulated stats, initializes. Start LoopPhase = iota - // Step is at each iteration of the loop. + // Step is each iteration of the loop. Step ) diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal index f0b3f72d08..3765db829d 100644 --- a/tensor/examples/datafs-sim/sim.goal +++ b/tensor/examples/datafs-sim/sim.goal @@ -17,6 +17,7 @@ import ( "cogentcore.org/core/tensor/stats/stats" ) +// Times are the looping time levels for running and statistics. type Times int32 //enums:enum const ( @@ -29,10 +30,10 @@ const ( type LoopPhase int32 //enums:enum const ( - // Start is the start of the loop. Start does initialization, and should be called at Init. + // Start is the start of the loop: resets accumulated stats, initializes. Start LoopPhase = iota - // Step is at each iteration of the loop. + // Step is each iteration of the loop. Step ) From 702d792a9a8f94b17f86fe241331ff921f03d8ea Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 13:56:17 -0800 Subject: [PATCH 302/311] rest of fix for #1232 -- call update on tree when first ext is added. --- filetree/tree.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/filetree/tree.go b/filetree/tree.go index 83e7356256..d0bb62db56 100644 --- a/filetree/tree.go +++ b/filetree/tree.go @@ -173,7 +173,7 @@ func (ft *Tree) OpenPathFS(fsys fs.FS, path string) *Tree { } // UpdatePath updates the tree at the directory level for given path -// and everything below it. It flags that it needs render update, +// and everything below it. It flags that it needs render update, // but if a deletion or insertion happened, then NeedsLayout should also // be called. func (ft *Tree) UpdatePath(path string) { @@ -355,8 +355,13 @@ func (ft *Tree) AddExternalFile(fpath string) (*Node, error) { if has, _ := ft.hasExternalFile(pth); has { return ft.externalNodeByPath(pth) } + newExt := len(ft.externalFiles) == 0 ft.externalFiles = append(ft.externalFiles, pth) - ft.Child(0).(Filer).AsFileNode().Update() + if newExt { + ft.Update() + } else { + ft.Child(0).(Filer).AsFileNode().Update() + } return ft.externalNodeByPath(pth) } From fc1706a4733ecf02f0c33c24bacb10372674b8af Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 14:11:58 -0800 Subject: [PATCH 303/311] yaegicore update and further browser plans --- tensor/databrowser/README.md | 2 + .../symbols/cogentcore_org-core-filetree.go | 11 +++ .../cogentcore_org-core-tensor-databrowser.go | 69 +++++++++++++++++++ 3 files changed, 82 insertions(+) diff --git a/tensor/databrowser/README.md b/tensor/databrowser/README.md index dfcf6ccb6a..18cae45067 100644 --- a/tensor/databrowser/README.md +++ b/tensor/databrowser/README.md @@ -8,6 +8,8 @@ The databrowser package provides GUI elements for data exploration and visualiza * `Terminal` running a `goal` shell that supports interactive commands operating on the `datafs` data etc. TODO! +* `Browser` provides a hub structure connecting the above elements, which can be included in an actual GUI widget, that also provides additional functionality / GUI elements. + The basic `Browser` puts the `FileTree` in a left `Splits` and the `Tabs` in the right, and supports interactive exploration and visualization of data. In the [emergent](https://github.com/emer) framework, these elements are combined with other GUI elements to provide a full neural network simulation environment on top of the databrowser foundation. diff --git a/yaegicore/symbols/cogentcore_org-core-filetree.go b/yaegicore/symbols/cogentcore_org-core-filetree.go index 442548dcc5..fb6b391bf1 100644 --- a/yaegicore/symbols/cogentcore_org-core-filetree.go +++ b/yaegicore/symbols/cogentcore_org-core-filetree.go @@ -18,6 +18,7 @@ func init() { Symbols["cogentcore.org/core/filetree/filetree"] = map[string]reflect.Value{ // function, constant and variable definitions "AsNode": reflect.ValueOf(filetree.AsNode), + "AsTree": reflect.ValueOf(filetree.AsTree), "FindLocationAll": reflect.ValueOf(filetree.FindLocationAll), "FindLocationDir": reflect.ValueOf(filetree.FindLocationDir), "FindLocationFile": reflect.ValueOf(filetree.FindLocationFile), @@ -41,11 +42,13 @@ func init() { "NodeNameCount": reflect.ValueOf((*filetree.NodeNameCount)(nil)), "SearchResults": reflect.ValueOf((*filetree.SearchResults)(nil)), "Tree": reflect.ValueOf((*filetree.Tree)(nil)), + "Treer": reflect.ValueOf((*filetree.Treer)(nil)), "VCSLog": reflect.ValueOf((*filetree.VCSLog)(nil)), // interface wrapper definitions "_Filer": reflect.ValueOf((*_cogentcore_org_core_filetree_Filer)(nil)), "_NodeEmbedder": reflect.ValueOf((*_cogentcore_org_core_filetree_NodeEmbedder)(nil)), + "_Treer": reflect.ValueOf((*_cogentcore_org_core_filetree_Treer)(nil)), } } @@ -142,3 +145,11 @@ type _cogentcore_org_core_filetree_NodeEmbedder struct { } func (W _cogentcore_org_core_filetree_NodeEmbedder) AsNode() *filetree.Node { return W.WAsNode() } + +// _cogentcore_org_core_filetree_Treer is an interface wrapper for Treer type +type _cogentcore_org_core_filetree_Treer struct { + IValue interface{} + WAsFileTree func() *filetree.Tree +} + +func (W _cogentcore_org_core_filetree_Treer) AsFileTree() *filetree.Tree { return W.WAsFileTree() } diff --git a/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go b/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go index c25a7f9fe7..67a2d6d15b 100644 --- a/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go +++ b/yaegicore/symbols/cogentcore_org-core-tensor-databrowser.go @@ -3,20 +3,29 @@ package symbols import ( + "cogentcore.org/core/core" + "cogentcore.org/core/plot/plotcore" + "cogentcore.org/core/tensor" "cogentcore.org/core/tensor/databrowser" + "cogentcore.org/core/tensor/table" + "cogentcore.org/core/tensor/tensorcore" + "cogentcore.org/core/texteditor" "reflect" ) func init() { Symbols["cogentcore.org/core/tensor/databrowser/databrowser"] = map[string]reflect.Value{ // function, constant and variable definitions + "AsDataTree": reflect.ValueOf(databrowser.AsDataTree), "DataFS": reflect.ValueOf(databrowser.DataFS), "FirstComment": reflect.ValueOf(databrowser.FirstComment), "IsTableFile": reflect.ValueOf(databrowser.IsTableFile), "NewBrowser": reflect.ValueOf(databrowser.NewBrowser), "NewBrowserWindow": reflect.ValueOf(databrowser.NewBrowserWindow), + "NewDataTree": reflect.ValueOf(databrowser.NewDataTree), "NewDiffBrowserDirs": reflect.ValueOf(databrowser.NewDiffBrowserDirs), "NewFileNode": reflect.ValueOf(databrowser.NewFileNode), + "NewTabs": reflect.ValueOf(databrowser.NewTabs), "ParentBrowser": reflect.ValueOf(databrowser.ParentBrowser), "PromptOKCancel": reflect.ValueOf(databrowser.PromptOKCancel), "PromptString": reflect.ValueOf(databrowser.PromptString), @@ -26,6 +35,66 @@ func init() { // type definitions "Browser": reflect.ValueOf((*databrowser.Browser)(nil)), + "DataTree": reflect.ValueOf((*databrowser.DataTree)(nil)), "FileNode": reflect.ValueOf((*databrowser.FileNode)(nil)), + "Tabber": reflect.ValueOf((*databrowser.Tabber)(nil)), + "Tabs": reflect.ValueOf((*databrowser.Tabs)(nil)), + "Treer": reflect.ValueOf((*databrowser.Treer)(nil)), + + // interface wrapper definitions + "_Tabber": reflect.ValueOf((*_cogentcore_org_core_tensor_databrowser_Tabber)(nil)), + "_Treer": reflect.ValueOf((*_cogentcore_org_core_tensor_databrowser_Treer)(nil)), } } + +// _cogentcore_org_core_tensor_databrowser_Tabber is an interface wrapper for Tabber type +type _cogentcore_org_core_tensor_databrowser_Tabber struct { + IValue interface{} + WAsTabs func() *databrowser.Tabs + WEditorFile func(label string, filename string) *texteditor.Editor + WEditorString func(label string, content string) *texteditor.Editor + WPlotTable func(label string, dt *table.Table) *plotcore.PlotEditor + WRecycleTab func(name string) *core.Frame + WSliceTable func(label string, slc any) *core.Table + WTensorEditor func(label string, tsr tensor.Tensor) *tensorcore.TensorEditor + WTensorGrid func(label string, tsr tensor.Tensor) *tensorcore.TensorGrid + WTensorTable func(label string, dt *table.Table) *tensorcore.Table +} + +func (W _cogentcore_org_core_tensor_databrowser_Tabber) AsTabs() *databrowser.Tabs { + return W.WAsTabs() +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) EditorFile(label string, filename string) *texteditor.Editor { + return W.WEditorFile(label, filename) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) EditorString(label string, content string) *texteditor.Editor { + return W.WEditorString(label, content) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { + return W.WPlotTable(label, dt) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) RecycleTab(name string) *core.Frame { + return W.WRecycleTab(name) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) SliceTable(label string, slc any) *core.Table { + return W.WSliceTable(label, slc) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) TensorEditor(label string, tsr tensor.Tensor) *tensorcore.TensorEditor { + return W.WTensorEditor(label, tsr) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) TensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGrid { + return W.WTensorGrid(label, tsr) +} +func (W _cogentcore_org_core_tensor_databrowser_Tabber) TensorTable(label string, dt *table.Table) *tensorcore.Table { + return W.WTensorTable(label, dt) +} + +// _cogentcore_org_core_tensor_databrowser_Treer is an interface wrapper for Treer type +type _cogentcore_org_core_tensor_databrowser_Treer struct { + IValue interface{} + WAsDataTree func() *databrowser.DataTree +} + +func (W _cogentcore_org_core_tensor_databrowser_Treer) AsDataTree() *databrowser.DataTree { + return W.WAsDataTree() +} From c18519b264179dade6bcfd67a05d08f35d036399 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 16:59:28 -0800 Subject: [PATCH 304/311] databrowser separated, Basic pulled out --- tensor/databrowser/basic.go | 91 ++++++++++++++++++++++ tensor/databrowser/browser.go | 114 +++++----------------------- tensor/databrowser/scripts.go | 4 +- tensor/databrowser/tabs.go | 5 +- tensor/databrowser/typegen.go | 33 +++++--- tensor/examples/datafs-sim/sim.go | 2 +- tensor/examples/datafs-sim/sim.goal | 2 +- 7 files changed, 141 insertions(+), 110 deletions(-) create mode 100644 tensor/databrowser/basic.go diff --git a/tensor/databrowser/basic.go b/tensor/databrowser/basic.go new file mode 100644 index 0000000000..8a33de6a41 --- /dev/null +++ b/tensor/databrowser/basic.go @@ -0,0 +1,91 @@ +// Copyright (c) 2024, Cogent Core. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package databrowser + +import ( + "io/fs" + "os" + "path/filepath" + + "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/fsx" + "cogentcore.org/core/core" + "cogentcore.org/core/events" + "cogentcore.org/core/styles" + "cogentcore.org/core/tree" + "cogentcore.org/core/types" +) + +// Basic is a basic data browser with the files as the left panel, +// and the Tabber as the right panel. +type Basic struct { + core.Frame + Browser +} + +// Init initializes with the data and script directories +func (br *Basic) Init() { + br.Frame.Init() + br.Styler(func(s *styles.Style) { + s.Grow.Set(1, 1) + }) + br.InitInterp() + + br.OnShow(func(e events.Event) { + br.UpdateFiles() + }) + + tree.AddChildAt(br, "splits", func(w *core.Splits) { + br.Splits = w + w.SetSplits(.15, .85) + tree.AddChildAt(w, "fileframe", func(w *core.Frame) { + w.Styler(func(s *styles.Style) { + s.Direction = styles.Column + s.Overflow.Set(styles.OverflowAuto) + s.Grow.Set(1, 1) + }) + tree.AddChildAt(w, "filetree", func(w *DataTree) { + br.Files = w + w.FileNodeType = types.For[FileNode]() + }) + }) + tree.AddChildAt(w, "tabs", func(w *Tabs) { + br.Tabs = w + w.Type = core.FunctionalTabs + }) + }) + br.Updater(func() { + if br.Files != nil { + br.Files.Tabber = br.Tabs + } + }) + +} + +// NewBasicWindow opens a new data Browser for given +// file system (nil for os files) and data directory. +func NewBasicWindow(fsys fs.FS, dataDir string) *Basic { + startDir, _ := os.Getwd() + startDir = errors.Log1(filepath.Abs(startDir)) + b := core.NewBody("Cogent Data Browser: " + fsx.DirAndFile(startDir)) + br := NewBasic(b) + br.FS = fsys + ddr := dataDir + if fsys == nil { + ddr = errors.Log1(filepath.Abs(dataDir)) + } + b.AddTopBar(func(bar *core.Frame) { + tb := core.NewToolbar(bar) + br.Toolbar = tb + tb.Maker(br.MakeToolbar) + }) + br.SetDataRoot(ddr) + br.SetScriptsDir(filepath.Join(ddr, "dbscripts")) + TheBrowser = &br.Browser + br.Interpreter.Eval("br := databrowser.TheBrowser") // grab it + br.UpdateScripts() + b.RunWindow() + return br +} diff --git a/tensor/databrowser/browser.go b/tensor/databrowser/browser.go index 2cef08aaa1..825648ab4a 100644 --- a/tensor/databrowser/browser.go +++ b/tensor/databrowser/browser.go @@ -8,19 +8,13 @@ package databrowser import ( "io/fs" - "os" - "path/filepath" "slices" - "cogentcore.org/core/base/errors" - "cogentcore.org/core/base/fsx" "cogentcore.org/core/core" "cogentcore.org/core/events" "cogentcore.org/core/goal/interpreter" "cogentcore.org/core/icons" - "cogentcore.org/core/styles" "cogentcore.org/core/tree" - "cogentcore.org/core/types" "golang.org/x/exp/maps" ) @@ -29,15 +23,16 @@ import ( // where it is used to get a local variable for subsequent use. var TheBrowser *Browser -// Browser is a data browser, for browsing data either on an OS filesystem -// or as a datafs virtual data filesystem. +// Browser holds all the elements of a data browser, for browsing data +// either on an OS filesystem or as a datafs virtual data filesystem. // It supports the automatic loading of [goal] scripts as toolbar actions to // perform pre-programmed tasks on the data, to create app-like functionality. // Scripts are ordered alphabetically and any leading #- prefix is automatically // removed from the label, so you can use numbers to specify a custom order. -type Browser struct { - core.Frame - +// It is not a [core.Widget] itself, and is intended to be incorporated into +// a [core.Frame] widget, potentially along with other custom elements. +// See [Basic] for a basic implementation. +type Browser struct { //types:add -setters // FS is the filesystem, if browsing an FS. FS fs.FS @@ -57,104 +52,31 @@ type Browser struct { // Interpreter is the interpreter to use for running Browser scripts Interpreter *interpreter.Interpreter `set:"-"` - toolbar *core.Toolbar - splits *core.Splits - files *DataTree - tabs *Tabs -} + // Files is the [DataTree] tree browser of the datafs or files. + Files *DataTree -// Init initializes with the data and script directories -func (br *Browser) Init() { - br.Frame.Init() - br.Styler(func(s *styles.Style) { - s.Grow.Set(1, 1) - }) - br.InitInterp() - - br.OnShow(func(e events.Event) { - br.UpdateFiles() - }) + // Tabs is the [Tabber] element managing tabs of data views. + Tabs Tabber - tree.AddChildAt(br, "splits", func(w *core.Splits) { - br.splits = w - w.SetSplits(.15, .85) - tree.AddChildAt(w, "fileframe", func(w *core.Frame) { - w.Styler(func(s *styles.Style) { - s.Direction = styles.Column - s.Overflow.Set(styles.OverflowAuto) - s.Grow.Set(1, 1) - }) - tree.AddChildAt(w, "filetree", func(w *DataTree) { - br.files = w - w.FileNodeType = types.For[FileNode]() - }) - }) - tree.AddChildAt(w, "tabs", func(w *Tabs) { - br.tabs = w - w.Type = core.FunctionalTabs - }) - }) - br.Updater(func() { - if br.files != nil { - br.files.Tabber = br.tabs - } - }) + // Toolbar is the top-level toolbar for the browser, if used. + Toolbar *core.Toolbar -} - -// NewBrowserWindow opens a new data Browser for given -// file system (nil for os files) and data directory. -func NewBrowserWindow(fsys fs.FS, dataDir string) *Browser { - startDir, _ := os.Getwd() - startDir = errors.Log1(filepath.Abs(startDir)) - b := core.NewBody("Cogent Data Browser: " + fsx.DirAndFile(startDir)) - br := NewBrowser(b) - br.FS = fsys - ddr := dataDir - if fsys == nil { - ddr = errors.Log1(filepath.Abs(dataDir)) - } - b.AddTopBar(func(bar *core.Frame) { - tb := core.NewToolbar(bar) - br.toolbar = tb - tb.Maker(br.MakeToolbar) - }) - br.SetDataRoot(ddr) - br.SetScriptsDir(filepath.Join(ddr, "dbscripts")) - TheBrowser = br - br.Interpreter.Eval("br := databrowser.TheBrowser") // grab it - br.UpdateScripts() - b.RunWindow() - return br -} - -// ParentBrowser returns the Browser parent of given node -func ParentBrowser(tn tree.Node) *Browser { - var res *Browser - tn.AsTree().WalkUp(func(n tree.Node) bool { - if c, ok := n.(*Browser); ok { - res = c - return false - } - return true - }) - return res + // Splits is the overall [core.Splits] for the browser. + Splits *core.Splits } // UpdateFiles Updates the files list. func (br *Browser) UpdateFiles() { //types:add - files := br.files + if br.Files == nil { + return + } + files := br.Files if br.FS != nil { files.SortByModTime = true files.OpenPathFS(br.FS, br.DataRoot) } else { files.OpenPath(br.DataRoot) } - br.Update() -} - -func (br *Browser) GetDataRoot() string { - return br.DataRoot } func (br *Browser) MakeToolbar(p *tree.Plan) { diff --git a/tensor/databrowser/scripts.go b/tensor/databrowser/scripts.go index c52b553846..2825be2657 100644 --- a/tensor/databrowser/scripts.go +++ b/tensor/databrowser/scripts.go @@ -65,8 +65,8 @@ func (br *Browser) UpdateScripts() { //types:add slog.Error(err.Error()) } } - if br.toolbar != nil { - br.toolbar.Update() + if br.Toolbar != nil { + br.Toolbar.Update() } } diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go index 4d8d77a92e..21b79d7052 100644 --- a/tensor/databrowser/tabs.go +++ b/tensor/databrowser/tabs.go @@ -5,6 +5,8 @@ package databrowser import ( + "fmt" + "cogentcore.org/core/core" "cogentcore.org/core/plot/plotcore" "cogentcore.org/core/styles" @@ -64,7 +66,8 @@ func NewTab[T any](tb Tabber, label string, mkfun func(tab *core.Frame) T) T { if tt, ok := tab.Child(1).(T); ok { return tt } - core.MessageSnackbar(tb.AsTabs(), "Name / Type conflict: tab "+label+" does not have the expected type of content") + err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content", label) + core.ErrorSnackbar(tb.AsTabs(), err) return zv } w := mkfun(tab) diff --git a/tensor/databrowser/typegen.go b/tensor/databrowser/typegen.go index 822525f714..90278c2680 100644 --- a/tensor/databrowser/typegen.go +++ b/tensor/databrowser/typegen.go @@ -5,20 +5,19 @@ package databrowser import ( "io/fs" + "cogentcore.org/core/core" "cogentcore.org/core/tree" "cogentcore.org/core/types" ) -var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Browser", IDName: "browser", Doc: "Browser is a data browser, for browsing data either on an OS filesystem\nor as a datafs virtual data filesystem.\nIt supports the automatic loading of [goal] scripts as toolbar actions to\nperform pre-programmed tasks on the data, to create app-like functionality.\nScripts are ordered alphabetically and any leading #- prefix is automatically\nremoved from the label, so you can use numbers to specify a custom order.", Methods: []types.Method{{Name: "UpdateFiles", Doc: "UpdateFiles Updates the files list.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "UpdateScripts", Doc: "UpdateScripts updates the Scripts and updates the toolbar.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Embeds: []types.Field{{Name: "Frame"}}, Fields: []types.Field{{Name: "FS", Doc: "FS is the filesystem, if browsing an FS."}, {Name: "DataRoot", Doc: "DataRoot is the path to the root of the data to browse."}, {Name: "StartDir", Doc: "StartDir is the starting directory, where the app was originally started."}, {Name: "ScriptsDir", Doc: "ScriptsDir is the directory containing scripts for toolbar actions.\nIt defaults to DataRoot/dbscripts"}, {Name: "Scripts", Doc: "Scripts"}, {Name: "Interpreter", Doc: "Interpreter is the interpreter to use for running Browser scripts"}, {Name: "toolbar"}, {Name: "splits"}, {Name: "files"}, {Name: "tabs"}}}) +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Basic", IDName: "basic", Doc: "Basic is a basic data browser with the files as the left panel,\nand the Tabber as the right panel.", Embeds: []types.Field{{Name: "Frame"}, {Name: "Browser"}}}) -// NewBrowser returns a new [Browser] with the given optional parent: -// Browser is a data browser, for browsing data either on an OS filesystem -// or as a datafs virtual data filesystem. -// It supports the automatic loading of [goal] scripts as toolbar actions to -// perform pre-programmed tasks on the data, to create app-like functionality. -// Scripts are ordered alphabetically and any leading #- prefix is automatically -// removed from the label, so you can use numbers to specify a custom order. -func NewBrowser(parent ...tree.Node) *Browser { return tree.New[Browser](parent...) } +// NewBasic returns a new [Basic] with the given optional parent: +// Basic is a basic data browser with the files as the left panel, +// and the Tabber as the right panel. +func NewBasic(parent ...tree.Node) *Basic { return tree.New[Basic](parent...) } + +var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.Browser", IDName: "browser", Doc: "Browser holds all the elements of a data browser, for browsing data\neither on an OS filesystem or as a datafs virtual data filesystem.\nIt supports the automatic loading of [goal] scripts as toolbar actions to\nperform pre-programmed tasks on the data, to create app-like functionality.\nScripts are ordered alphabetically and any leading #- prefix is automatically\nremoved from the label, so you can use numbers to specify a custom order.\nIt is not a [core.Widget] itself, and is intended to be incorporated into\na [core.Frame] widget, potentially along with other custom elements.\nSee [Basic] for a basic implementation.", Directives: []types.Directive{{Tool: "types", Directive: "add", Args: []string{"-setters"}}}, Methods: []types.Method{{Name: "UpdateFiles", Doc: "UpdateFiles Updates the files list.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}, {Name: "UpdateScripts", Doc: "UpdateScripts updates the Scripts and updates the toolbar.", Directives: []types.Directive{{Tool: "types", Directive: "add"}}}}, Fields: []types.Field{{Name: "FS", Doc: "FS is the filesystem, if browsing an FS."}, {Name: "DataRoot", Doc: "DataRoot is the path to the root of the data to browse."}, {Name: "StartDir", Doc: "StartDir is the starting directory, where the app was originally started."}, {Name: "ScriptsDir", Doc: "ScriptsDir is the directory containing scripts for toolbar actions.\nIt defaults to DataRoot/dbscripts"}, {Name: "Scripts", Doc: "Scripts"}, {Name: "Interpreter", Doc: "Interpreter is the interpreter to use for running Browser scripts"}, {Name: "Files", Doc: "Files is the [DataTree] tree browser of the datafs or files."}, {Name: "Tabs", Doc: "Tabs is the [Tabber] element managing tabs of data views."}, {Name: "Toolbar", Doc: "Toolbar is the top-level toolbar for the browser, if used."}, {Name: "Splits", Doc: "Splits is the overall [core.Splits] for the browser."}}}) // SetFS sets the [Browser.FS]: // FS is the filesystem, if browsing an FS. @@ -37,6 +36,22 @@ func (t *Browser) SetStartDir(v string) *Browser { t.StartDir = v; return t } // It defaults to DataRoot/dbscripts func (t *Browser) SetScriptsDir(v string) *Browser { t.ScriptsDir = v; return t } +// SetFiles sets the [Browser.Files]: +// Files is the [DataTree] tree browser of the datafs or files. +func (t *Browser) SetFiles(v *DataTree) *Browser { t.Files = v; return t } + +// SetTabs sets the [Browser.Tabs]: +// Tabs is the [Tabber] element managing tabs of data views. +func (t *Browser) SetTabs(v Tabber) *Browser { t.Tabs = v; return t } + +// SetToolbar sets the [Browser.Toolbar]: +// Toolbar is the top-level toolbar for the browser, if used. +func (t *Browser) SetToolbar(v *core.Toolbar) *Browser { t.Toolbar = v; return t } + +// SetSplits sets the [Browser.Splits]: +// Splits is the overall [core.Splits] for the browser. +func (t *Browser) SetSplits(v *core.Splits) *Browser { t.Splits = v; return t } + var _ = types.AddType(&types.Type{Name: "cogentcore.org/core/tensor/databrowser.DataTree", IDName: "data-tree", Doc: "DataTree is the databrowser version of [filetree.Tree],\nwhich provides the Tabber to show data editors.", Embeds: []types.Field{{Name: "Tree"}}, Fields: []types.Field{{Name: "Tabber", Doc: "Tabber is the [Tabber] for this tree."}}}) // NewDataTree returns a new [DataTree] with the given optional parent: diff --git a/tensor/examples/datafs-sim/sim.go b/tensor/examples/datafs-sim/sim.go index 153b5f7d7d..ea117afcd2 100644 --- a/tensor/examples/datafs-sim/sim.go +++ b/tensor/examples/datafs-sim/sim.go @@ -213,6 +213,6 @@ func main() { ss.ConfigAll() ss.Run() - databrowser.NewBrowserWindow(ss.Root, "Root") + databrowser.NewBasicWindow(ss.Root, "Root") core.Wait() } diff --git a/tensor/examples/datafs-sim/sim.goal b/tensor/examples/datafs-sim/sim.goal index 3765db829d..b500616ddf 100644 --- a/tensor/examples/datafs-sim/sim.goal +++ b/tensor/examples/datafs-sim/sim.goal @@ -211,6 +211,6 @@ func main() { ss.ConfigAll() ss.Run() - databrowser.NewBrowserWindow(ss.Root, "Root") + databrowser.NewBasicWindow(ss.Root, "Root") core.Wait() } From 06d5c43414337b19de5c7109d6521b91d2f8be22 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Thu, 14 Nov 2024 23:57:54 -0800 Subject: [PATCH 305/311] fix goal parsing of static initializer Field: expressions. databrowser fixes --- goal/transpile/paths.go | 5 +++++ goal/transpile/transpile.go | 8 ++++++-- goal/transpile/transpile_test.go | 17 +++++++++++++---- tensor/databrowser/basic.go | 3 --- tensor/databrowser/filetree.go | 2 ++ tensor/databrowser/tabs.go | 19 ++++++++++++------- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/goal/transpile/paths.go b/goal/transpile/paths.go index 0803d5f227..669da539d4 100644 --- a/goal/transpile/paths.go +++ b/goal/transpile/paths.go @@ -57,6 +57,11 @@ func (tk Tokens) Path(idx0 bool) (string, int) { ci += tin + 1 str = tid + tk[tin].String() lastEnd += len(str) + if ci >= n || int(tk[ci].Pos) > lastEnd { // just 2 or 2 and a space + if tk[tin].Tok == token.COLON { // go Ident: static initializer + return "", 0 + } + } } prevWasDelim := true for { diff --git a/goal/transpile/transpile.go b/goal/transpile/transpile.go index 5849628b5e..2e86475639 100644 --- a/goal/transpile/transpile.go +++ b/goal/transpile/transpile.go @@ -28,13 +28,14 @@ func (st *State) TranspileLine(code string) string { st.ParenDepth += paren st.BraceDepth += brace st.BrackDepth += brack + // logx.PrintlnDebug("depths: ", st.ParenDepth, st.BraceDepth, st.BrackDepth, st.TypeDepth, st.DeclDepth) if st.TypeDepth > 0 && st.BraceDepth == 0 { st.TypeDepth = 0 } if st.DeclDepth > 0 && (st.ParenDepth == 0 && st.BraceDepth == 0) { st.DeclDepth = 0 } - // logx.PrintlnDebug("depths: ", sh.ParenDepth, sh.BraceDepth, sh.BrackDepth) + // logx.PrintlnDebug("depths: ", st.ParenDepth, st.BraceDepth, st.BrackDepth, st.TypeDepth, st.DeclDepth) return toks.Code() } @@ -70,7 +71,7 @@ func (st *State) TranspileLineTokens(code string) Tokens { if toks[0].Tok == token.IMPORT || toks[0].Tok == token.VAR || toks[0].Tok == token.CONST { st.DeclDepth++ } - + // logx.PrintlnDebug("depths: ", st.ParenDepth, st.BraceDepth, st.BrackDepth, st.TypeDepth, st.DeclDepth) if st.TypeDepth > 0 || st.DeclDepth > 0 { logx.PrintlnDebug("go: type / decl defn") return st.TranspileGo(toks, code) @@ -81,6 +82,9 @@ func (st *State) TranspileLineTokens(code string) Tokens { en := len(ewords) f0exec := (t0.Tok == token.IDENT && ExecWordIsCommand(ewords[0])) + if f0exec && n > 1 && toks[1].Tok == token.COLON { // go Ident: static initializer + f0exec = false + } switch { case t0.Tok == token.ILLEGAL: diff --git a/goal/transpile/transpile_test.go b/goal/transpile/transpile_test.go index 77006ef627..f0b3bbc465 100644 --- a/goal/transpile/transpile_test.go +++ b/goal/transpile/transpile_test.go @@ -200,15 +200,24 @@ func TestCommand(t *testing.T) { tests := []exIn{ { `command list { -ls -la args... -}`, + ls -la args... + }`, `goal.AddCommand("list", func(args ...string) { goal.Run("ls", "-la", "args...") })`}, + { + ` ss.GUI.AddToolbarItem(p, egui.ToolbarItem{ + Label: "Reset RunLog", + }) +`, + `ss.GUI.AddToolbarItem(p, egui.ToolbarItem { +Label: "Reset RunLog", +} ) +`}, } - st := NewState() for _, test := range tests { + st := NewState() st.TranspileCode(test.i) o := st.Code() assert.Equal(t, test.e, o) @@ -219,7 +228,7 @@ goal.Run("ls", "-la", "args...") func TestCur(t *testing.T) { // logx.UserLevel = slog.LevelDebug tests := []exIn{ - {"&Data[idx, Integ] = integ", `Data.ValuePtr(int(idx), int(Integ)) = integ`}, + {`Label: "Reset RunLog",`, `Label: "Reset RunLog",`}, } st := NewState() st.MathRecord = false diff --git a/tensor/databrowser/basic.go b/tensor/databrowser/basic.go index 8a33de6a41..26020282c1 100644 --- a/tensor/databrowser/basic.go +++ b/tensor/databrowser/basic.go @@ -15,7 +15,6 @@ import ( "cogentcore.org/core/events" "cogentcore.org/core/styles" "cogentcore.org/core/tree" - "cogentcore.org/core/types" ) // Basic is a basic data browser with the files as the left panel, @@ -48,12 +47,10 @@ func (br *Basic) Init() { }) tree.AddChildAt(w, "filetree", func(w *DataTree) { br.Files = w - w.FileNodeType = types.For[FileNode]() }) }) tree.AddChildAt(w, "tabs", func(w *Tabs) { br.Tabs = w - w.Type = core.FunctionalTabs }) }) br.Updater(func() { diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index b3f017e2f2..82bb9c221e 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -22,6 +22,7 @@ import ( "cogentcore.org/core/tensor/table" "cogentcore.org/core/texteditor/diffbrowser" "cogentcore.org/core/tree" + "cogentcore.org/core/types" ) // Treer is an interface for getting the Root node as a DataTree struct. @@ -54,6 +55,7 @@ func (ft *DataTree) AsDataTree() *DataTree { func (ft *DataTree) Init() { ft.Tree.Init() ft.Root = ft + ft.FileNodeType = types.For[FileNode]() } // FileNode is databrowser version of FileNode for FileTree diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go index 21b79d7052..6080361d71 100644 --- a/tensor/databrowser/tabs.go +++ b/tensor/databrowser/tabs.go @@ -79,6 +79,11 @@ type Tabs struct { core.Tabs } +func (ts *Tabs) Init() { + ts.Tabs.Init() + ts.Type = core.FunctionalTabs +} + func (ts *Tabs) AsTabs() *Tabs { return ts } @@ -89,7 +94,7 @@ func (ts *Tabs) AsTabs() *Tabs { // Use tv.Table.Sequential to update the Indexed to view // all of the rows when done updating the Table, and then call br.Update() func (ts *Tabs) TensorTable(label string, dt *table.Table) *tensorcore.Table { - tv := NewTab[*tensorcore.Table](ts, label, func(tab *core.Frame) *tensorcore.Table { + tv := NewTab(ts, label, func(tab *core.Frame) *tensorcore.Table { tb := core.NewToolbar(tab) tv := tensorcore.NewTable(tab) tb.Maker(tv.MakeToolbar) @@ -103,7 +108,7 @@ func (ts *Tabs) TensorTable(label string, dt *table.Table) *tensorcore.Table { // TensorEditor recycles a tab with a tensorcore.TensorEditor widget // to view given Tensor. func (ts *Tabs) TensorEditor(label string, tsr tensor.Tensor) *tensorcore.TensorEditor { - tv := NewTab[*tensorcore.TensorEditor](ts, label, func(tab *core.Frame) *tensorcore.TensorEditor { + tv := NewTab(ts, label, func(tab *core.Frame) *tensorcore.TensorEditor { tb := core.NewToolbar(tab) tv := tensorcore.NewTensorEditor(tab) tb.Maker(tv.MakeToolbar) @@ -117,7 +122,7 @@ func (ts *Tabs) TensorEditor(label string, tsr tensor.Tensor) *tensorcore.Tensor // TensorGrid recycles a tab with a tensorcore.TensorGrid widget // to view given Tensor. func (ts *Tabs) TensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGrid { - tv := NewTab[*tensorcore.TensorGrid](ts, label, func(tab *core.Frame) *tensorcore.TensorGrid { + tv := NewTab(ts, label, func(tab *core.Frame) *tensorcore.TensorGrid { // tb := core.NewToolbar(tab) tv := tensorcore.NewTensorGrid(tab) // tb.Maker(tv.MakeToolbar) @@ -130,7 +135,7 @@ func (ts *Tabs) TensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGr // PlotTable recycles a tab with a Plot of given table.Table. func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { - pl := NewTab[*plotcore.PlotEditor](ts, label, func(tab *core.Frame) *plotcore.PlotEditor { + pl := NewTab(ts, label, func(tab *core.Frame) *plotcore.PlotEditor { return plotcore.NewSubPlot(tab) }) pl.SetTable(dt) @@ -141,7 +146,7 @@ func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { // SliceTable recycles a tab with a core.Table widget // to view the given slice of structs. func (ts *Tabs) SliceTable(label string, slc any) *core.Table { - tv := NewTab[*core.Table](ts, label, func(tab *core.Frame) *core.Table { + tv := NewTab(ts, label, func(tab *core.Frame) *core.Table { return core.NewTable(tab) }) tv.SetSlice(slc) @@ -151,7 +156,7 @@ func (ts *Tabs) SliceTable(label string, slc any) *core.Table { // EditorString recycles a [texteditor.Editor] tab, displaying given string. func (ts *Tabs) EditorString(label, content string) *texteditor.Editor { - ed := NewTab[*texteditor.Editor](ts, label, func(tab *core.Frame) *texteditor.Editor { + ed := NewTab(ts, label, func(tab *core.Frame) *texteditor.Editor { ed := texteditor.NewEditor(tab) ed.Styler(func(s *styles.Style) { s.Grow.Set(1, 1) @@ -167,7 +172,7 @@ func (ts *Tabs) EditorString(label, content string) *texteditor.Editor { // EditorFile opens an editor tab for given file. func (ts *Tabs) EditorFile(label, filename string) *texteditor.Editor { - ed := NewTab[*texteditor.Editor](ts, label, func(tab *core.Frame) *texteditor.Editor { + ed := NewTab(ts, label, func(tab *core.Frame) *texteditor.Editor { ed := texteditor.NewEditor(tab) ed.Styler(func(s *styles.Style) { s.Grow.Set(1, 1) From 5f3420143604825de9ec158452c812ef31815d4f Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 15 Nov 2024 14:43:25 -0800 Subject: [PATCH 306/311] goal: some crash protections --- plot/table.go | 15 ++++++++++----- tensor/databrowser/tabs.go | 6 ++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/plot/table.go b/plot/table.go index 2c4d7d26de..b3389ba8c7 100644 --- a/plot/table.go +++ b/plot/table.go @@ -8,6 +8,7 @@ import ( "fmt" "cogentcore.org/core/base/errors" + "cogentcore.org/core/base/reflectx" "cogentcore.org/core/tensor/table" "golang.org/x/exp/maps" ) @@ -154,7 +155,7 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { } } pl := pt.New(data) - if pl == nil { + if reflectx.AnyIsNil(pl) { err = fmt.Errorf("plot.NewTablePlot: error in creating plotter type: %q", ptyp) errs = append(errs, err) continue @@ -182,10 +183,14 @@ func NewTablePlot(dt *table.Table) (*Plot, error) { if csty[xi].Label != "" { lbl = csty[xi].Label } - pl0 := plt.Plotters[0] - pl0.Stylers().Add(func(s *Style) { - s.Plot.XAxis.Label = lbl - }) + if len(plt.Plotters) > 0 { + pl0 := plt.Plotters[0] + if pl0 != nil { + pl0.Stylers().Add(func(s *Style) { + s.Plot.XAxis.Label = lbl + }) + } + } } nbar := len(barCols) if nbar > 1 { diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go index 6080361d71..46958e657d 100644 --- a/tensor/databrowser/tabs.go +++ b/tensor/databrowser/tabs.go @@ -138,8 +138,10 @@ func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { pl := NewTab(ts, label, func(tab *core.Frame) *plotcore.PlotEditor { return plotcore.NewSubPlot(tab) }) - pl.SetTable(dt) - ts.Update() + if pl != nil { + pl.SetTable(dt) + ts.Update() + } return pl } From 28961a68c6735d0714b7181bdfed11c3d250e608 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Fri, 15 Nov 2024 17:26:30 -0800 Subject: [PATCH 307/311] goal: plotting is robust to nil and plot update function in databrowser tabs --- plot/data.go | 3 --- plot/plots/xy.go | 5 ++++- tensor/databrowser/tabs.go | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/plot/data.go b/plot/data.go index 8e3a8718c4..4778a03d8f 100644 --- a/plot/data.go +++ b/plot/data.go @@ -175,9 +175,6 @@ func CopyValues(data Valuer) (Values, error) { if data == nil { return nil, ErrNoData } - if data.Len() == 0 { - return nil, ErrNoData - } cpy := make(Values, 0, data.Len()) for i := 0; i < data.Len(); i++ { v := data.Float1D(i) diff --git a/plot/plots/xy.go b/plot/plots/xy.go index 765ec29911..695d68b966 100644 --- a/plot/plots/xy.go +++ b/plot/plots/xy.go @@ -116,10 +116,13 @@ func (ln *XY) Data() (data plot.Data, pixX, pixY []float32) { // Plot does the drawing, implementing the plot.Plotter interface. func (ln *XY) Plot(plt *plot.Plot) { - pc := plt.Paint ln.PX = plot.PlotX(plt, ln.X) ln.PY = plot.PlotY(plt, ln.Y) np := len(ln.PX) + if np == 0 { + return + } + pc := plt.Paint if ln.Style.Line.HasFill() { pc.FillStyle.Color = ln.Style.Line.Fill minY := plt.PY(plt.Y.Range.Min) diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go index 46958e657d..9f478b6cd4 100644 --- a/tensor/databrowser/tabs.go +++ b/tensor/databrowser/tabs.go @@ -24,6 +24,9 @@ type Tabber interface { // AsTabs returns the underlying [Tabs] widget. AsTabs() *Tabs + // TabByName returns a tab with the given name, nil if not found. + TabByName(name string) *core.Frame + // RecycleTab returns a tab with the given name, first by looking for an existing one, // and if not found, making a new one. It returns the frame for the tab. RecycleTab(name string) *core.Frame @@ -43,6 +46,10 @@ type Tabber interface { // PlotTable recycles a tab with a Plot of given [table.Table]. PlotTable(label string, dt *table.Table) *plotcore.PlotEditor + // GoUpdatePlot calls GoUpdatePlot on plot at tab with given name. + // Does nothing if tab name doesn't exist (returns nil). + GoUpdatePlot(label string) *plotcore.PlotEditor + // todo: PlotData of plot.Data // SliceTable recycles a tab with a [core.Table] widget @@ -74,6 +81,24 @@ func NewTab[T any](tb Tabber, label string, mkfun func(tab *core.Frame) T) T { return w } +// TabAt returns widget of given type at tab of given name, nil if tab not found. +func TabAt[T any](tb Tabber, label string) T { + var zv T + tab := tb.TabByName(label) + if tab == nil { + return zv + } + if !tab.HasChildren() { // shouldn't happen + return zv + } + if tt, ok := tab.Child(1).(T); ok { + return tt + } + err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content", label) + core.ErrorSnackbar(tb.AsTabs(), err) + return zv +} + // Tabs implements the [Tabber] interface. type Tabs struct { core.Tabs @@ -145,6 +170,16 @@ func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { return pl } +// GoUpdatePlot calls GoUpdatePlot on plot at tab with given name. +// Does nothing if tab name doesn't exist (returns nil). +func (ts *Tabs) GoUpdatePlot(label string) *plotcore.PlotEditor { + pl := TabAt[*plotcore.PlotEditor](ts, label) + if pl != nil { + pl.GoUpdatePlot() + } + return pl +} + // SliceTable recycles a tab with a core.Table widget // to view the given slice of structs. func (ts *Tabs) SliceTable(label string, slc any) *core.Table { From 7487521d4562f136ffd5edabda33db68cdf6e668 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 16 Nov 2024 00:38:56 -0800 Subject: [PATCH 308/311] key fix to databrowser tabs logic: target item is last widget in tab frame -- can't use subplot --- tensor/databrowser/filetree.go | 46 ++++++++--------------------- tensor/databrowser/tabs.go | 54 +++++++++++++++++++++++++++++----- 2 files changed, 59 insertions(+), 41 deletions(-) diff --git a/tensor/databrowser/filetree.go b/tensor/databrowser/filetree.go index 82bb9c221e..0c2bcec4c2 100644 --- a/tensor/databrowser/filetree.go +++ b/tensor/databrowser/filetree.go @@ -249,44 +249,22 @@ func (fn *FileNode) PlotFile() { return } d := DataFS(fn.AsNode()) + if d != nil { + ts.PlotDataFS(d) + return + } + if fn.Info.Cat != fileinfo.Data { + return + } df := fsx.DirAndFile(string(fn.Filepath)) ptab := df + " Plot" - var dt *table.Table - switch { - case fn.IsDir(): - dt = d.GetDirTable(nil) - case fn.Info.Cat == fileinfo.Data: - switch fn.Info.Known { - case fileinfo.Tensor: - tsr := d.Data - dt = table.New(df) - dt.Columns.Rows = tsr.DimSize(0) - if ix, ok := tsr.(*tensor.Rows); ok { - dt.Indexes = ix.Indexes - } - rc := dt.AddIntColumn("Row") - for r := range dt.Columns.Rows { - rc.Values[r] = r - } - dt.AddColumn(fn.Name, tsr.AsValues()) - // case fileinfo.Table: - // dt = d.AsTable() - default: - dt = table.New(df) - err := dt.OpenCSV(fsx.Filename(fn.Filepath), tensor.Tab) // todo: need more flexible data handling mode - if err != nil { - core.ErrorSnackbar(fn, err) - dt = nil - } - } - } - if dt == nil { + dt := table.New(df) + err := dt.OpenCSV(fsx.Filename(fn.Filepath), tensor.Tab) // todo: need more flexible data handling mode + if err != nil { + core.ErrorSnackbar(fn, err) return } - pl := ts.PlotTable(ptab, dt) - _ = pl - // pl.Options.Title = df - // TODO: apply column and plot level options. + ts.PlotTable(ptab, dt) } // DiffDirs displays a browser with differences between two selected directories diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go index 9f478b6cd4..aaac020395 100644 --- a/tensor/databrowser/tabs.go +++ b/tensor/databrowser/tabs.go @@ -7,10 +7,12 @@ package databrowser import ( "fmt" + "cogentcore.org/core/base/fsx" "cogentcore.org/core/core" "cogentcore.org/core/plot/plotcore" "cogentcore.org/core/styles" "cogentcore.org/core/tensor" + "cogentcore.org/core/tensor/datafs" "cogentcore.org/core/tensor/table" "cogentcore.org/core/tensor/tensorcore" "cogentcore.org/core/texteditor" @@ -46,6 +48,10 @@ type Tabber interface { // PlotTable recycles a tab with a Plot of given [table.Table]. PlotTable(label string, dt *table.Table) *plotcore.PlotEditor + // PlotDataFS recycles a tab with a Plot of given [datafs.Data], + // automatically using the Dir/File name of the data node for the label. + PlotDataFS(dfs *datafs.Data) *plotcore.PlotEditor + // GoUpdatePlot calls GoUpdatePlot on plot at tab with given name. // Does nothing if tab name doesn't exist (returns nil). GoUpdatePlot(label string) *plotcore.PlotEditor @@ -64,16 +70,20 @@ type Tabber interface { } // NewTab recycles a tab with given label, or returns the existing one -// with given type of widget within it. mkfun function is called to create -// and configure a new widget if not already existing. +// with given type of widget within it. The existing that is returned +// is the last one in the frame, allowing for there to be a toolbar at the top. +// mkfun function is called to create and configure a new widget +// if not already existing. func NewTab[T any](tb Tabber, label string, mkfun func(tab *core.Frame) T) T { tab := tb.RecycleTab(label) var zv T if tab.HasChildren() { - if tt, ok := tab.Child(1).(T); ok { + nc := tab.NumChildren() + lc := tab.Child(nc - 1) + if tt, ok := lc.(T); ok { return tt } - err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content", label) + err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content: is %T", label, lc) core.ErrorSnackbar(tb.AsTabs(), err) return zv } @@ -91,10 +101,13 @@ func TabAt[T any](tb Tabber, label string) T { if !tab.HasChildren() { // shouldn't happen return zv } - if tt, ok := tab.Child(1).(T); ok { + nc := tab.NumChildren() + lc := tab.Child(nc - 1) + if tt, ok := lc.(T); ok { return tt } - err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content", label) + + err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content: %T", label, lc) core.ErrorSnackbar(tb.AsTabs(), err) return zv } @@ -161,7 +174,14 @@ func (ts *Tabs) TensorGrid(label string, tsr tensor.Tensor) *tensorcore.TensorGr // PlotTable recycles a tab with a Plot of given table.Table. func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { pl := NewTab(ts, label, func(tab *core.Frame) *plotcore.PlotEditor { - return plotcore.NewSubPlot(tab) + tb := core.NewToolbar(tab) + pl := plotcore.NewPlotEditor(tab) + tab.Styler(func(s *styles.Style) { + s.Direction = styles.Column + s.Grow.Set(1, 1) + }) + tb.Maker(pl.MakeToolbar) + return pl }) if pl != nil { pl.SetTable(dt) @@ -170,6 +190,26 @@ func (ts *Tabs) PlotTable(label string, dt *table.Table) *plotcore.PlotEditor { return pl } +// PlotDataFS recycles a tab with a Plot of given [datafs.Data]. +func (ts *Tabs) PlotDataFS(dfs *datafs.Data) *plotcore.PlotEditor { + label := fsx.DirAndFile(dfs.Path()) + " Plot" + if dfs.IsDir() { + return ts.PlotTable(label, dfs.GetDirTable(nil)) + } + tsr := dfs.Data + dt := table.New(label) + dt.Columns.Rows = tsr.DimSize(0) + if ix, ok := tsr.(*tensor.Rows); ok { + dt.Indexes = ix.Indexes + } + rc := dt.AddIntColumn("Row") + for r := range dt.Columns.Rows { + rc.Values[r] = r + } + dt.AddColumn(dfs.Name(), tsr.AsValues()) + return ts.PlotTable(label, dt) +} + // GoUpdatePlot calls GoUpdatePlot on plot at tab with given name. // Does nothing if tab name doesn't exist (returns nil). func (ts *Tabs) GoUpdatePlot(label string) *plotcore.PlotEditor { From a82e893a4b0697dbfb5defa10ba07024f40257ca Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 16 Nov 2024 01:08:15 -0800 Subject: [PATCH 309/311] gosl runs CPU functions using threaded calls --- goal/gosl/examples/basic/gosl.go | 14 +++++----- goal/gosl/examples/basic/shaders/Atomic.wgsl | 2 ++ goal/gosl/examples/basic/shaders/Compute.wgsl | 2 ++ goal/gosl/gotosl/gengpu.go | 5 +--- gpu/compute.go | 27 +++++++++++++++++++ 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/goal/gosl/examples/basic/gosl.go b/goal/gosl/examples/basic/gosl.go index 62a14a3e89..249eec6248 100644 --- a/goal/gosl/examples/basic/gosl.go +++ b/goal/gosl/examples/basic/gosl.go @@ -32,7 +32,11 @@ const ( // GPUInit initializes the GPU compute system, // configuring system(s), variables and kernels. +// It is safe to call multiple times: detects if already run. func GPUInit() { + if ComputeGPU != nil { + return + } gp := gpu.NewComputeGPU() ComputeGPU = gp { @@ -86,10 +90,7 @@ func RunAtomicGPU(n int) { // RunAtomicCPU runs the Atomic kernel on the CPU. func RunAtomicCPU(n int) { - // todo: need threaded api -- not tensor - for i := range n { - Atomic(uint32(i)) - } + gpu.VectorizeFunc(0, n, Atomic) } // RunOneAtomic runs the Atomic kernel with given number of elements, @@ -131,10 +132,7 @@ func RunComputeGPU(n int) { // RunComputeCPU runs the Compute kernel on the CPU. func RunComputeCPU(n int) { - // todo: need threaded api -- not tensor - for i := range n { - Compute(uint32(i)) - } + gpu.VectorizeFunc(0, n, Compute) } // RunOneCompute runs the Compute kernel with given number of elements, diff --git a/goal/gosl/examples/basic/shaders/Atomic.wgsl b/goal/gosl/examples/basic/shaders/Atomic.wgsl index 4da119c151..464d4281ba 100644 --- a/goal/gosl/examples/basic/shaders/Atomic.wgsl +++ b/goal/gosl/examples/basic/shaders/Atomic.wgsl @@ -9,6 +9,8 @@ var Data: array; @group(0) @binding(2) var IntData: array>; +alias GPUVars = i32; + @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { Atomic(idx.x); diff --git a/goal/gosl/examples/basic/shaders/Compute.wgsl b/goal/gosl/examples/basic/shaders/Compute.wgsl index c7cc6c8618..767e8fadac 100644 --- a/goal/gosl/examples/basic/shaders/Compute.wgsl +++ b/goal/gosl/examples/basic/shaders/Compute.wgsl @@ -9,6 +9,8 @@ var Data: array; @group(0) @binding(2) var IntData: array; +alias GPUVars = i32; + @compute @workgroup_size(64, 1, 1) fn main(@builtin(global_invocation_id) idx: vec3) { Compute(idx.x); diff --git a/goal/gosl/gotosl/gengpu.go b/goal/gosl/gotosl/gengpu.go index 92e88f32ab..40901f418e 100644 --- a/goal/gosl/gotosl/gengpu.go +++ b/goal/gosl/gotosl/gengpu.go @@ -191,10 +191,7 @@ func Run%[1]sGPU(n int) { // Run%[1]sCPU runs the %[1]s kernel on the CPU. func Run%[1]sCPU(n int) { - // todo: need threaded api -- not tensor - for i := range n { - %[1]s(uint32(i)) - } + gpu.VectorizeFunc(0, n, %[1]s) } // RunOne%[1]s runs the %[1]s kernel with given number of elements, diff --git a/gpu/compute.go b/gpu/compute.go index e6f5c822d4..536623c3aa 100644 --- a/gpu/compute.go +++ b/gpu/compute.go @@ -7,6 +7,8 @@ package gpu import ( "fmt" "math" + "runtime" + "sync" "cogentcore.org/core/base/errors" "github.com/cogentcore/webgpu/wgpu" @@ -166,3 +168,28 @@ func (sy *ComputeSystem) EndComputePass() error { func Warps(n, threads int) int { return int(math.Ceil(float64(n) / float64(threads))) } + +// VectorizeFunc runs given GPU kernel function taking a uint32 index +// on the CPU, using given number of threads with goroutines, for n iterations. +// If threads is 0, then GOMAXPROCS is used. +func VectorizeFunc(threads, n int, fun func(idx uint32)) { + if threads == 0 { + threads = runtime.GOMAXPROCS(0) + } + nper := int(math.Ceil(float64(n) / float64(threads))) + wait := sync.WaitGroup{} + for start := 0; start < n; start += nper { + end := start + nper + if end > n { + end = n + } + wait.Add(1) + go func() { + for idx := start; idx < end; idx++ { + fun(uint32(idx)) + } + wait.Done() + }() + } + wait.Wait() +} From 8d540d57f004f3c6db965ff45bbbc984d8907ed6 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 16 Nov 2024 03:07:50 -0800 Subject: [PATCH 310/311] plot: fix hover to actually report data instead of pixels! --- plot/plotcore/plot.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/plot/plotcore/plot.go b/plot/plotcore/plot.go index e0d4c1b30b..286ae8f3a9 100644 --- a/plot/plotcore/plot.go +++ b/plot/plotcore/plot.go @@ -128,12 +128,20 @@ func (pt *Plot) WidgetTooltip(pos image.Point) (string, image.Point) { return pt.Tooltip, pt.DefaultTooltipPos() } wpos := pos.Sub(pt.Geom.ContentBBox.Min) - plt, _, idx, dist, data, _, legend := pt.Plot.ClosestDataToPixel(wpos.X, wpos.Y) + plt, _, idx, dist, _, data, legend := pt.Plot.ClosestDataToPixel(wpos.X, wpos.Y) if dist <= 10 { pt.Plot.HighlightPlotter = plt pt.Plot.HighlightIndex = idx pt.updatePlot() - return fmt.Sprintf("%s[%d]: (%g, %g)", legend, idx, data.X, data.Y), pos + dx := 0.0 + if data[plot.X] != nil { + dx = data[plot.X].Float1D(idx) + } + dy := 0.0 + if data[plot.Y] != nil { + dy = data[plot.Y].Float1D(idx) + } + return fmt.Sprintf("%s[%d]: (%g, %g)", legend, idx, dx, dy), pos } else { if pt.Plot.HighlightPlotter != nil { pt.Plot.HighlightPlotter = nil From 4a7f4917774843a414b31ae1aa15fcf10d6b9022 Mon Sep 17 00:00:00 2001 From: "Randall C. O'Reilly" Date: Sat, 16 Nov 2024 03:41:40 -0800 Subject: [PATCH 311/311] core.Tabs also needs a Tabber interface to get parent Tabs when it could be a derived type, as in databrowser.Tabs. --- core/tabs.go | 12 +++++++++++- tensor/databrowser/tabs.go | 10 +++++----- tensor/datafs/dir.go | 8 +++++--- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/core/tabs.go b/core/tabs.go index 4970a8fefe..4a19a7080b 100644 --- a/core/tabs.go +++ b/core/tabs.go @@ -20,6 +20,11 @@ import ( "cogentcore.org/core/tree" ) +// Tabber is an interface for getting the parent Tabs of tab buttons. +type Tabber interface { + AsCoreTabs() *Tabs +} + // Tabs divide widgets into logical groups and give users the ability // to freely navigate between them using tab buttons. type Tabs struct { @@ -101,6 +106,8 @@ func (tt TabTypes) isColumn() bool { return tt == NavigationDrawer } +func (ts *Tabs) AsCoreTabs() *Tabs { return ts } + func (ts *Tabs) Init() { ts.Frame.Init() ts.maxChars = 16 @@ -543,5 +550,8 @@ func (tb *Tab) Init() { // tabs returns the parent [Tabs] of this [Tab]. func (tb *Tab) tabs() *Tabs { - return tb.Parent.AsTree().Parent.(*Tabs) + if tbr, ok := tb.Parent.AsTree().Parent.(Tabber); ok { + return tbr.AsCoreTabs() + } + return nil } diff --git a/tensor/databrowser/tabs.go b/tensor/databrowser/tabs.go index aaac020395..69f3d0d42e 100644 --- a/tensor/databrowser/tabs.go +++ b/tensor/databrowser/tabs.go @@ -23,8 +23,8 @@ import ( // among others. type Tabber interface { - // AsTabs returns the underlying [Tabs] widget. - AsTabs() *Tabs + // AsDataTabs returns the underlying [databrowser.Tabs] widget. + AsDataTabs() *Tabs // TabByName returns a tab with the given name, nil if not found. TabByName(name string) *core.Frame @@ -84,7 +84,7 @@ func NewTab[T any](tb Tabber, label string, mkfun func(tab *core.Frame) T) T { return tt } err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content: is %T", label, lc) - core.ErrorSnackbar(tb.AsTabs(), err) + core.ErrorSnackbar(tb.AsDataTabs(), err) return zv } w := mkfun(tab) @@ -108,7 +108,7 @@ func TabAt[T any](tb Tabber, label string) T { } err := fmt.Errorf("Name / Type conflict: tab %q does not have the expected type of content: %T", label, lc) - core.ErrorSnackbar(tb.AsTabs(), err) + core.ErrorSnackbar(tb.AsDataTabs(), err) return zv } @@ -122,7 +122,7 @@ func (ts *Tabs) Init() { ts.Type = core.FunctionalTabs } -func (ts *Tabs) AsTabs() *Tabs { +func (ts *Tabs) AsDataTabs() *Tabs { return ts } diff --git a/tensor/datafs/dir.go b/tensor/datafs/dir.go index f2726ca19f..664fba87ab 100644 --- a/tensor/datafs/dir.go +++ b/tensor/datafs/dir.go @@ -386,11 +386,13 @@ func (d *Data) Recycle(it *Data) *Data { // Row count is updated to current max row. // Set DirTable = nil to regenerate. func (d *Data) GetDirTable(fun func(item *Data) bool) *table.Table { + its := d.FlatItemsFunc(fun) if d.DirTable != nil { - d.DirTable.SetNumRowsToMax() - return d.DirTable + if d.DirTable.NumColumns() == len(its) { + d.DirTable.SetNumRowsToMax() + return d.DirTable + } } - its := d.FlatItemsFunc(fun) dt := table.New(fsx.DirAndFile(string(d.Path()))) for _, it := range its { tsr := it.Data