diff --git a/CHANGELOG.md b/CHANGELOG.md index da95cf4a..26eff1c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ Types of changes - `Added` columns information and export type using the `lino table extract` command, columns and keys organized according to the database order. - `Added` flag `--only-tables` to `lino table extract` command. This flag allows for the extraction of table information exclusively, excluding columns. It has been included to maintain the previous behavior. - `Added` flag `--with-db-infos` to `lino table extract` command. This flag enables the extraction of information regarding column types, length, size, and precision if the column has been configured with these specifications. +- `Added` flag `--autotruncate` to `lino push` command. This flag will enable a truncate on each value based each `dbinfo`.`length` parameters set in the table.yaml file for each columns. +- `Added` property `dbinfo`.`bytes` to column definition in table.yaml file. Set it to true to truncate the value based on a maximum number of bytes and not characters (assuming utf-8 encoding for now). +- `Added` flags `--max-length` and `--bytes` to `lino table add-column` command. Use it to edit the properties `dbinfo`.`length` and `dbinfo`.`bytes` of the table.yaml file. ## [2.6.1] diff --git a/README.md b/README.md index 0c99376b..7fc4763d 100755 --- a/README.md +++ b/README.md @@ -343,6 +343,36 @@ Each line is a filter and `lino` apply it to the start table to extract data. The `push` sub-command import a **json** line stream (jsonline format http://jsonlines.org/) in each table, following the ingress descriptor defined in current directory. +### Autotruncate values + +Use the `autotruncate` flag to automatically truncate string values that overflows the maximum length accepted by the database. + +``` +$ lino push truncate dest --table actor --autotruncate < actors.jsonl +``` + +LINO will truncate each value based each `dbinfo`.`length` parameters set in the table.yaml file for each columns. + +Additionnaly, if your database maximum value is not defined in number of characters but in number of bytes, set the `dbinfo`.`bytes` to true. LINO will truncate the value based on a maximum number of bytes and not characters (assuming utf-8 encoding for now). + +```yaml +version: v1 +tables: + - name: actor + keys: + - actor_id + columns: + - name: actor_id + dbinfo: + type: INT4 + - name: first_name + export: string + dbinfo: + type: VARCHAR + length: 45 + bytes: true +``` + ### How to update primary key Let's say you have this record in database : diff --git a/internal/app/push/cli.go b/internal/app/push/cli.go index 52f8a1fe..552947b1 100755 --- a/internal/app/push/cli.go +++ b/internal/app/push/cli.go @@ -80,6 +80,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra pkTranslations map[string]string whereField string savepoint string + autoTruncate bool ) cmd := &cobra.Command{ @@ -127,7 +128,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra os.Exit(1) } - plan, e2 := getPlan(idStorageFactory(table, ingressDescriptor)) + plan, e2 := getPlan(idStorageFactory(table, ingressDescriptor), autoTruncate) if e2 != nil { fmt.Fprintln(err, e2.Error()) os.Exit(2) @@ -151,7 +152,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra os.Exit(1) } - e3 := push.Push(rowIteratorFactory(in), datadestination, plan, mode, commitSize, disableConstraints, rowExporter, translator, whereField, savepoint) + e3 := push.Push(rowIteratorFactory(in), datadestination, plan, mode, commitSize, disableConstraints, rowExporter, translator, whereField, savepoint, autoTruncate) if e3 != nil { log.Fatal().AnErr("error", e3).Msg("Fatal error stop the push command") os.Exit(1) @@ -172,6 +173,7 @@ func NewCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra cmd.Flags().StringToStringVar(&pkTranslations, "pk-translation", map[string]string{}, "list of dictionaries old value / new value for primary key update") cmd.Flags().StringVar(&whereField, "using-pk-field", "__usingpk__", "Name of the data field that can be used as pk for update queries") cmd.Flags().StringVar(&savepoint, "savepoint", "", "Name of a file to write primary keys of effectively processed lines (commit to database)") + cmd.Flags().BoolVarP(&autoTruncate, "autotruncate", "a", false, "Automatically truncate values to the maximum length defined in table.yaml") cmd.SetOut(out) cmd.SetErr(err) cmd.SetIn(in) @@ -241,7 +243,7 @@ func getDataDestination(dataconnectorName string) (push.DataDestination, *push.E return datadestinationFactory.New(u.URL.String(), alias.Schema), nil } -func getPlan(idStorage id.Storage) (push.Plan, *push.Error) { +func getPlan(idStorage id.Storage, autoTruncate bool) (push.Plan, *push.Error) { id, err1 := idStorage.Read() if err1 != nil { return nil, &push.Error{Description: err1.Error()} @@ -274,7 +276,7 @@ func getPlan(idStorage id.Storage) (push.Plan, *push.Error) { pushtmap: map[string]push.Table{}, } - return converter.getPlan(id), nil + return converter.getPlan(id, autoTruncate), nil } type idToPushConverter struct { @@ -285,7 +287,7 @@ type idToPushConverter struct { pushtmap map[string]push.Table } -func (c idToPushConverter) getTable(name string) push.Table { +func (c idToPushConverter) getTable(name string, autoTruncate bool) push.Table { if pushtable, ok := c.pushtmap[name]; ok { return pushtable } @@ -300,13 +302,13 @@ func (c idToPushConverter) getTable(name string) push.Table { columns := []push.Column{} for _, col := range table.Columns { - columns = append(columns, push.NewColumn(col.Name, col.Export, col.Import)) + columns = append(columns, push.NewColumn(col.Name, col.Export, col.Import, col.DBInfo.Length, col.DBInfo.ByteBased, autoTruncate)) } return push.NewTable(table.Name, table.Keys, push.NewColumnList(columns)) } -func (c idToPushConverter) getRelation(name string) push.Relation { +func (c idToPushConverter) getRelation(name string, autoTruncate bool) push.Relation { if pushrelation, ok := c.pushrmap[name]; ok { return pushrelation } @@ -321,12 +323,12 @@ func (c idToPushConverter) getRelation(name string) push.Relation { return push.NewRelation( relation.Name, - c.getTable(relation.Parent.Name), - c.getTable(relation.Child.Name), + c.getTable(relation.Parent.Name, autoTruncate), + c.getTable(relation.Child.Name, autoTruncate), ) } -func (c idToPushConverter) getPlan(idesc id.IngressDescriptor) push.Plan { +func (c idToPushConverter) getPlan(idesc id.IngressDescriptor, autoTruncate bool) push.Plan { relations := []push.Relation{} activeTables, err := id.GetActiveTables(idesc) @@ -338,9 +340,9 @@ func (c idToPushConverter) getPlan(idesc id.IngressDescriptor) push.Plan { rel := idesc.Relations().Relation(idx) if (activeTables.Contains(rel.Child().Name()) && rel.LookUpChild()) || (activeTables.Contains(rel.Parent().Name()) && rel.LookUpParent()) { - relations = append(relations, c.getRelation(rel.Name())) + relations = append(relations, c.getRelation(rel.Name(), autoTruncate)) } } - return push.NewPlan(c.getTable(idesc.StartTable().Name()), relations) + return push.NewPlan(c.getTable(idesc.StartTable().Name(), autoTruncate), relations) } diff --git a/internal/app/push/http.go b/internal/app/push/http.go index b2ee7d4e..92b732f3 100644 --- a/internal/app/push/http.go +++ b/internal/app/push/http.go @@ -81,7 +81,23 @@ func Handler(w http.ResponseWriter, r *http.Request, mode push.Mode, ingressDesc return } - plan, e2 := getPlan(idStorageFactory(query.Get("table"), ingressDescriptor)) + autoTruncate := false + if query.Get("auto-truncate") != "" { + var err error + autoTruncate, err = strconv.ParseBool(query.Get("auto-truncate")) + if err != nil { + log.Error().Err(err).Msg("can't parse auto-truncate") + w.WriteHeader(http.StatusBadRequest) + _, ew := w.Write([]byte("{\"error\" : \"param auto-truncate must be a boolean\"}\n")) + if ew != nil { + log.Error().Err(ew).Msg("Write failed") + return + } + return + } + } + + plan, e2 := getPlan(idStorageFactory(query.Get("table"), ingressDescriptor), autoTruncate) if e2 != nil { log.Error().Err(e2).Msg("") w.WriteHeader(http.StatusNotFound) @@ -130,7 +146,7 @@ func Handler(w http.ResponseWriter, r *http.Request, mode push.Mode, ingressDesc log.Debug().Msg(fmt.Sprintf("call Push with mode %s", mode)) - e3 := push.Push(rowIteratorFactory(r.Body), datadestination, plan, mode, commitSize, disableConstraints, push.NoErrorCaptureRowWriter{}, nil, query.Get("using-pk-field"), "") + e3 := push.Push(rowIteratorFactory(r.Body), datadestination, plan, mode, commitSize, disableConstraints, push.NoErrorCaptureRowWriter{}, nil, query.Get("using-pk-field"), "", false) if e3 != nil { log.Error().Err(e3).Msg("") w.WriteHeader(http.StatusNotFound) diff --git a/internal/app/table/add-columns.go b/internal/app/table/add-columns.go index 5b1f08f2..8f8186dd 100644 --- a/internal/app/table/add-columns.go +++ b/internal/app/table/add-columns.go @@ -29,6 +29,8 @@ import ( func newAddColumnCommand(fullName string, err *os.File, out *os.File, in *os.File) *cobra.Command { // local flags var exportType, importType string + var maxLength int64 + var byteBased bool cmd := &cobra.Command{ Use: "add-column [Table Name] [Column Name]", @@ -40,7 +42,7 @@ func newAddColumnCommand(fullName string, err *os.File, out *os.File, in *os.Fil tableName := args[0] columnName := args[1] - _, e1 := table.AddOrUpdateColumn(tableStorage, tableName, columnName, exportType, importType) + _, e1 := table.AddOrUpdateColumn(tableStorage, tableName, columnName, exportType, importType, maxLength, byteBased) if e1 != nil { fmt.Fprintln(err, e1.Description) os.Exit(1) @@ -54,5 +56,7 @@ func newAddColumnCommand(fullName string, err *os.File, out *os.File, in *os.Fil cmd.SetIn(in) cmd.Flags().StringVarP(&exportType, "export", "e", "", "export type for the column") cmd.Flags().StringVarP(&importType, "import", "i", "", "import type for the column") + cmd.Flags().Int64VarP(&maxLength, "max-length", "l", 0, "set optional maximum length for this column that can be used with --autotruncate flag on push") + cmd.Flags().BoolVarP(&byteBased, "bytes", "b", false, "maximum length is expressed in bytes, not in characters") return cmd } diff --git a/internal/infra/table/extractor_postgres.go b/internal/infra/table/extractor_postgres.go index cdf92594..dfb6f697 100755 --- a/internal/infra/table/extractor_postgres.go +++ b/internal/infra/table/extractor_postgres.go @@ -79,12 +79,12 @@ func (d PostgresDialect) GetExportType(dbtype string) (string, bool) { return "string", true // Numeric types case "NUMERIC", "DECIMAL", "FLOAT", "REAL", "DOUBLE PRECISION", "MONEY", "INTEGER", "BIGINT", - "NUMBER", "BINARY_FLOAT", "BINARY_DOUBLE", "INT", "TINYINT", "SMALLINT", "MEDIUMINT": + "NUMBER", "BINARY_FLOAT", "BINARY_DOUBLE", "INT", "TINYINT", "SMALLINT", "MEDIUMINT", "INT4", "INT2", "BOOL": return "numeric", true // Timestamp types case "TIMESTAMP", "TIMESTAMPTZ", "TIMESTAMP WITH TIME ZONE", "TIMESTAMP WITH LOCAL TIME ZONE": - return "timestamp", true + return "datetime", true // export timestamps to datetime is the only option that works well with lino push // Datetime types case "DATE", "DATETIME2", "SMALLDATETIME", "DATETIME": return "datetime", true @@ -92,6 +92,6 @@ func (d PostgresDialect) GetExportType(dbtype string) (string, bool) { case "BYTEA", "BLOB": return "base64", true default: - return "", false + return "string", true // default to export string since it will work most of the time (binary types are already handled) } } diff --git a/internal/infra/table/storage_yaml.go b/internal/infra/table/storage_yaml.go index aeb5ba13..68f226b7 100755 --- a/internal/infra/table/storage_yaml.go +++ b/internal/infra/table/storage_yaml.go @@ -56,6 +56,7 @@ type YAMLDBInfo struct { Length int64 `yaml:"length,omitempty"` Size int64 `yaml:"size,omitempty"` Precision int64 `yaml:"precision,omitempty"` + ByteBased bool `yaml:"bytes,omitempty"` } // YAMLStorage provides storage in a local YAML file diff --git a/pkg/push/driver.go b/pkg/push/driver.go index 209530b7..94a1a078 100755 --- a/pkg/push/driver.go +++ b/pkg/push/driver.go @@ -26,7 +26,7 @@ import ( ) // Push write rows to target table -func Push(ri RowIterator, destination DataDestination, plan Plan, mode Mode, commitSize uint, disableConstraints bool, catchError RowWriter, translator Translator, whereField string, savepointPath string) (err *Error) { +func Push(ri RowIterator, destination DataDestination, plan Plan, mode Mode, commitSize uint, disableConstraints bool, catchError RowWriter, translator Translator, whereField string, savepointPath string, autotruncate bool) (err *Error) { err1 := destination.Open(plan, mode, disableConstraints) if err1 != nil { return err1 diff --git a/pkg/push/driver_test.go b/pkg/push/driver_test.go index f6133d0e..0aaef714 100755 --- a/pkg/push/driver_test.go +++ b/pkg/push/driver_test.go @@ -49,7 +49,7 @@ func TestSimplePush(t *testing.T) { } dest := memoryDataDestination{tables, false, false, false} - err := push.Push(&ri, &dest, plan, push.Insert, 2, true, push.NoErrorCaptureRowWriter{}, nil, "", "") + err := push.Push(&ri, &dest, plan, push.Insert, 2, true, push.NoErrorCaptureRowWriter{}, nil, "", "", false) assert.Nil(t, err) assert.Equal(t, true, dest.closed) @@ -88,7 +88,7 @@ func TestRelationPush(t *testing.T) { } dest := memoryDataDestination{tables, false, false, false} - err := push.Push(&ri, &dest, plan, push.Insert, 2, true, push.NoErrorCaptureRowWriter{}, nil, "", "") + err := push.Push(&ri, &dest, plan, push.Insert, 2, true, push.NoErrorCaptureRowWriter{}, nil, "", "", false) // no error assert.Nil(t, err) @@ -137,7 +137,7 @@ func TestRelationPushWithEmptyRelation(t *testing.T) { } dest := memoryDataDestination{tables, false, false, false} - err := push.Push(&ri, &dest, plan, push.Insert, 2, true, push.NoErrorCaptureRowWriter{}, nil, "", "") + err := push.Push(&ri, &dest, plan, push.Insert, 2, true, push.NoErrorCaptureRowWriter{}, nil, "", "", false) // no error assert.Nil(t, err) @@ -188,7 +188,7 @@ func TestInversseRelationPush(t *testing.T) { } dest := memoryDataDestination{tables, false, false, false} - err := push.Push(&ri, &dest, plan, push.Insert, 5, true, push.NoErrorCaptureRowWriter{}, nil, "", "") + err := push.Push(&ri, &dest, plan, push.Insert, 5, true, push.NoErrorCaptureRowWriter{}, nil, "", "", false) // no error assert.Nil(t, err) diff --git a/pkg/push/model.go b/pkg/push/model.go index 8ed8ba5b..b1fc7d10 100755 --- a/pkg/push/model.go +++ b/pkg/push/model.go @@ -45,6 +45,9 @@ type Column interface { Name() string Export() string Import() string + Length() int64 + LengthInBytes() bool + Truncate() bool } // Plan describe how to push data diff --git a/pkg/push/model_table.go b/pkg/push/model_table.go index 73cd40a6..42e26c61 100755 --- a/pkg/push/model_table.go +++ b/pkg/push/model_table.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" "time" + "unicode/utf8" "github.com/cgi-fr/jsonline/pkg/jsonline" "github.com/rs/zerolog/log" @@ -89,19 +90,25 @@ func (l columnList) String() string { } type column struct { - name string - exp string - imp string + name string + exp string + imp string + lgth int64 + inbytes bool + truncate bool } // NewColumn initialize a new Column object -func NewColumn(name string, exp string, imp string) Column { - return column{name, exp, imp} +func NewColumn(name string, exp string, imp string, lgth int64, inbytes bool, truncate bool) Column { + return column{name, exp, imp, lgth, inbytes, truncate} } -func (c column) Name() string { return c.name } -func (c column) Export() string { return c.exp } -func (c column) Import() string { return c.imp } +func (c column) Name() string { return c.name } +func (c column) Export() string { return c.exp } +func (c column) Import() string { return c.imp } +func (c column) Length() int64 { return c.lgth } +func (c column) LengthInBytes() bool { return c.inbytes } +func (c column) Truncate() bool { return c.truncate } type ImportedRow struct { jsonline.Row @@ -176,6 +183,16 @@ func (t table) Import(row map[string]interface{}) (ImportedRow, *Error) { } result.SetValue(key, jsonline.NewValueAuto(bytes)) } + + // autotruncate + value, exists := result.GetValue(key) + if exists && col.Truncate() && col.Length() > 0 && value.GetFormat() == jsonline.String { + if col.LengthInBytes() { + result.Set(key, truncateUTF8String(result.GetString(key), int(col.Length()))) + } else { + result.Set(key, truncateRuneString(result.GetString(key), int(col.Length()))) + } + } } } @@ -234,3 +251,30 @@ func parseFormatWithType(option string) (string, string) { } return parts[0], strings.Trim(parts[1], ")") } + +// truncateUTF8String truncate s to n bytes or less. If len(s) is more than n, +// truncate before the start of the first rune that doesn't fit. s should +// consist of valid utf-8. +func truncateUTF8String(s string, n int) string { + if len(s) <= n { + return s + } + for n > 0 && !utf8.RuneStart(s[n]) { + n-- + } + + return s[:n] +} + +// truncateRuneString truncate s to n runes or less. +func truncateRuneString(s string, n int) string { + if n <= 0 { + return "" + } + + if utf8.RuneCountInString(s) < n { + return s + } + + return string([]rune(s)[:n]) +} diff --git a/pkg/table/driver.go b/pkg/table/driver.go index 78fd950b..c8b65263 100755 --- a/pkg/table/driver.go +++ b/pkg/table/driver.go @@ -60,7 +60,7 @@ func Count(s Storage, e Extractor) ([]TableCount, *Error) { } // AddOrUpdateColumn will update table definitions with given export and import types, it will add the column if necessary -func AddOrUpdateColumn(s Storage, tableName, columnName, exportType, importType string) (int, *Error) { +func AddOrUpdateColumn(s Storage, tableName, columnName, exportType, importType string, maxLength int64, inBytes bool) (int, *Error) { tables, err := s.List() if err != nil { return 0, err @@ -71,7 +71,7 @@ func AddOrUpdateColumn(s Storage, tableName, columnName, exportType, importType updatedTables := []Table{} for _, table := range tables { if table.Name == tableName { - updatedTables = append(updatedTables, addOrUpdateColumn(table, columnName, exportType, importType)) + updatedTables = append(updatedTables, addOrUpdateColumn(table, columnName, exportType, importType, maxLength, inBytes)) count++ } else { updatedTables = append(updatedTables, table) @@ -89,7 +89,7 @@ func AddOrUpdateColumn(s Storage, tableName, columnName, exportType, importType return count, nil } -func addOrUpdateColumn(table Table, columnName, exportType, importType string) Table { +func addOrUpdateColumn(table Table, columnName, exportType, importType string, maxLength int64, inBytes bool) Table { count := 0 updatedColumns := []Column{} @@ -104,10 +104,15 @@ func addOrUpdateColumn(table Table, columnName, exportType, importType string) T if importType != "" { importUpdate = importType } + if maxLength > 0 { + column.DBInfo.Length = maxLength + column.DBInfo.ByteBased = inBytes + } updatedColumns = append(updatedColumns, Column{ Name: columnName, Export: exportUpdate, Import: importUpdate, + DBInfo: column.DBInfo, }) count++ } else { diff --git a/pkg/table/model.go b/pkg/table/model.go index 403986b1..a7dbe2e4 100755 --- a/pkg/table/model.go +++ b/pkg/table/model.go @@ -23,6 +23,7 @@ type DBInfo struct { Length int64 Size int64 Precision int64 + ByteBased bool } // Column holds the name of a column. diff --git a/tests/data/expected.yml b/tests/data/expected.yml index b541a983..a5a9994e 100644 --- a/tests/data/expected.yml +++ b/tests/data/expected.yml @@ -5,17 +5,19 @@ tables: - actor_id columns: - name: actor_id + export: numeric - name: first_name export: string - name: last_name export: string - name: last_update - export: timestamp + export: datetime - name: address keys: - address_id columns: - name: address_id + export: numeric - name: address export: string - name: address2 @@ -23,46 +25,53 @@ tables: - name: district export: string - name: city_id + export: numeric - name: postal_code export: string - name: phone export: string - name: last_update - export: timestamp + export: datetime - name: category keys: - category_id columns: - name: category_id + export: numeric - name: name export: string - name: last_update - export: timestamp + export: datetime - name: city keys: - city_id columns: - name: city_id + export: numeric - name: city export: string - name: country_id + export: numeric - name: last_update - export: timestamp + export: datetime - name: country keys: - country_id columns: - name: country_id + export: numeric - name: country export: string - name: last_update - export: timestamp + export: datetime - name: customer keys: - customer_id columns: - name: customer_id + export: numeric - name: store_id + export: numeric - name: first_name export: string - name: last_name @@ -70,33 +79,43 @@ tables: - name: email export: string - name: address_id + export: numeric - name: activebool + export: numeric - name: create_date export: datetime - name: last_update - export: timestamp + export: datetime - name: active + export: numeric - name: film keys: - film_id columns: - name: film_id + export: numeric - name: title export: string - name: description export: string - name: release_year + export: numeric - name: language_id + export: numeric - name: original_language_id + export: numeric - name: rental_duration + export: numeric - name: rental_rate export: numeric - name: length + export: numeric - name: replacement_cost export: numeric - name: rating + export: string - name: last_update - export: timestamp + export: datetime - name: special_features export: string - name: fulltext @@ -107,82 +126,102 @@ tables: - film_id columns: - name: actor_id + export: numeric - name: film_id + export: numeric - name: last_update - export: timestamp + export: datetime - name: film_category keys: - film_id - category_id columns: - name: film_id + export: numeric - name: category_id + export: numeric - name: last_update - export: timestamp + export: datetime - name: inventory keys: - inventory_id columns: - name: inventory_id + export: numeric - name: film_id + export: numeric - name: store_id + export: numeric - name: last_update - export: timestamp + export: datetime - name: language keys: - language_id columns: - name: language_id + export: numeric - name: name export: string - name: last_update - export: timestamp + export: datetime - name: payment keys: - payment_id columns: - name: payment_id + export: numeric - name: customer_id + export: numeric - name: staff_id + export: numeric - name: rental_id + export: numeric - name: amount export: numeric - name: payment_date - export: timestamp + export: datetime - name: rental keys: - rental_id columns: - name: rental_id + export: numeric - name: rental_date - export: timestamp + export: datetime - name: inventory_id + export: numeric - name: customer_id + export: numeric - name: return_date - export: timestamp + export: datetime - name: staff_id + export: numeric - name: last_update - export: timestamp + export: datetime - name: staff keys: - staff_id columns: - name: staff_id + export: numeric - name: first_name export: string - name: last_name export: string - name: address_id + export: numeric - name: email export: string - name: store_id + export: numeric - name: active + export: numeric - name: username export: string - name: password export: string - name: last_update - export: timestamp + export: datetime - name: picture export: base64 - name: store @@ -190,7 +229,10 @@ tables: - store_id columns: - name: store_id + export: numeric - name: manager_staff_id + export: numeric - name: address_id + export: numeric - name: last_update - export: timestamp + export: datetime diff --git a/tests/data/expected_with_db_infos.yaml b/tests/data/expected_with_db_infos.yaml index 3d7c6b70..39b6c434 100644 --- a/tests/data/expected_with_db_infos.yaml +++ b/tests/data/expected_with_db_infos.yaml @@ -5,6 +5,7 @@ tables: - actor_id columns: - name: actor_id + export: numeric dbinfo: type: INT4 - name: first_name @@ -18,7 +19,7 @@ tables: type: VARCHAR length: 45 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: address @@ -26,6 +27,7 @@ tables: - address_id columns: - name: address_id + export: numeric dbinfo: type: INT4 - name: address @@ -44,6 +46,7 @@ tables: type: VARCHAR length: 20 - name: city_id + export: numeric dbinfo: type: INT2 - name: postal_code @@ -57,7 +60,7 @@ tables: type: VARCHAR length: 20 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: category @@ -65,6 +68,7 @@ tables: - category_id columns: - name: category_id + export: numeric dbinfo: type: INT4 - name: name @@ -73,7 +77,7 @@ tables: type: VARCHAR length: 25 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: city @@ -81,6 +85,7 @@ tables: - city_id columns: - name: city_id + export: numeric dbinfo: type: INT4 - name: city @@ -89,10 +94,11 @@ tables: type: VARCHAR length: 50 - name: country_id + export: numeric dbinfo: type: INT2 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: country @@ -100,6 +106,7 @@ tables: - country_id columns: - name: country_id + export: numeric dbinfo: type: INT4 - name: country @@ -108,7 +115,7 @@ tables: type: VARCHAR length: 50 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: customer @@ -116,9 +123,11 @@ tables: - customer_id columns: - name: customer_id + export: numeric dbinfo: type: INT4 - name: store_id + export: numeric dbinfo: type: INT2 - name: first_name @@ -137,9 +146,11 @@ tables: type: VARCHAR length: 50 - name: address_id + export: numeric dbinfo: type: INT2 - name: activebool + export: numeric dbinfo: type: BOOL - name: create_date @@ -147,10 +158,11 @@ tables: dbinfo: type: DATE - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: active + export: numeric dbinfo: type: INT4 - name: film @@ -158,6 +170,7 @@ tables: - film_id columns: - name: film_id + export: numeric dbinfo: type: INT4 - name: title @@ -171,15 +184,19 @@ tables: type: TEXT length: 9223372036854775807 - name: release_year + export: numeric dbinfo: type: INT4 - name: language_id + export: numeric dbinfo: type: INT2 - name: original_language_id + export: numeric dbinfo: type: INT2 - name: rental_duration + export: numeric dbinfo: type: INT2 - name: rental_rate @@ -189,6 +206,7 @@ tables: size: 2 precision: 4 - name: length + export: numeric dbinfo: type: INT2 - name: replacement_cost @@ -198,8 +216,9 @@ tables: size: 2 precision: 5 - name: rating + export: string - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: special_features @@ -216,13 +235,15 @@ tables: - film_id columns: - name: actor_id + export: numeric dbinfo: type: INT2 - name: film_id + export: numeric dbinfo: type: INT2 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: film_category @@ -231,13 +252,15 @@ tables: - category_id columns: - name: film_id + export: numeric dbinfo: type: INT2 - name: category_id + export: numeric dbinfo: type: INT2 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: inventory @@ -245,16 +268,19 @@ tables: - inventory_id columns: - name: inventory_id + export: numeric dbinfo: type: INT4 - name: film_id + export: numeric dbinfo: type: INT2 - name: store_id + export: numeric dbinfo: type: INT2 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: language @@ -262,6 +288,7 @@ tables: - language_id columns: - name: language_id + export: numeric dbinfo: type: INT4 - name: name @@ -270,7 +297,7 @@ tables: type: BPCHAR length: 20 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: payment @@ -278,15 +305,19 @@ tables: - payment_id columns: - name: payment_id + export: numeric dbinfo: type: INT4 - name: customer_id + export: numeric dbinfo: type: INT2 - name: staff_id + export: numeric dbinfo: type: INT2 - name: rental_id + export: numeric dbinfo: type: INT4 - name: amount @@ -296,7 +327,7 @@ tables: size: 2 precision: 5 - name: payment_date - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: rental @@ -304,27 +335,31 @@ tables: - rental_id columns: - name: rental_id + export: numeric dbinfo: type: INT4 - name: rental_date - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: inventory_id + export: numeric dbinfo: type: INT4 - name: customer_id + export: numeric dbinfo: type: INT2 - name: return_date - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: staff_id + export: numeric dbinfo: type: INT2 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: staff @@ -332,6 +367,7 @@ tables: - staff_id columns: - name: staff_id + export: numeric dbinfo: type: INT4 - name: first_name @@ -345,6 +381,7 @@ tables: type: VARCHAR length: 45 - name: address_id + export: numeric dbinfo: type: INT2 - name: email @@ -353,9 +390,11 @@ tables: type: VARCHAR length: 50 - name: store_id + export: numeric dbinfo: type: INT2 - name: active + export: numeric dbinfo: type: BOOL - name: username @@ -369,7 +408,7 @@ tables: type: VARCHAR length: 40 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP - name: picture @@ -382,15 +421,18 @@ tables: - store_id columns: - name: store_id + export: numeric dbinfo: type: INT4 - name: manager_staff_id + export: numeric dbinfo: type: INT2 - name: address_id + export: numeric dbinfo: type: INT2 - name: last_update - export: timestamp + export: datetime dbinfo: type: TIMESTAMP diff --git a/tests/suites/push/autotruncate.yml b/tests/suites/push/autotruncate.yml new file mode 100644 index 00000000..04e933ce --- /dev/null +++ b/tests/suites/push/autotruncate.yml @@ -0,0 +1,64 @@ +# Copyright (C) 2024 CGI France +# +# This file is part of LINO. +# +# LINO is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# LINO is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with LINO. If not, see . + +name: push with autotruncate option +testcases: + - name: prepare test + steps: + # Clean working directory + - script: rm -f *yaml + - script: lino dataconnector add dest 'postgresql://postgres:sakila@dest:5432/postgres?sslmode=disable' + - script: lino relation extract dest + - script: lino table extract dest --with-db-infos + - script: lino table remove-column actor last_update + + - name: truncate field to maximum length + steps: + - script: echo '{"actor_id":1,"first_name":"VERY VERY VERY VERY VERY VERY VERY VERY LONG NAME","last_name":"GUINESS"}' | lino push truncate dest --table actor --autotruncate + - script: lino pull dest --table actor --filter actor_id=1 + assertions: + - result.code ShouldEqual 0 + - result.systemout ShouldEqual {"actor_id":1,"first_name":"VERY VERY VERY VERY VERY VERY VERY VERY LONG ","last_name":"GUINESS"} + - result.systemerr ShouldBeEmpty + + - name: truncate field to maximum length with accents + steps: + - script: echo '{"actor_id":1,"first_name":"VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY LONG NAME","last_name":"GUINESS"}' | lino push truncate dest --table actor --autotruncate + - script: lino pull dest --table actor --filter actor_id=1 + assertions: + - result.code ShouldEqual 0 + - result.systemout ShouldEqual {"actor_id":1,"first_name":"VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY LONG ","last_name":"GUINESS"} + - result.systemerr ShouldBeEmpty + + - name: truncate field to maximum length with accents in bytes + steps: + - script: lino table add-column actor first_name --max-length 45 --bytes + - script: echo '{"actor_id":1,"first_name":"VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY LONG NAME","last_name":"GUINESS"}' | lino push truncate dest --table actor --autotruncate + - script: lino pull dest --table actor --filter actor_id=1 + assertions: + - result.code ShouldEqual 0 + - result.systemout ShouldEqual {"actor_id":1,"first_name":"VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉ","last_name":"GUINESS"} + - result.systemerr ShouldBeEmpty + + - name: truncate field to maximum length with accents in bytes do not split codepoint + steps: + - script: echo '{"actor_id":1,"first_name":"VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY 11É","last_name":"GUINESS"}' | lino push truncate dest --table actor --autotruncate + - script: lino pull dest --table actor --filter actor_id=1 + assertions: + - result.code ShouldEqual 0 + - result.systemout ShouldEqual {"actor_id":1,"first_name":"VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY VÉRY 11","last_name":"GUINESS"} + - result.systemerr ShouldBeEmpty