Skip to content

Commit

Permalink
Merge pull request #184 from jrnd-io/183-feat-add-lua-script-producer
Browse files Browse the repository at this point in the history
feat: add lua script producer
  • Loading branch information
vmarchese authored Aug 21, 2024
2 parents 1ecd59c + 01be427 commit 367ae4d
Show file tree
Hide file tree
Showing 9 changed files with 349 additions and 0 deletions.
23 changes: 23 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ require (
github.com/spf13/cobra v1.8.1
github.com/spf13/viper v1.19.0
github.com/squeeze69/generacodicefiscale v1.0.5
github.com/vadv/gopher-lua-libs v0.5.0
github.com/yuin/gopher-lua v1.1.1
go.mongodb.org/mongo-driver v1.16.0
golang.org/x/text v0.16.0
)
Expand All @@ -40,6 +42,8 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.0.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
github.com/VividCortex/ewma v1.1.1 // indirect
github.com/alessio/shellescape v1.4.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.26.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.27.10 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.10 // indirect
Expand All @@ -55,13 +59,19 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/aws/smithy-go v1.20.2 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/cbroglie/mustache v1.0.1 // indirect
github.com/cenkalti/backoff/v3 v3.0.0 // indirect
github.com/cheggaaa/pb/v3 v3.0.5 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
github.com/go-logr/logr v1.4.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.5.0 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.4 // indirect
Expand All @@ -85,18 +95,28 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lib/pq v1.7.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/mattn/go-sqlite3 v1.14.3 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/prometheus/client_golang v1.17.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/technoweenie/multipartstreamer v1.0.1 // indirect
github.com/tink-crypto/tink-go-gcpkms/v2 v2.1.0 // indirect
github.com/tink-crypto/tink-go-hcvault/v2 v2.1.0 // indirect
github.com/tink-crypto/tink-go/v2 v2.1.0 // indirect
Expand All @@ -105,6 +125,7 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
github.com/yuin/gluamapper v0.0.0-20150323120927-d836955830e7 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
Expand All @@ -125,6 +146,8 @@ require (
google.golang.org/grpc v1.64.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/xmlpath.v2 v2.0.0-20150820204837-860cbeca3ebc // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

require (
Expand Down
127 changes: 127 additions & 0 deletions go.sum

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions pkg/cmd/producerList.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ var producerListCmd = &cobra.Command{
fmt.Printf("%sAZBlobStorage%s (--output = azblobstorage)\n", Green, Reset)
fmt.Printf("%sAZCosmosDB%s (--output = azcosmosdb)\n", Green, Reset)
fmt.Printf("%sCassandra%s (--output = cassandra)\n", Green, Reset)
fmt.Printf("%sLUA Script%s (--output = luascript)\n", Green, Reset)
fmt.Println()

},
Expand Down
3 changes: 3 additions & 0 deletions pkg/cmd/templateRun.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,8 @@ jr template run --template "{{name}}"
configuration.GlobalCfg.HTTPConfig, _ = cmd.Flags().GetString(f.Name)
case "cassandraConfig":
configuration.GlobalCfg.CassandraConfig, _ = cmd.Flags().GetString(f.Name)
case "luascriptConfig":
configuration.GlobalCfg.LUAScriptConfig, _ = cmd.Flags().GetString(f.Name)
}
}
})
Expand Down Expand Up @@ -185,5 +187,6 @@ func init() {
templateRunCmd.Flags().String("azBlobStorageConfig", "", "Azure Blob storage configuration")
templateRunCmd.Flags().String("azCosmosDBConfig", "", "Azure CosmosDB configuration")
templateRunCmd.Flags().String("cassandraConfig", "", "Cassandra configuration")
templateRunCmd.Flags().String("luascriptConfig", "", "LUA Script configuration")

}
1 change: 1 addition & 0 deletions pkg/configuration/configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type GlobalConfiguration struct {
GCSConfig string
HTTPConfig string
CassandraConfig string
LUAScriptConfig string
Url string
EmbeddedTemplate bool
FileNameTemplate bool
Expand Down
12 changes: 12 additions & 0 deletions pkg/emitter/emitter.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"github.com/jrnd-io/jr/pkg/producers/gcs"
"github.com/jrnd-io/jr/pkg/producers/http"
"github.com/jrnd-io/jr/pkg/producers/kafka"
"github.com/jrnd-io/jr/pkg/producers/luascript"
"github.com/jrnd-io/jr/pkg/producers/mongoDB"
"github.com/jrnd-io/jr/pkg/producers/redis"
"github.com/jrnd-io/jr/pkg/producers/s3"
Expand Down Expand Up @@ -156,6 +157,10 @@ func (e *Emitter) Initialize(conf configuration.GlobalConfiguration) {
e.Producer = createCassandraProducer(conf.CassandraConfig)
return
}
if e.Output == "luascript" {
e.Producer = createLUAScriptProducer(conf.LUAScriptConfig)
return
}

}

Expand Down Expand Up @@ -243,6 +248,13 @@ func createCassandraProducer(config string) Producer {
return producer
}

func createLUAScriptProducer(config string) Producer {
producer := &luascript.Producer{}
producer.Initialize(config)

return producer
}

func createKafkaProducer(conf configuration.GlobalConfiguration, topic string, templateType string) *kafka.KafkaManager {

kManager := &kafka.KafkaManager{
Expand Down
25 changes: 25 additions & 0 deletions pkg/producers/luascript/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright © 2024 JR team
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package luascript

type Config struct {
ScriptFile string `json:"script_file"`
Script string `json:"script"`
}
104 changes: 104 additions & 0 deletions pkg/producers/luascript/producer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// Copyright © 2024 JR team
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package luascript

import (
"encoding/json"
"os"
"strings"

"github.com/rs/zerolog/log"
libs "github.com/vadv/gopher-lua-libs"
lua "github.com/yuin/gopher-lua"
"github.com/yuin/gopher-lua/parse"
)

const (
ScriptName = "luajr"
)

type Producer struct {
configuration Config
luaProtoFunction *lua.FunctionProto
}

func (p *Producer) Initialize(configFile string) {
cfgBytes, err := os.ReadFile(configFile)
if err != nil {
log.Fatal().Err(err).Msg("Failed to read config file")
}

config := Config{}
if err := json.Unmarshal(cfgBytes, &config); err != nil {
log.Fatal().Err(err).Msg("Failed to unmarshal config")
}

p.InitializeFromConfig(config)
}

func (p *Producer) InitializeFromConfig(config Config) {

var err error
if config.Script == "" && config.ScriptFile == "" {
log.Fatal().Msg("script or script_file is required")
}

var scriptBytes []byte
if config.ScriptFile != "" {
scriptBytes, err = os.ReadFile(config.ScriptFile)
if err != nil {
log.Fatal().Err(err).Str("script_file", config.ScriptFile).Msg("Failed to read script file")
}
} else {
scriptBytes = []byte(config.Script)
}

chunk, err := parse.Parse(strings.NewReader(string(scriptBytes)), ScriptName)
if err != nil {
log.Fatal().Err(err).Msg("Failed to parse script")
}
p.luaProtoFunction, err = lua.Compile(chunk, ScriptName)
if err != nil {
log.Fatal().Err(err).Msg("Failed to compile script")
}

p.configuration = config

}

func (p *Producer) Produce(k []byte, v []byte, _ any) {

L := lua.NewState()
libs.Preload(L)

L.SetGlobal("k", lua.LString(k))
L.SetGlobal("v", lua.LString(string(v)))

lf := L.NewFunctionFromProto(p.luaProtoFunction)
L.Push(lf)
err := L.PCall(0, 0, nil)
if err != nil {
log.Fatal().Err(err).Msg("Failed to execute script")
}

}

func (p *Producer) Close() error {
return nil
}
53 changes: 53 additions & 0 deletions pkg/producers/luascript/producer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright © 2024 JR team
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
package luascript_test

import (
"testing"

"github.com/jrnd-io/jr/pkg/producers/luascript"
)

func TestProducer(t *testing.T) {

testCases := []struct {
name string
config luascript.Config
}{
{
name: "testprint",
config: luascript.Config{
Script: ` json = require "json"
print(k)
json.decode(v)`,
},
},
}
for _, tc := range testCases {
tc := tc
someJson := `{"key": "value"}`

t.Run(tc.name, func(t *testing.T) {
p := &luascript.Producer{}
p.InitializeFromConfig(tc.config)
p.Produce([]byte("somekey"), []byte(someJson), nil)
})

}
}

0 comments on commit 367ae4d

Please sign in to comment.