From 66ad4abf8a716a367d7c4ad10dd8fd4edc7cea5d Mon Sep 17 00:00:00 2001 From: Louis-Philippe Gauthier Date: Tue, 30 Jun 2015 00:27:53 -0400 Subject: [PATCH] Update for Erlang 18 + /etest/eunit/ --- .travis.yml | 2 +- Makefile | 12 +- README.md | 101 ++++++++----- doc/README.md | 143 ------------------ doc/edoc-info | 1 - doc/overview.edoc | 101 ------------- doc/swirl.md | 5 +- doc/swirl_code_server.md | 14 -- doc/swirl_config.md | 17 --- doc/swirl_ets_manager.md | 13 +- doc/swirl_flow.md | 26 ---- doc/swirl_mapper.md | 48 ------ doc/swirl_ql.md | 22 --- doc/swirl_ql_lexer.md | 8 - doc/swirl_ql_parser.md | 10 -- doc/swirl_reducer.md | 28 ---- doc/swirl_stream.md | 28 ---- doc/swirl_sup.md | 4 +- doc/swirl_tracker.md | 38 ----- doc/swirl_utils.md | 15 -- rebar.config | 19 +-- src/swirl.app.src | 2 +- src/swirl_ql.erl | 95 ------------ src/swirl_utils.erl | 3 +- ...irl_flow_test.erl => swirl_flow_tests.erl} | 66 ++++---- test/swirl_ql_tests.erl | 93 ++++++++++++ test/test.hrl | 5 + 27 files changed, 213 insertions(+), 706 deletions(-) delete mode 100644 doc/README.md delete mode 100644 doc/overview.edoc rename test/{swirl_flow_test.erl => swirl_flow_tests.erl} (71%) create mode 100644 test/swirl_ql_tests.erl create mode 100644 test/test.hrl diff --git a/.travis.yml b/.travis.yml index 2a3f919..7f4feec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,6 @@ language: erlang otp_release: - 17.1 - 17.3 -script: "make compile build-plt eunit etest dialyze" +script: "make compile build-plt eunit dialyze" notifications: email: false diff --git a/Makefile b/Makefile index a500fe5..ed89d54 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,12 @@ PROJECT=swirl REBAR=./rebar -.PHONY: deps doc - all: deps compile doc build-plt: all @dialyzer --build_plt --output_plt ~/.$(PROJECT).plt \ --apps erts kernel stdlib crypto public_key ssl -check-plt: - @dialyzer --check_plt --plt ~/.$(PROJECT).plt - clean: @$(REBAR) clean @rm -rf deps ebin doc/edoc-info doc/*.md README.md @@ -36,9 +31,6 @@ eunit: @echo "Running EUnit suite..." @$(REBAR) skip_deps=true eunit -etest: - @echo "Running ETest suite..." - @ERL_LIBS=deps erlc -pa ebin -o test test/*.erl - @priv/etest-runner +test: all eunit -test: all eunit etest +.PHONY: deps doc diff --git a/README.md b/README.md index dd00910..cf58803 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ - - -# swirl # +# swirl __Authors:__ Louis-Philippe Gauthier. @@ -8,12 +6,36 @@ Lightweight Distributed Stream Processor [![Build Status](https://travis-ci.org/lpgauth/swirl.svg?branch=master)](https://travis-ci.org/lpgauth/swirl) -#### Requirements: #### -- Erlang 17.0+ - -#### Examples: #### - -##### Starting a flow: ##### +### Requirements + +* Erlang 17.0 + + +### Environment variables + + + + + + + + + + + + + + + + + + + + +
NameTypeDefaultDescription
mappers_maxpos_integer()100maximum number of mappers
reducers_maxpos_integer()100maximum number of reducers
+ +## Examples + +#### Starting a flow ```erlang ok = application:start(swirl), @@ -35,7 +57,7 @@ swirl_stream:emit(StreamName, Event), ok = swirl_flow:stop(Flow) ``` -##### Implementing a flow: ##### +#### Implementing a flow ```erlang -module(swirl_flow_example). @@ -67,16 +89,16 @@ output(_Flow, _Period, Rows, OutputOpts) -> io:format("rows: ~p~n", [Rows]), ``` -#### Stream Filter: #### -##### Examples: ##### +#### Stream Filter ```erlang - exchange_id = 3 AND bidder_id IS NOT NULL flight_id in (10, 12, 23) OR tag_id = 20 buyer_id notnull AND seller_id > 103 ``` -##### Swirl QL: ##### + +#### Swirl QL + variables: ``` @@ -108,36 +130,39 @@ null operators: null | notnull ``` -#### Resource Limitation: #### +## TODO +* node discovery +* boolean expression indexing -configurable via: +## Tests -```erlang -application:set_env(swirl, mappers_max, 140) -application:set_env(swirl, reducers_max, 200) +```makefile +make eunit +make build-plt && make dialyze ``` -#### TODO: #### -- node discovery -- boolean expression indexing +## License +```license +The MIT License (MIT) -## Modules ## +Copyright (c) 2015 Louis-Philippe Gauthier +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - - - - - - - - - - - - - - -
swirl
swirl_code_server
swirl_config
swirl_ets_manager
swirl_flow
swirl_mapper
swirl_ql
swirl_ql_lexer
swirl_ql_parser
swirl_reducer
swirl_stream
swirl_sup
swirl_tracker
swirl_utils
+The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +``` diff --git a/doc/README.md b/doc/README.md deleted file mode 100644 index f1327a8..0000000 --- a/doc/README.md +++ /dev/null @@ -1,143 +0,0 @@ - - -# swirl # - -__Authors:__ Louis-Philippe Gauthier. - -Lightweight Distributed Stream Processor - -[![Build Status](https://travis-ci.org/lpgauth/swirl.svg?branch=master)](https://travis-ci.org/lpgauth/swirl) - -#### Requirements: #### -- Erlang 17.0+ - -#### Examples: #### - -##### Starting a flow: ##### - -```erlang -ok = application:start(swirl), - -FlowMod = swirl_flow_example, -FlowOpts = [ - {stream_names, [delivery]}, - {stream_filter, "exchange_id = 3 AND bidder_id IS NOT NULL"} -], -MapperNodes = [node()], -ReducerNode = node(), -{ok, Flow} = swirl_flow:start(FlowMod, FlowOpts, MapperNodes, ReducerNode), - -StreamName = delivery, -Event = #{exchange_id => 1, bidder_id => 10}, - -swirl_stream:emit(StreamName, Event), - -ok = swirl_flow:stop(Flow) -``` - -##### Implementing a flow: ##### - -```erlang --module(swirl_flow_example). --include_lib("swirl/include/swirl.hrl"). - --behavior(swirl_flow). --export([ - map/3, - reduce/3, - output/4 -]). - -%% swirl_flow callbacks -map(StreamName, Event, _MapperOpts) -> - Type = ?L(type, Event), - ExchangeId = ?L(exchange_id, Event), - BidderId = ?L(bidder_id, Event), - - Key = {Type, StreamName, ExchangeId, BidderId}, - CounterIncrements = {1, 10}, - - {Key, CounterIncrements}. - -reduce(_Flow, Row, _ReducerOpts) -> - Row. - -output(_Flow, _Period, Rows, OutputOpts) -> - %% do something with the output - io:format("rows: ~p~n", [Rows]), -``` - -#### Stream Filter: #### -##### Examples: ##### - -```erlang - -exchange_id = 3 AND bidder_id IS NOT NULL -flight_id in (10, 12, 23) OR tag_id = 20 -buyer_id notnull AND seller_id > 103 -``` -##### Swirl QL: ##### -variables: - -``` -atom() -``` -values: - -``` -integer() | float() | binary() -``` -boolean operators: - -``` -'and' | 'or' -``` -comparison operators: - -``` -'<' | '<=' | '=' | '>=' | '>' | '<>' -``` -inclusion operators: - -``` -in | notin -``` -null operators: - -``` -null | notnull -``` - -#### Resource Limitation: #### - -configurable via: - -```erlang -application:set_env(swirl, mappers_max, 140) -application:set_env(swirl, reducers_max, 200) -``` - -#### TODO: #### -- node discovery -- boolean expression indexing - - -## Modules ## - - - - - - - - - - - - - - - - -
swirl
swirl_code_server
swirl_config
swirl_ets_manager
swirl_flow
swirl_mapper
swirl_ql
swirl_ql_lexer
swirl_ql_parser
swirl_reducer
swirl_stream
swirl_sup
swirl_tracker
swirl_utils
- diff --git a/doc/edoc-info b/doc/edoc-info index 51008b9..4e0a270 100644 --- a/doc/edoc-info +++ b/doc/edoc-info @@ -1,6 +1,5 @@ %% encoding: UTF-8 {application,swirl}. -{packages,[]}. {modules,[swirl,swirl_code_server,swirl_config,swirl_ets_manager,swirl_flow, swirl_mapper,swirl_ql,swirl_ql_lexer,swirl_ql_parser,swirl_reducer, swirl_stream,swirl_sup,swirl_tracker,swirl_utils]}. diff --git a/doc/overview.edoc b/doc/overview.edoc deleted file mode 100644 index 48039a4..0000000 --- a/doc/overview.edoc +++ /dev/null @@ -1,101 +0,0 @@ -@author Louis-Philippe Gauthier -@doc Lightweight Distributed Stream Processor - -[![Build Status](https://travis-ci.org/lpgauth/swirl.svg?branch=master)](https://travis-ci.org/lpgauth/swirl) - -#### Requirements: #### -- Erlang 17.0+ - -#### Examples: #### - -##### Starting a flow: ##### - -
ok = application:start(swirl),
-
-FlowMod = swirl_flow_example,
-FlowOpts = [
-    {stream_names, [delivery]},
-    {stream_filter, "exchange_id = 3 AND bidder_id IS NOT NULL"}
-],
-MapperNodes = [node()],
-ReducerNode = node(),
-{ok, Flow} = swirl_flow:start(FlowMod, FlowOpts, MapperNodes, ReducerNode),
-
-StreamName = delivery,
-Event = #{exchange_id => 1, bidder_id => 10},
-
-swirl_stream:emit(StreamName, Event),
-
-ok = swirl_flow:stop(Flow)
- -##### Implementing a flow: ##### - -
-module(swirl_flow_example).
--include_lib("swirl/include/swirl.hrl").
-
--behavior(swirl_flow).
--export([
-    map/3,
-    reduce/3,
-    output/4
-]).
-
-%% swirl_flow callbacks
-map(StreamName, Event, _MapperOpts) ->
-    Type = ?L(type, Event),
-    ExchangeId = ?L(exchange_id, Event),
-    BidderId = ?L(bidder_id, Event),
-
-    Key = {Type, StreamName, ExchangeId, BidderId},
-    CounterIncrements = {1, 10},
-
-    {Key, CounterIncrements}.
-
-reduce(_Flow, Row, _ReducerOpts) ->
-    Row.
-
-output(_Flow, _Period, Rows, OutputOpts) ->
-    %% do something with the output
-    io:format("rows: ~p~n", [Rows]),
-
- -#### Stream Filter: #### - -##### Examples: ##### -
-exchange_id = 3 AND bidder_id IS NOT NULL
-flight_id in (10, 12, 23) OR tag_id = 20
-buyer_id notnull AND seller_id > 103
- -##### Swirl QL: ##### -variables: -
atom()
- -values: -
integer() | float() | binary()
- -boolean operators: -
'and' | 'or'
- -comparison operators: -
=' | '>' | '<>']]>
- -inclusion operators: -
in | notin
- -null operators: -
null | notnull
- -#### Resource Limitation: #### - -configurable via: - -
application:set_env(swirl, mappers_max, 140)
-application:set_env(swirl, reducers_max, 200)
- -#### TODO: #### - -- node discovery -- boolean expression indexing - -@end \ No newline at end of file diff --git a/doc/swirl.md b/doc/swirl.md index 18bad36..ccda25c 100644 --- a/doc/swirl.md +++ b/doc/swirl.md @@ -5,6 +5,7 @@ * [Function Details](#functions) __Behaviours:__ [`application`](application.md). + ## Function Index ## @@ -21,24 +22,20 @@ __Behaviours:__ [`application`](application.md). ### start/0 ### - 

 start() -> ok

- ### start/2 ### `start(StartType, StartArgs) -> any()` - ### stop/1 ### `stop(State) -> any()` - diff --git a/doc/swirl_code_server.md b/doc/swirl_code_server.md index f8bc41f..8a6893c 100644 --- a/doc/swirl_code_server.md +++ b/doc/swirl_code_server.md @@ -17,12 +17,10 @@ __Behaviours:__ [`gen_server`](gen_server.md). ### module_vsn() ### - 

 module_vsn() = pos_integer()
- ## Function Index ## @@ -41,72 +39,60 @@ module_vsn() = pos_integer() `code_change(OldVsn, State, Extra) -> any()` - ### get_module/3 ### - 

 get_module(Node::node(), Module::module(), ModuleVsn::module_vsn()) -> ok

- ### handle_call/3 ### `handle_call(Request, From, State) -> any()` - ### handle_cast/2 ### `handle_cast(Msg, State) -> any()` - ### handle_info/2 ### `handle_info(Info, State) -> any()` - ### init/1 ### `init(X1) -> any()` - ### start_link/0 ### - 

 start_link() -> {ok, pid()}

- ### terminate/2 ### `terminate(Reason, State) -> any()` - ### version/1 ### - 

 version(Module::module()) -> {ok, module_vsn()} | {error, term()}

- diff --git a/doc/swirl_config.md b/doc/swirl_config.md index 5ef2e21..5d77174 100644 --- a/doc/swirl_config.md +++ b/doc/swirl_config.md @@ -4,7 +4,6 @@ * [Function Index](#index) * [Function Details](#functions) - ## Function Index ## @@ -21,87 +20,71 @@ ### flows/0 ### - 

 flows() -> [tuple()]

- ### flows_count/0 ### - 

 flows_count() -> non_neg_integer()

- ### mappers/0 ### - 

 mappers() -> [tuple()]

- ### mappers_count/0 ### - 

 mappers_count() -> non_neg_integer()

- ### mappers_max/0 ### - 

 mappers_max() -> non_neg_integer()

- ### reducers/0 ### - 

 reducers() -> [tuple()]

- ### reducers_count/0 ### - 

 reducers_count() -> non_neg_integer()

- ### reducers_max/0 ### - 

 reducers_max() -> non_neg_integer()

- diff --git a/doc/swirl_ets_manager.md b/doc/swirl_ets_manager.md index d8e8e40..75840ad 100644 --- a/doc/swirl_ets_manager.md +++ b/doc/swirl_ets_manager.md @@ -5,6 +5,7 @@ * [Function Details](#functions) __Behaviours:__ [`gen_server`](gen_server.md). + ## Function Index ## @@ -23,72 +24,60 @@ __Behaviours:__ [`gen_server`](gen_server.md). `code_change(OldVsn, State, Extra) -> any()` - ### handle_call/3 ### `handle_call(Request, From, State) -> any()` - ### handle_cast/2 ### `handle_cast(Msg, State) -> any()` - ### handle_info/2 ### `handle_info(Msg, State) -> any()` - ### init/1 ### `init(X1) -> any()` - ### new_table/3 ### - 

 new_table(Name::atom(), Options::[atom() | tuple()], Server::atom() | pid()) -> ok

- ### start_link/0 ### - 

 start_link() -> {ok, pid()}

- ### table/3 ### - 

 table(Name::atom(), Options::[atom() | tuple()], Server::atom() | pid()) -> ok

- ### terminate/2 ### `terminate(Reason, State) -> any()` - diff --git a/doc/swirl_flow.md b/doc/swirl_flow.md index 647cdd5..bc30e09 100644 --- a/doc/swirl_flow.md +++ b/doc/swirl_flow.md @@ -17,7 +17,6 @@ __This module defines the `swirl_flow` behaviour.__
Required callback func ### flow() ### - 

 flow() = #flow{id = undefined | binary(), module = undefined | module(), module_vsn = undefined | module_vsn(), stream_filter = undefined | string(), stream_names = undefined | stream_names(), mapper_window = undefined | pos_integer(), mapper_nodes = undefined | [node()], mapper_opts = undefined | mapper_opts(), reducer_window = undefined | pos_integer(), reducer_node = undefined | node(), reducer_opts = undefined | reducer_opts(), reducer_skip = undefined | boolean(), output_opts = undefined | output_opts(), heartbeat = undefined | pos_integer(), window_sync = undefined | boolean(), started_at = undefined | erlang:timestamp(), start_node = undefined | node()}
@@ -25,11 +24,9 @@ flow() = #flow{id = undefined | binary(), module = undefined | module(), module_ - ### flow_opts() ### - 

 flow_opts() = {heartbeat, pos_integer()} | {mapper_opts, mapper_opts()} | {mapper_window, pos_integer()} | {output_opts, output_opts()} | {reducer_opts, reducer_opts()} | {reducer_skip, boolean()} | {reducer_window, pos_integer()} | {stream_filter, string()} | {stream_names, stream_names()} | {window_sync, boolean()}
@@ -37,11 +34,9 @@ flow_opts() = {heartbeat, pos_integer()} | {mapper_opts, mapper_opts() ### - 

 mapper_opts() = term()
@@ -49,11 +44,9 @@ mapper_opts() = term() - ### module_vsn() ### - 

 module_vsn() = pos_integer()
@@ -61,11 +54,9 @@ module_vsn() = pos_integer() - ### output_opts() ### - 

 output_opts() = term()
@@ -73,11 +64,9 @@ output_opts() = term() - ### reducer_opts() ### - 

 reducer_opts() = term()
@@ -85,11 +74,9 @@ reducer_opts() = term() - ### stream_name() ### - 

 stream_name() = atom()
@@ -97,16 +84,13 @@ stream_name() = atom() - ### stream_names() ### - 

 stream_names() = [stream_name()]
- ## Function Index ## @@ -123,54 +107,44 @@ stream_names() = [stream_name()] ### lookup/1 ### - 

 lookup(FlowId::binary() | flow()) -> undefined | flow()

- ### register/1 ### - 

 register(Flow::flow()) -> true

- ### start/4 ### - 

 start(FlowMod::atom(), FlowOpts::[flow_opts()], MapperNodes::[node()], ReducerNode::node()) -> {ok, flow()} | {error, flow_mod_undef | {bad_flow_opts, list()}}

- ### stop/1 ### - 

 stop(Flow::flow()) -> ok

- ### unregister/1 ### - 

 unregister(Flow::flow()) -> true

- diff --git a/doc/swirl_mapper.md b/doc/swirl_mapper.md index a638787..a804e3f 100644 --- a/doc/swirl_mapper.md +++ b/doc/swirl_mapper.md @@ -17,7 +17,6 @@ __Behaviours:__ [`gen_server`](gen_server.md). ### boolean_op() ### - 

 boolean_op() = 'and' | 'or'
@@ -25,11 +24,9 @@ boolean_op() = 'and' | 'or' - ### comparison_op() ### - 

 comparison_op() = '<' | '<=' | '=' | '>=' | '>' | '<>'
@@ -37,11 +34,9 @@ comparison_op() = '<' | '<=' | '=' | '>=' | '>' | '<>' - ### event() ### - 

 event() = [{atom(), value()}]
@@ -49,11 +44,9 @@ event() = [{atom(), value()}] - ### exp_tree() ### - 

 exp_tree() = {boolean_op(), exp_tree(), exp_tree()} | {comparison_op(), variable(), value()} | {inclusion_op(), variable(), [value(), ...]} | {null_op(), variable()}
@@ -61,11 +54,9 @@ exp_tree() = {boolean_op(), flow() ### - 

 flow() = #flow{id = undefined | binary(), module = undefined | module(), module_vsn = undefined | module_vsn(), stream_filter = undefined | string(), stream_names = undefined | stream_names(), mapper_window = undefined | pos_integer(), mapper_nodes = undefined | [node()], mapper_opts = undefined | mapper_opts(), reducer_window = undefined | pos_integer(), reducer_node = undefined | node(), reducer_opts = undefined | reducer_opts(), reducer_skip = undefined | boolean(), output_opts = undefined | output_opts(), heartbeat = undefined | pos_integer(), window_sync = undefined | boolean(), started_at = undefined | erlang:timestamp(), start_node = undefined | node()}
@@ -73,11 +64,9 @@ flow() = #flow{id = undefined | binary(), module = undefined | module(), module_ - ### inclusion_op() ### - 

 inclusion_op() = in | notin
@@ -85,11 +74,9 @@ inclusion_op() = in | notin - ### mapper_opts() ### - 

 mapper_opts() = term()
@@ -97,11 +84,9 @@ mapper_opts() = term() - ### module_vsn() ### - 

 module_vsn() = pos_integer()
@@ -109,11 +94,9 @@ module_vsn() = pos_integer() - ### null_op() ### - 

 null_op() = null | notnull
@@ -121,11 +104,9 @@ null_op() = null | notnull - ### output_opts() ### - 

 output_opts() = term()
@@ -133,11 +114,9 @@ output_opts() = term() - ### reducer_opts() ### - 

 reducer_opts() = term()
@@ -145,11 +124,9 @@ reducer_opts() = term() - ### stream() ### - 

 stream() = #stream{flow_id = undefined | binary(), flow_mod = undefined | module(), flow_mod_vsn = undefined | module_vsn(), start_node = undefined | node(), exp_tree = undefined | exp_tree(), mapper_opts = undefined | mapper_opts(), table_id = undefined | ets:tab()}
@@ -157,11 +134,9 @@ stream() = #stream{flow_id = undefined | binary(), flow_mod = undefined | module - ### stream_name() ### - 

 stream_name() = atom()
@@ -169,11 +144,9 @@ stream_name() = atom() - ### stream_names() ### - 

 stream_names() = [stream_name()]
@@ -181,11 +154,9 @@ stream_names() = [stream_name()] - ### value() ### - 

 value() = integer() | float() | binary()
@@ -193,16 +164,13 @@ value() = integer() | float() | binary() - ### variable() ### - 

 variable() = atom()
- ## Function Index ## @@ -221,94 +189,78 @@ variable() = atom() `code_change(OldVsn, State, Extra) -> any()` - ### handle_call/3 ### `handle_call(Request, From, State) -> any()` - ### handle_cast/2 ### `handle_cast(Msg, State) -> any()` - ### handle_info/2 ### `handle_info(Msg, State) -> any()` - ### init/1 ### `init(Flow) -> any()` - ### lookup/1 ### - 

 lookup(FlowId::binary() | flow()) -> undefined | pid

- ### map/3 ### - 

 map(StreamName::atom(), Event::event(), Stream::stream()) -> ok

- ### register/1 ### - 

 register(Flow::flow()) -> true

- ### start/1 ### - 

 start(Flow::flow()) -> {ok, pid()} | {error, mappers_max}

- ### terminate/2 ### `terminate(Reason, State) -> any()` - ### unregister/1 ### - 

 unregister(Flow::flow()) -> true

- diff --git a/doc/swirl_ql.md b/doc/swirl_ql.md index b277c96..962f4ba 100644 --- a/doc/swirl_ql.md +++ b/doc/swirl_ql.md @@ -5,8 +5,6 @@ * [Function Index](#index) * [Function Details](#functions) - - ## Data Types ## @@ -17,7 +15,6 @@ ### boolean_op() ### - 

 boolean_op() = 'and' | 'or'
@@ -25,11 +22,9 @@ boolean_op() = 'and' | 'or' - ### comparison_op() ### - 

 comparison_op() = '<' | '<=' | '=' | '>=' | '>' | '<>'
@@ -37,11 +32,9 @@ comparison_op() = '<' | '<=' | '=' | '>=' | '>' | '<>' - ### event() ### - 

 event() = [{atom(), value()}]
@@ -49,11 +42,9 @@ event() = [{atom(), value()}] - ### exp_tree() ### - 

 exp_tree() = {boolean_op(), exp_tree(), exp_tree()} | {comparison_op(), variable(), value()} | {inclusion_op(), variable(), [value(), ...]} | {null_op(), variable()}
@@ -61,11 +52,9 @@ exp_tree() = {boolean_op(), inclusion_op() ### - 

 inclusion_op() = in | notin
@@ -73,11 +62,9 @@ inclusion_op() = in | notin - ### null_op() ### - 

 null_op() = null | notnull
@@ -85,11 +72,9 @@ null_op() = null | notnull - ### value() ### - 

 value() = integer() | float() | binary()
@@ -97,16 +82,13 @@ value() = integer() | float() | binary() - ### variable() ### - 

 variable() = atom()
- ## Function Index ## @@ -123,21 +105,17 @@ variable() = atom() ### evaluate/2 ### - 

 evaluate(X1::exp_tree(), Vars::event()) -> boolean()

- ### parse/1 ### - 

 parse(String::string() | binary()) -> {ok, exp_tree()} | {error, term()}

- diff --git a/doc/swirl_ql_lexer.md b/doc/swirl_ql_lexer.md index f2558da..8ef6719 100644 --- a/doc/swirl_ql_lexer.md +++ b/doc/swirl_ql_lexer.md @@ -4,7 +4,6 @@ * [Function Index](#index) * [Function Details](#functions) - ## Function Index ## @@ -23,46 +22,39 @@ `format_error(X1) -> any()` - ### string/1 ### `string(String) -> any()` - ### string/2 ### `string(String, Line) -> any()` - ### token/2 ### `token(Cont, Chars) -> any()` - ### token/3 ### `token(X1, Chars, Line) -> any()` - ### tokens/2 ### `tokens(Cont, Chars) -> any()` - ### tokens/3 ### `tokens(X1, Chars, Line) -> any()` - diff --git a/doc/swirl_ql_parser.md b/doc/swirl_ql_parser.md index e5049e3..7ce0fa2 100644 --- a/doc/swirl_ql_parser.md +++ b/doc/swirl_ql_parser.md @@ -5,8 +5,6 @@ * [Function Index](#index) * [Function Details](#functions) - - ## Data Types ## @@ -17,12 +15,10 @@ ### yecc_ret() ### - 

 yecc_ret() = {error, term()} | {ok, term()}
- ## Function Index ## @@ -39,32 +35,26 @@ yecc_ret() = {error, term()} | {ok, term()} ### format_error/1 ### - 

 format_error(Message::any()) -> [char() | list()]

- ### parse/1 ### - 

 parse(Tokens::list()) -> yecc_ret()

- ### parse_and_scan/1 ### - 

 parse_and_scan(X1::{function() | {atom(), atom()}, [term()]} | {atom(), atom(), [term()]}) -> yecc_ret()

- diff --git a/doc/swirl_reducer.md b/doc/swirl_reducer.md index c901642..e96ce71 100644 --- a/doc/swirl_reducer.md +++ b/doc/swirl_reducer.md @@ -17,7 +17,6 @@ __Behaviours:__ [`gen_server`](gen_server.md). ### flow() ### - 

 flow() = #flow{id = undefined | binary(), module = undefined | module(), module_vsn = undefined | module_vsn(), stream_filter = undefined | string(), stream_names = undefined | stream_names(), mapper_window = undefined | pos_integer(), mapper_nodes = undefined | [node()], mapper_opts = undefined | mapper_opts(), reducer_window = undefined | pos_integer(), reducer_node = undefined | node(), reducer_opts = undefined | reducer_opts(), reducer_skip = undefined | boolean(), output_opts = undefined | output_opts(), heartbeat = undefined | pos_integer(), window_sync = undefined | boolean(), started_at = undefined | erlang:timestamp(), start_node = undefined | node()}
@@ -25,11 +24,9 @@ flow() = #flow{id = undefined | binary(), module = undefined | module(), module_ - ### mapper_opts() ### - 

 mapper_opts() = term()
@@ -37,11 +34,9 @@ mapper_opts() = term() - ### module_vsn() ### - 

 module_vsn() = pos_integer()
@@ -49,11 +44,9 @@ module_vsn() = pos_integer() - ### output_opts() ### - 

 output_opts() = term()
@@ -61,11 +54,9 @@ output_opts() = term() - ### reducer_opts() ### - 

 reducer_opts() = term()
@@ -73,11 +64,9 @@ reducer_opts() = term() - ### stream_name() ### - 

 stream_name() = atom()
@@ -85,16 +74,13 @@ stream_name() = atom() - ### stream_names() ### - 

 stream_names() = [stream_name()]
- ## Function Index ## @@ -113,83 +99,69 @@ stream_names() = [stream_name()] `code_change(OldVsn, State, Extra) -> any()` - ### handle_call/3 ### `handle_call(Request, From, State) -> any()` - ### handle_cast/2 ### `handle_cast(Msg, State) -> any()` - ### handle_info/2 ### `handle_info(Msg, State) -> any()` - ### init/1 ### `init(Flow) -> any()` - ### lookup/1 ### - 

 lookup(FlowId::binary() | flow()) -> undefined | pid()

- ### register/1 ### - 

 register(Flow::flow()) -> true

- ### start/1 ### - 

 start(Flow::flow()) -> {ok, pid()} | {error, reducers_max}

- ### terminate/2 ### `terminate(Reason, State) -> any()` - ### unregister/1 ### - 

 unregister(Flow::flow()) -> true

- diff --git a/doc/swirl_stream.md b/doc/swirl_stream.md index 0f2b3f9..df50061 100644 --- a/doc/swirl_stream.md +++ b/doc/swirl_stream.md @@ -5,8 +5,6 @@ * [Function Index](#index) * [Function Details](#functions) - - ## Data Types ## @@ -17,7 +15,6 @@ ### event() ### - 

 event() = [{atom(), value()}]
@@ -25,11 +22,9 @@ event() = [{atom(), value()}] - ### flow() ### - 

 flow() = #flow{id = undefined | binary(), module = undefined | module(), module_vsn = undefined | module_vsn(), stream_filter = undefined | string(), stream_names = undefined | stream_names(), mapper_window = undefined | pos_integer(), mapper_nodes = undefined | [node()], mapper_opts = undefined | mapper_opts(), reducer_window = undefined | pos_integer(), reducer_node = undefined | node(), reducer_opts = undefined | reducer_opts(), reducer_skip = undefined | boolean(), output_opts = undefined | output_opts(), heartbeat = undefined | pos_integer(), window_sync = undefined | boolean(), started_at = undefined | erlang:timestamp(), start_node = undefined | node()}
@@ -37,11 +32,9 @@ flow() = #flow{id = undefined | binary(), module = undefined | module(), module_ - ### mapper_opts() ### - 

 mapper_opts() = term()
@@ -49,11 +42,9 @@ mapper_opts() = term() - ### module_vsn() ### - 

 module_vsn() = pos_integer()
@@ -61,11 +52,9 @@ module_vsn() = pos_integer() - ### output_opts() ### - 

 output_opts() = term()
@@ -73,11 +62,9 @@ output_opts() = term() - ### reducer_opts() ### - 

 reducer_opts() = term()
@@ -85,11 +72,9 @@ reducer_opts() = term() - ### stream_name() ### - 

 stream_name() = atom()
@@ -97,11 +82,9 @@ stream_name() = atom() - ### stream_names() ### - 

 stream_names() = [stream_name()]
@@ -109,16 +92,13 @@ stream_names() = [stream_name()] - ### value() ### - 

 value() = integer() | float() | binary()
- ## Function Index ## @@ -135,43 +115,35 @@ value() = integer() | float() | binary() ### emit/2 ### - 

 emit(StreamName::stream_name(), Event::event()) -> ok

- ### lookup/1 ### - 

 lookup(StreamName::stream_name()) -> [tuple()]

- ### register/2 ### - 

 register(Flow::flow(), TableId::ets:tab()) -> true

- ### unregister/1 ### - 

 unregister(Flow::flow()) -> true

- diff --git a/doc/swirl_sup.md b/doc/swirl_sup.md index eb46b06..b412d0a 100644 --- a/doc/swirl_sup.md +++ b/doc/swirl_sup.md @@ -5,6 +5,7 @@ * [Function Details](#functions) __Behaviours:__ [`supervisor`](supervisor.md). + ## Function Index ## @@ -23,15 +24,12 @@ __Behaviours:__ [`supervisor`](supervisor.md). `init(X1) -> any()` - ### start_link/0 ### - 

 start_link() -> {ok, pid()}

- diff --git a/doc/swirl_tracker.md b/doc/swirl_tracker.md index 0b94734..fa21233 100644 --- a/doc/swirl_tracker.md +++ b/doc/swirl_tracker.md @@ -17,7 +17,6 @@ __Behaviours:__ [`gen_server`](gen_server.md). ### flow() ### - 

 flow() = #flow{id = undefined | binary(), module = undefined | module(), module_vsn = undefined | module_vsn(), stream_filter = undefined | string(), stream_names = undefined | stream_names(), mapper_window = undefined | pos_integer(), mapper_nodes = undefined | [node()], mapper_opts = undefined | mapper_opts(), reducer_window = undefined | pos_integer(), reducer_node = undefined | node(), reducer_opts = undefined | reducer_opts(), reducer_skip = undefined | boolean(), output_opts = undefined | output_opts(), heartbeat = undefined | pos_integer(), window_sync = undefined | boolean(), started_at = undefined | erlang:timestamp(), start_node = undefined | node()}
@@ -25,11 +24,9 @@ flow() = #flow{id = undefined | binary(), module = undefined | module(), module_ - ### mapper_opts() ### - 

 mapper_opts() = term()
@@ -37,11 +34,9 @@ mapper_opts() = term() - ### module_vsn() ### - 

 module_vsn() = pos_integer()
@@ -49,11 +44,9 @@ module_vsn() = pos_integer() - ### output_opts() ### - 

 output_opts() = term()
@@ -61,11 +54,9 @@ output_opts() = term() - ### reducer_opts() ### - 

 reducer_opts() = term()
@@ -73,11 +64,9 @@ reducer_opts() = term() - ### stream_name() ### - 

 stream_name() = atom()
@@ -85,16 +74,13 @@ stream_name() = atom() - ### stream_names() ### - 

 stream_names() = [stream_name()]
- ## Function Index ## @@ -113,138 +99,114 @@ stream_names() = [stream_name()] `code_change(OldVsn, State, Extra) -> any()` - ### handle_call/3 ### `handle_call(Request, From, State) -> any()` - ### handle_cast/2 ### `handle_cast(Msg, State) -> any()` - ### handle_info/2 ### `handle_info(Info, State) -> any()` - ### init/1 ### `init(X1) -> any()` - ### lookup/2 ### - 

 lookup(TableId::ets:tab(), Key::term()) -> term()

- ### message/3 ### - 

 message(Node::node(), FlowId::binary(), Msg::term()) -> ok

- ### register/3 ### - 

 register(TableId::ets:tab(), Key::term(), Value::term()) -> true

- ### start_link/0 ### - 

 start_link() -> {ok, pid()}

- ### start_mappers/1 ### - 

 start_mappers(Flow::flow()) -> ok

- ### start_reducer/1 ### - 

 start_reducer(Flow::flow()) -> ok

- ### stop_mappers/1 ### - 

 stop_mappers(Flow::flow()) -> ok

- ### stop_reducer/1 ### - 

 stop_reducer(Flow::flow()) -> ok

- ### terminate/2 ### `terminate(Reason, State) -> any()` - ### unregister/2 ### - 

 unregister(TableId::ets:tab(), Key::term()) -> true

- diff --git a/doc/swirl_utils.md b/doc/swirl_utils.md index 77c5546..5fd5e9e 100644 --- a/doc/swirl_utils.md +++ b/doc/swirl_utils.md @@ -4,7 +4,6 @@ * [Function Index](#index) * [Function Details](#functions) - ## Function Index ## @@ -23,95 +22,81 @@ `lookup(Key, List) -> any()` - ### lookup/3 ### `lookup(Key, List, Default) -> any()` - ### new_timer/2 ### `new_timer(Time, Msg) -> any()` - ### new_timer/3 ### `new_timer(Time, Msg, X3) -> any()` - ### proplist_to_record/2 ### `proplist_to_record(Proplist, Record) -> any()` - ### record_to_proplist/1 ### `record_to_proplist(Flow) -> any()` - ### safe_dict_fetch/2 ### `safe_dict_fetch(Key, Dict) -> any()` - ### safe_ets_delete/1 ### `safe_ets_delete(TableId) -> any()` - ### safe_ets_increment/3 ### `safe_ets_increment(TableId, Key, Counters) -> any()` - ### safe_ets_lookup_element/2 ### `safe_ets_lookup_element(TableId, Key) -> any()` - ### tab2list/1 ### `tab2list(Tid) -> any()` - ### unix_tstamp_ms/0 ### `unix_tstamp_ms() -> any()` - ### update_op/1 ### `update_op(Counters) -> any()` - ### uuid/0 ### `uuid() -> any()` - diff --git a/rebar.config b/rebar.config index aa7a581..258b5c9 100644 --- a/rebar.config +++ b/rebar.config @@ -1,17 +1,11 @@ -{clean_files, ["test/*.beam"]}. +{cover_enabled, true}. +{cover_print_enabled, true}. {deps, [ - %% dependencies - {uuid, "1.3.1", - {git, "https://github.com/okeuday/uuid.git", {tag, "v1.3.1"}}}, - - %% tests - {etest, "1.0.0.beta", - {git, "https://github.com/wooga/etest.git", {branch, "master"}}}, - - %% docs {edown, ".*", - {git, "https://github.com/uwiger/edown.git", "HEAD"}} + {git, "https://github.com/uwiger/edown.git", "HEAD"}}, + {uuid, ".*", + {git, "https://github.com/okeuday/uuid.git", {tag, "v1.5.0"}}} ]}. {edoc_opts, [ @@ -21,8 +15,7 @@ {includes, ["include"]}, {preprocess, true}, {stylesheet, ""}, - {title, "swirl"}, - {top_level_readme, {"./README.md", "http://github.com/lpgauth/swirl"}} + {title, "swirl"} ]}. {erl_opts, [ diff --git a/src/swirl.app.src b/src/swirl.app.src index 244c7d1..227461d 100644 --- a/src/swirl.app.src +++ b/src/swirl.app.src @@ -1,6 +1,6 @@ {application, swirl, [ {description, "lightweight distributed stream processor"}, - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {registered, [ swirl_ets_manager, swirl_sup, diff --git a/src/swirl_ql.erl b/src/swirl_ql.erl index b79ced5..8aaf18c 100644 --- a/src/swirl_ql.erl +++ b/src/swirl_ql.erl @@ -66,98 +66,3 @@ compare('<>', A, B) when is_number(A) or is_number(B) or is_binary(B) -> A /= B. - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -%% tests -benchmark_test() -> - {ok, ExpTree} = parse("exchange_id = 1 AND exchange_seller_id = 181 AND bidder_id IN (1, 5) AND buyer_spend > 150"), - - Vars = [ - {exchange_id, 1}, - {exchange_seller_id, 181}, - {bidder_id, 1}, - {buyer_spend, 200} - ], - - FunEvaluate = fun() -> evaluate(ExpTree, Vars) end, - benchmark(evaluate, FunEvaluate, 100000). - -evaluate_test() -> - % comp predictate - assert_eval({comp, '=', bidder_id, 1}, [{bidder_id, 1}]), - assert_not_eval({comp, '=', bidder_id, 1}, [{bidder_id, 2}]), - assert_eval({comp, '<', price, 100}, [{price, 60}]), - assert_not_eval({comp, '<', price, 100}, [{price, 160}]), - assert_eval({comp, '<=', price, 100}, [{price, 100}]), - assert_not_eval({comp, '<=', price, 100}, [{price, 160}]), - assert_eval({comp, '>=', price, 100}, [{price, 100}]), - assert_not_eval({comp, '>=', price, 160}, [{price, 100}]), - assert_eval({comp, '>', price, 100}, [{price, 160}]), - assert_not_eval({comp, '>', price, 100}, [{price, 60}]), - assert_eval({comp, '<>', price, 100}, [{price, 160}]), - assert_not_eval({comp, '<>', price, 100}, [{price, 100}]), - - % in predictate - assert_eval({in, exchange_id, [1 , 2]}, [{exchange_id, 2}]), - assert_not_eval({in, exchange_id, [1 , 2]}, [{exchange_id, 3}]), - assert_eval({notin, exchange_id, [1 , 2]}, [{exchange_id, 3}]), - assert_not_eval({notin, exchange_id, [1 , 2]}, [{exchange_id, 2}]), - assert_eval({in_var, 54, segment_ids}, [{segment_ids, [12, 54]}]), - assert_not_eval({in_var, 54, segment_ids}, [{segment_ids, [12]}]), - assert_eval({notin_var, 54, segment_ids}, [{segment_ids, [12]}]), - assert_not_eval({notin_var, 54, segment_ids}, [{segment_ids, [12, 54]}]), - - % null predictate - assert_eval({null, exchange_id}, []), - assert_not_eval({null, exchange_id}, [{exchange_id, 3}]), - assert_eval({notnull, exchange_id}, [{exchange_id, 3}]), - assert_not_eval({notnull, exchange_id}, [{exchange_id, ?NULL}]), - - % and - assert_eval({'and', {comp, '=', bidder_id, 1}, {comp, '=', bidder_id, 1}}, - [{bidder_id, 1}]), - assert_not_eval({'and', {comp, '=', bidder_id, 1}, {comp, '=', exchange_id, 1}}, - [{bidder_id, 1}, {exchange_id, 2}]), - - % or - assert_eval({'or', {comp, '=', bidder_id, 2}, {comp, '=', bidder_id, 1}}, - [{bidder_id, 1}]), - assert_not_eval({'or', {comp, '=', bidder_id, 2}, {comp, '=', bidder_id, 3}}, - [{bidder_id, 1}]). - -parse_test() -> - assert_parse({comp, '=', bidder_id, 1}, "bidder_id = 1"), - assert_parse({comp, '=', domain, <<"ebay.ca">>}, "domain = 'ebay.ca'"), - assert_parse({comp, '=', domain, <<"ebay.ca">>}, "domain = \"ebay.ca\""), - assert_parse({in, exchange_id, [1, 2, 3]}, "exchange_id IN (1, 2, 3)"), - assert_parse({in_var, 4, segment_ids}, "4 IN segment_ids"), - assert_parse({notin_var, 8, segment_ids}, "8 NOT IN segment_ids"), - assert_parse({'and', {comp, '=', bidder_id, 1}, {'or', {notin, exchange_id, [1 , 2]}, - {comp, '=', domain, <<"ebay.ca">>}}}, "bidder_id = 1 AND (exchange_id NOT IN (1, 2) OR domain = 'ebay.ca')"). - -%% test_utils -assert_eval(ExpTree, Vars) -> - ?assert(evaluate(ExpTree, Vars)). - -assert_not_eval(ExpTree, Vars) -> - ?assertNot(evaluate(ExpTree, Vars)). - -assert_parse(Expected, Expression) -> - {ok, ExpTree} = parse(Expression), - ?assertEqual(Expected, ExpTree). - -benchmark(Name, Fun, N) -> - Timestamp = os:timestamp(), - ok = loop(Fun, N), - Time = timer:now_diff(os:timestamp(), Timestamp) / N, - ?debugFmt("~p: ~p microseconds", [Name, Time]). - -loop(_, 0) -> - ok; -loop(Fun, N) -> - Fun(), - loop(Fun, N - 1). - --endif. diff --git a/src/swirl_utils.erl b/src/swirl_utils.erl index 821f313..7c13af5 100644 --- a/src/swirl_utils.erl +++ b/src/swirl_utils.erl @@ -101,7 +101,8 @@ update_op(Counters) when is_list(Counters) -> update_op(Counters, 2). uuid() -> - uuid:get_v1(uuid:new(self(), os)). + {Uuid, _UuidState} = uuid:get_v1(uuid:new(self(), os)), + Uuid. %% private match_all('$end_of_table') -> diff --git a/test/swirl_flow_test.erl b/test/swirl_flow_tests.erl similarity index 71% rename from test/swirl_flow_test.erl rename to test/swirl_flow_tests.erl index b51c1d0..6023da1 100644 --- a/test/swirl_flow_test.erl +++ b/test/swirl_flow_tests.erl @@ -1,23 +1,19 @@ --module(swirl_flow_test). --include_lib("etest/include/etest.hrl"). - --export([ - after_suite/0, - before_suite/0, - test_benchmark_emit/0, - test_swirl_flow/0 -]). - --define(N, 10000). - -%% public -after_suite() -> - ok = application:stop(swirl). - -before_suite() -> - random:seed(erlang:now()), - application:ensure_all_started(swirl). - +-module(swirl_flow_tests). +-include("test.hrl"). + +-compile(export_all). + +%% runners +swirl_test_() -> + {setup, + fun () -> setup() end, + fun (_) -> cleanup() end, + {inparallel, [ + ?T(test_benchmark_emit), + ?T(test_swirl_flow) + ]}}. + +%% tests test_benchmark_emit() -> Flows = [new_flow() || _ <- lists:seq(1, 100)], timer:sleep(timer:seconds(1)), @@ -25,7 +21,7 @@ test_benchmark_emit() -> Timestamp = os:timestamp(), emit_loop(?N), Delta = timer:now_diff(os:timestamp(), Timestamp), - io:format("~p microseconds~n", [Delta / ?N]), + ?debugFmt("~p microseconds~n", [Delta / ?N]), [swirl_flow:stop(Flow) || Flow <- Flows]. @@ -52,15 +48,20 @@ test_swirl_flow() -> Rows = receive_loop(), Expected = [ - {{start,requests,3,50},{1,10}}, {{start,delivery,3,1},{1,10}}, - {{start,delivery,3,10},{1,10}} + {{start,delivery,3,10},{1,10}}, + {{start,requests,3,50},{1,10}} ], - ?assert_equal(Expected, Rows), + ?assertEqual(Expected, lists:usort(Rows)), swirl_flow:stop(Flow). -%% private +%% utils +cleanup() -> + error_logger:tty(false), + application:stop(swirl), + error_logger:tty(true). + emit_loop(0) -> ok; emit_loop(N) -> @@ -98,8 +99,15 @@ random_type() -> receive_loop() -> receive - [] -> - receive_loop(); - Aggregates -> - Aggregates + [] -> receive_loop(); + Aggregates -> Aggregates end. + +setup() -> + error_logger:tty(false), + application:stop(swirl), + swirl:start(), + error_logger:tty(true). + +test(Test) -> + {atom_to_list(Test), ?MODULE, Test}. diff --git a/test/swirl_ql_tests.erl b/test/swirl_ql_tests.erl new file mode 100644 index 0000000..3a7529a --- /dev/null +++ b/test/swirl_ql_tests.erl @@ -0,0 +1,93 @@ +-module(swirl_ql_tests). +-include("test.hrl"). + +-compile(export_all). + +benchmark_test() -> + {ok, ExpTree} = swirl_ql:parse("exchange_id = 1 AND exchange_seller_id = 181 AND bidder_id IN (1, 5) AND buyer_spend > 150"), + + Vars = [ + {exchange_id, 1}, + {exchange_seller_id, 181}, + {bidder_id, 1}, + {buyer_spend, 200} + ], + + FunEvaluate = fun() -> swirl_ql:evaluate(ExpTree, Vars) end, + benchmark(evaluate, FunEvaluate, 100000). + +evaluate_test() -> + % comp predictate + assert_eval({comp, '=', bidder_id, 1}, [{bidder_id, 1}]), + assert_not_eval({comp, '=', bidder_id, 1}, [{bidder_id, 2}]), + assert_eval({comp, '<', price, 100}, [{price, 60}]), + assert_not_eval({comp, '<', price, 100}, [{price, 160}]), + assert_eval({comp, '<=', price, 100}, [{price, 100}]), + assert_not_eval({comp, '<=', price, 100}, [{price, 160}]), + assert_eval({comp, '>=', price, 100}, [{price, 100}]), + assert_not_eval({comp, '>=', price, 160}, [{price, 100}]), + assert_eval({comp, '>', price, 100}, [{price, 160}]), + assert_not_eval({comp, '>', price, 100}, [{price, 60}]), + assert_eval({comp, '<>', price, 100}, [{price, 160}]), + assert_not_eval({comp, '<>', price, 100}, [{price, 100}]), + + % in predictate + assert_eval({in, exchange_id, [1 , 2]}, [{exchange_id, 2}]), + assert_not_eval({in, exchange_id, [1 , 2]}, [{exchange_id, 3}]), + assert_eval({notin, exchange_id, [1 , 2]}, [{exchange_id, 3}]), + assert_not_eval({notin, exchange_id, [1 , 2]}, [{exchange_id, 2}]), + assert_eval({in_var, 54, segment_ids}, [{segment_ids, [12, 54]}]), + assert_not_eval({in_var, 54, segment_ids}, [{segment_ids, [12]}]), + assert_eval({notin_var, 54, segment_ids}, [{segment_ids, [12]}]), + assert_not_eval({notin_var, 54, segment_ids}, [{segment_ids, [12, 54]}]), + + % null predictate + assert_eval({null, exchange_id}, []), + assert_not_eval({null, exchange_id}, [{exchange_id, 3}]), + assert_eval({notnull, exchange_id}, [{exchange_id, 3}]), + assert_not_eval({notnull, exchange_id}, [{exchange_id, ?NULL}]), + + % and + assert_eval({'and', {comp, '=', bidder_id, 1}, {comp, '=', bidder_id, 1}}, + [{bidder_id, 1}]), + assert_not_eval({'and', {comp, '=', bidder_id, 1}, {comp, '=', exchange_id, 1}}, + [{bidder_id, 1}, {exchange_id, 2}]), + + % or + assert_eval({'or', {comp, '=', bidder_id, 2}, {comp, '=', bidder_id, 1}}, + [{bidder_id, 1}]), + assert_not_eval({'or', {comp, '=', bidder_id, 2}, {comp, '=', bidder_id, 3}}, + [{bidder_id, 1}]). + +parse_test() -> + assert_parse({comp, '=', bidder_id, 1}, "bidder_id = 1"), + assert_parse({comp, '=', domain, <<"ebay.ca">>}, "domain = 'ebay.ca'"), + assert_parse({comp, '=', domain, <<"ebay.ca">>}, "domain = \"ebay.ca\""), + assert_parse({in, exchange_id, [1, 2, 3]}, "exchange_id IN (1, 2, 3)"), + assert_parse({in_var, 4, segment_ids}, "4 IN segment_ids"), + assert_parse({notin_var, 8, segment_ids}, "8 NOT IN segment_ids"), + assert_parse({'and', {comp, '=', bidder_id, 1}, {'or', {notin, exchange_id, [1 , 2]}, + {comp, '=', domain, <<"ebay.ca">>}}}, "bidder_id = 1 AND (exchange_id NOT IN (1, 2) OR domain = 'ebay.ca')"). + +%% test_utils +assert_eval(ExpTree, Vars) -> + ?assert(swirl_ql:evaluate(ExpTree, Vars)). + +assert_not_eval(ExpTree, Vars) -> + ?assertNot(swirl_ql:evaluate(ExpTree, Vars)). + +assert_parse(Expected, Expression) -> + {ok, ExpTree} = swirl_ql:parse(Expression), + ?assertEqual(Expected, ExpTree). + +benchmark(Name, Fun, N) -> + Timestamp = os:timestamp(), + ok = loop(Fun, N), + Time = timer:now_diff(os:timestamp(), Timestamp) / N, + ?debugFmt("~p: ~p microseconds", [Name, Time]). + +loop(_, 0) -> + ok; +loop(Fun, N) -> + Fun(), + loop(Fun, N - 1). diff --git a/test/test.hrl b/test/test.hrl new file mode 100644 index 0000000..e759a0e --- /dev/null +++ b/test/test.hrl @@ -0,0 +1,5 @@ +-include_lib("eunit/include/eunit.hrl"). +-include_lib("swirl/include/swirl.hrl"). + +-define(N, 1000). +-define(T, fun (Test) -> test(Test) end).