-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat(http-connector): update workspace * feat(http-connector): proof of concept extract tables * feat(http-connector): proof of concept extract relations * feat(http-connector): url param * feat(http-connector): better http * feat(http-connector): add pull * feat(http-connector): add venom test * feat(http-connector): add http pinger * feat(http-connector): fix venom test * feat(http-connector): add schema param * feat(pull): filter body * feat(pull): filter body * feat(http): add content-type * feat(http): wip code review push * feat(http): wip fix close resource and lint * feat(http): fix data push content-type * feat(http): push add mode and disableContraints * feat(http): wip remove debug data * fix(httpconnector): log pull body * fix(httpconnector): wait for request completion * fix(httpconnector): venom test impact * fix(httpconnector): venom test impact * docs(httpconnector): changelog and readme * fix(httpconnector): add passing of configured pks * feat(httpconnector): possibility to select columns * feat(httpconnector): possibility to select columns
- Loading branch information
1 parent
4a30808
commit 61e4af1
Showing
25 changed files
with
851 additions
and
94 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // Copyright (C) 2021 CGI France | ||
| // | ||
| // This file is part of LINO. | ||
| // | ||
| // LINO is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // LINO is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU General Public License | ||
| // along with LINO. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package dataconnector | ||
|
|
||
| import ( | ||
| "context" | ||
| "net/http" | ||
|
|
||
| "github.com/cgi-fr/lino/pkg/dataconnector" | ||
| ) | ||
|
|
||
| type HTTPDataPingerFactory struct{} | ||
|
|
||
| // NewHTTPDataPinger creates a new HTTP pinger. | ||
| func NewHTTPDataPingerFactory() *HTTPDataPingerFactory { | ||
| return &HTTPDataPingerFactory{} | ||
| } | ||
|
|
||
| func (pdpf HTTPDataPingerFactory) New(url string) dataconnector.DataPinger { | ||
| return NewHTTPDataPinger(url) | ||
| } | ||
|
|
||
| func NewHTTPDataPinger(url string) HTTPDataPinger { | ||
| return HTTPDataPinger{url} | ||
| } | ||
|
|
||
| type HTTPDataPinger struct { | ||
| url string | ||
| } | ||
|
|
||
| func (pdp HTTPDataPinger) Ping() *dataconnector.Error { | ||
| req, err := http.NewRequestWithContext(context.Background(), http.MethodHead, pdp.url, nil) | ||
| if err != nil { | ||
| return &dataconnector.Error{Description: err.Error()} | ||
| } | ||
| resp, err := http.DefaultClient.Do(req) | ||
| if err != nil { | ||
| return &dataconnector.Error{Description: err.Error()} | ||
| } | ||
| resp.Body.Close() | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| // Copyright (C) 2021 CGI France | ||
| // | ||
| // This file is part of LINO. | ||
| // | ||
| // LINO is free software: you can redistribute it and/or modify | ||
| // it under the terms of the GNU General Public License as published by | ||
| // the Free Software Foundation, either version 3 of the License, or | ||
| // (at your option) any later version. | ||
| // | ||
| // LINO is distributed in the hope that it will be useful, | ||
| // but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| // GNU General Public License for more details. | ||
| // | ||
| // You should have received a copy of the GNU General Public License | ||
| // along with LINO. If not, see <http://www.gnu.org/licenses/>. | ||
|
|
||
| package pull | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "io" | ||
| "net/http" | ||
| "strings" | ||
|
|
||
| "github.com/cgi-fr/lino/pkg/pull" | ||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| // HTTPDataSourceFactory exposes methods to create new HTTP pullers. | ||
| type HTTPDataSourceFactory struct{} | ||
|
|
||
| // NewHTTPDataSourceFactory creates a new HTTP datasource factory. | ||
| func NewHTTPDataSourceFactory() *HTTPDataSourceFactory { | ||
| return &HTTPDataSourceFactory{} | ||
| } | ||
|
|
||
| // New return a HTTP puller | ||
| func (e *HTTPDataSourceFactory) New(url string, schema string) pull.DataSource { | ||
| return &HTTPDataSource{ | ||
| url: url, | ||
| schema: schema, | ||
| } | ||
| } | ||
|
|
||
| // HTTPDataSource to read in the pull process. | ||
| type HTTPDataSource struct { | ||
| url string | ||
| schema string | ||
| result io.ReadCloser | ||
| } | ||
|
|
||
| // Open a connection to the HTTP DB | ||
| func (ds *HTTPDataSource) Open() *pull.Error { | ||
| return nil | ||
| } | ||
|
|
||
| // RowReader iterate over rows in table with filter | ||
| func (ds *HTTPDataSource) RowReader(source pull.Table, filter pull.Filter) (pull.RowReader, *pull.Error) { | ||
| b, err := json.Marshal(struct { | ||
| Values pull.Row `json:"values"` | ||
| Limit uint `json:"limit"` | ||
| Where string `json:"where"` | ||
| }{ | ||
| Values: filter.Values(), | ||
| Limit: filter.Limit(), | ||
| Where: filter.Where(), | ||
| }) | ||
| if err != nil { | ||
| return nil, &pull.Error{Description: err.Error()} | ||
| } | ||
| reqbody := strings.NewReader(string(b)) | ||
|
|
||
| url := ds.url + "/data/" + source.Name() | ||
| if len(ds.schema) > 0 { | ||
| url = url + "?schema=" + ds.schema | ||
| } | ||
|
|
||
| log.Debug().RawJSON("body", b).Str("url", url).Msg("External connector request") | ||
|
|
||
| req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, reqbody) | ||
| if err != nil { | ||
| return nil, &pull.Error{Description: err.Error()} | ||
| } | ||
| req.Header.Add("Content-Type", "application/json") | ||
|
|
||
| if pcols := source.Columns(); pcols != nil && pcols.Len() > 0 { | ||
| pcolsList := []string{} | ||
| for idx := uint(0); idx < pcols.Len(); idx++ { | ||
| pcolsList = append(pcolsList, pcols.Column(idx).Name()) | ||
| } | ||
| b, err = json.Marshal(pcolsList) | ||
| if err != nil { | ||
| return nil, &pull.Error{Description: err.Error()} | ||
| } | ||
| pcolsJSON := string(b) | ||
| req.Header.Add("Select-Columns", pcolsJSON) | ||
| } | ||
|
|
||
| resp, err := http.DefaultClient.Do(req) | ||
| if err != nil { | ||
| return nil, &pull.Error{Description: err.Error()} | ||
| } | ||
| ds.result = resp.Body | ||
|
|
||
| return NewJSONRowReader(resp.Body), nil | ||
| } | ||
|
|
||
| // Close a connection to the HTTP DB | ||
| func (ds *HTTPDataSource) Close() *pull.Error { | ||
| if ds.result != nil { | ||
| ds.result.Close() | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.