store curl_get result code in datalink

lacanoid · Jun 3, 2021 · 888dc2c · 888dc2c
1 parent 2cc412c
commit 888dc2c
Show file tree

Hide file tree

Showing 6 changed files with 118 additions and 16 deletions.
diff --git a/bin/pg_datalinker b/bin/pg_datalinker
@@ -13,6 +13,7 @@ BEGIN { $ENV{LC_ALL} = "C"; }
 
 our $opt_v = 0;  # verbosity
 our $opt_D = 0;  # really delete linked files with ON UNLINK DELETE
+our $opt_R = 1;  # revert old files from backups
 our $opt_C = 0;  # only check configuration
 our $opt_n = 10; # number of iterations in fast inner loop
 
@@ -21,14 +22,12 @@ getopt('');
 #---------------------------------------
 
 our %config = (
-    appname => 'SQL/MED like datalinker',
+    appname => 'pg_datalinker',
     dsn => 'dbi:Pg:service=datalinker',
     db_owner => 'postgres.postgres',
     db_mode => '0440',
-    url_archive => "/var/www/datalink_url",
-    revert => 1, # revert files from backups
     prefix_file => "/etc/postgresql-common/pg_datalinker.prefix",
-    prefix => []
+    prefix => ["/var/www/datalink/"]
 );
 
 my $operation=shift @ARGV;
@@ -37,14 +36,26 @@ config_load();
 if($operation eq 'list') { config_list(); exit; }
 if($operation eq 'add') { config_add(@ARGV[0]); exit; }
 if($operation eq 'del') { config_del(@ARGV[0]); exit; }
+
+if($operation eq 'start') { daemon_start(); exit; }
+if($operation eq 'restart') { daemon_start(); exit; }
+if($operation eq 'stop') { daemon_stop(); exit; }
+if($operation eq 'status') { daemon_status(); exit; }
+if($operation eq 'log') { daemon_log(); exit; }
+
 if($operation eq 'run') { main(); exit; }
 
 print STDERR qq{Usage: $0 <command> [arguments...]
 Commands:
     list   - list volumes
     add    - add volume
     del    - delete volume
-    run    - run datalinker
+
+    start  - start datalinker
+    stop   - stop datalinker
+    status - check is datalinker is running
+    run    - run datalinker interactively
+    log    - monitor log
 };
 
 exit 1;
@@ -233,8 +244,11 @@ sub config_observe {
 }
 # create inital system configuration if there is none
 sub config_init {
-    $config{prefix}=['/var/www/datalink/'];
-    config_save();
+    if(-f $config{prefix_file}) {
+        log_err("FILE EXISTS","Config file ".$config{prefix_file}." exists.");
+    } else {
+        config_save();
+    }
 }
 # create inital system configuration if there is none
 sub config_add {
@@ -302,7 +316,7 @@ sub file_link {
     	file_read_access_db($path,$opt);
     }
 
-    if($config{revert} && file_has_backup($path,$opt)) {
+    if($opt_R && file_has_backup($path,$opt)) {
         log_msg('REVERT',"$path#".$opt->{token});
         file_backup_restore($path,$opt);
     } else {
@@ -523,3 +537,46 @@ sub log_fatal {
     }
     die;
 }
+
+########################################
+# daemon stuff
+
+our $NAME;
+our $DAEMON;
+our $DAEMON_LOG;
+our $DAEMON_PID;
+our @DAEMON_ARGS;
+sub daemon_setup {
+    $NAME=$config{appname};
+    $DAEMON = `realpath $0`;
+    chomp($DAEMON);
+    $DAEMON_LOG="/var/log/postgresql/$NAME.log";
+    $DAEMON_PID="/var/run/postgresql/$NAME.pid";
+}
+
+sub daemon_start {
+    daemon_stop();
+    sys("start-stop-daemon --start ".
+        "--pidfile $DAEMON_PID --make-pidfile ".
+        "--background --no-close ".
+        "--exec $DAEMON -- run");
+}
+
+sub daemon_stop {
+    daemon_setup();
+    sys("start-stop-daemon","--stop",
+        "--pidfile",$DAEMON_PID,"--remove-pidfile",
+        "--name",$NAME);
+}
+
+sub daemon_status {
+    daemon_setup();
+    sys("start-stop-daemon","--status",
+        "--pidfile",$DAEMON_PID,
+         "--name",$NAME);
+}
+
+sub daemon_log {
+    daemon_setup();
+    if(-f $DAEMON_LOG) { sys("less","+F",$DAEMON_LOG); }
+}
diff --git a/datalink.sql b/datalink.sql
@@ -403,7 +403,7 @@ begin
  if fstat is null then
       raise exception 'datalink exception - referenced file not valid' 
             using errcode = 'HW007',
-                  detail = file_path;
+                  detail = format('stat failed for %s',file_path);
  end if;
 
  addr := array[fstat->>'dev',fstat->>'inode']::text;
@@ -892,6 +892,7 @@ begin
     end if;
   end if; -- file link control,
 
+  link := jsonb_set(link,array['rc'],to_jsonb(r.response_code));
   link := dlpreviouscopy(link,has_token);
 
   if lco.integrity = 'ALL' and dlurlscheme($1)='file' then
@@ -1247,6 +1248,23 @@ select exists (
 $function$
 ;
 
+---------------------------------------------------
+CREATE OR REPLACE FUNCTION datalink.have_datalinker()
+ RETURNS boolean
+ LANGUAGE sql
+ STABLE
+AS $function$
+select exists (
+ select usename
+   from pg_stat_activity
+  where datname is not null 
+    and application_name='pg_datalinker'
+)
+$function$
+;
+COMMENT ON FUNCTION datalink.have_datalinker()
+     IS 'Is datalinker process currently running?';
+
 ---------------------------------------------------
 -- play tables
 ---------------------------------------------------

diff --git a/docs/primer.md b/docs/primer.md
@@ -20,15 +20,15 @@ URLs are checked for syntax and wrong ones throw errors.
     ERROR:  invalid input syntax for type uri at or near " bar"
     CONTEXT:  PL/pgSQL function dlvalue(text,datalink.dl_linktype,text) line 15 at assignment
 
-URLs are normalized before they are converted to datalinks.
+URLs are normalized before they are converted to datalinks, so things like .. are resolved.
 
     mydb=# select dlvalue('http://www.github.io/a/b/c/d/../../e');
                     dlvalue                
     ---------------------------------------
      {"url": "http://www.github.io/a/b/e"}
     (1 row)
 
-Use `dlurlcompleteonly()` function to convert a datalink back to URL.
+Use `dlurlcompleteonly()` function to convert datalink back to URL.
 
     mydb=# select dlurlcompleteonly(dlvalue('http://www.github.io/a/b/c/d/../../e'));
           dlurlcompleteonly      
@@ -52,3 +52,29 @@ Use `dlurlpathonly()` function to get file path from a datalink.
      /var/www/datalink/index.html
     (1 row)
 
+Referential integrity
+---------------------
+
+One can use datalinks to check whether resources pointed to by URLs exist.
+
+For this, one must first create some table with datalink column with integrity='SELECTIVE'.
+
+    mydb=# create table my_table (link datalink);
+    CREATE TABLE
+    mydb=# update datalink.column_options set integrity='SELECTIVE' where table_name='my_table';
+    UPDATE 1
+    mydb=# select * from datalink.column_options where table_name='my_table';
+     table_name | column_name | link_control | integrity | read_access | write_access | recovery | on_unlink 
+    ------------+-------------+--------------+-----------+-------------+--------------+----------+-----------
+     my_table   | link        | FILE         | SELECTIVE | FS          | FS           | NO       | NONE
+    (1 row)
+
+Please note that currently only the super user can change column options.
+
+
+
+
+
+
+
+
diff --git a/test/expected/init.out b/test/expected/init.out
@@ -37,6 +37,7 @@ select p.pronamespace::regnamespace,p.oid::regprocedure,l.lanname,obj_descriptio
  datalink     | dl_lco(dl_link_control,dl_integrity,dl_read_access,dl_write_access,dl_recovery,dl_on_unlink) | sql     | Calculate dl_lco from individual options
  datalink     | dl_lco(regclass,name)                                                                        | sql     | Find dl_lco for a column
  datalink     | file_stat(file_path)                                                                         | plperlu | Return info record from stat(2)
+ datalink     | have_datalinker()                                                                            | sql     | Is datalinker process currently running?
  datalink     | link_control_options(dl_lco)                                                                 | sql     | Calculate link_control_options from dl_lco
  datalink     | modlco(regclass,name,dl_lco)                                                                 | plpgsql | Modify link control options for datalink column
  datalink     | uri_get(text,text)                                                                           | sql     | Get (extract) parts of URI
@@ -55,5 +56,5 @@ select p.pronamespace::regnamespace,p.oid::regprocedure,l.lanname,obj_descriptio
  datalink     | file_unlink(file_path)                                                                       | plpgsql | 
  datalink     | is_valid_prefix(file_path)                                                                   | sql     | 
  datalink     | uuid_generate_v4()                                                                           | c       | 
-(34 rows)
+(35 rows)
 
diff --git a/test/expected/link.out b/test/expected/link.out
@@ -18,7 +18,7 @@ DETAIL:  unknown file volume (prefix) in /etc/passwd
 HINT:  run "pg_datalinker add" to add volumes
 CONTEXT:  PL/pgSQL function file_link(file_path,dl_token,dl_lco,regclass,name) line 14 at RAISE
 SQL statement "SELECT datalink.file_link(dlurlpathonly(link),(link->>'token')::datalink.dl_token,link_options,regclass,column_name)"
-PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 35 at PERFORM
+PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 36 at PERFORM
 PL/pgSQL function dl_trigger_table() line 49 at assignment
 insert into sample_datalinks4 (link)
 values (dlvalue('/var/www/datalink/test1.txt','FS','Sample file datalink 2'));
@@ -38,7 +38,7 @@ DETAIL:  unknown file volume (prefix) in /etc/hosts
 HINT:  run "pg_datalinker add" to add volumes
 CONTEXT:  PL/pgSQL function file_link(file_path,dl_token,dl_lco,regclass,name) line 14 at RAISE
 SQL statement "SELECT datalink.file_link(dlurlpathonly(link),(link->>'token')::datalink.dl_token,link_options,regclass,column_name)"
-PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 35 at PERFORM
+PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 36 at PERFORM
 PL/pgSQL function dl_trigger_table() line 49 at assignment
 insert into sample_datalinks4 (link)
 values (dlvalue('/var/www/datalink/test2.txt','FS','Sample file datalink 4'));
@@ -82,7 +82,7 @@ DETAIL:  unknown file volume (prefix) in /etc/passwd
 HINT:  run "pg_datalinker add" to add volumes
 CONTEXT:  PL/pgSQL function file_link(file_path,dl_token,dl_lco,regclass,name) line 14 at RAISE
 SQL statement "SELECT datalink.file_link(dlurlpathonly(link),(link->>'token')::datalink.dl_token,link_options,regclass,column_name)"
-PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 35 at PERFORM
+PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 36 at PERFORM
 PL/pgSQL function dl_trigger_table() line 49 at assignment
 insert into sample_datalinks5 (link)
 values (dlvalue('/var/www/datalink/test1.txt','FS','Sample file datalink'));

diff --git a/test/expected/linker.out b/test/expected/linker.out
@@ -24,7 +24,7 @@ ERROR:  datalink exception - external file already linked
 DETAIL:  from sample_datalinks6.link as '/var/www/datalink/CHANGELOG.md'
 CONTEXT:  PL/pgSQL function file_link(file_path,dl_token,dl_lco,regclass,name) line 46 at RAISE
 SQL statement "SELECT datalink.file_link(dlurlpathonly(link),(link->>'token')::datalink.dl_token,link_options,regclass,column_name)"
-PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 35 at PERFORM
+PL/pgSQL function dl_ref(datalink,dl_lco,regclass,name) line 36 at PERFORM
 PL/pgSQL function dl_trigger_table() line 49 at assignment
 update sample_datalinks6 set link = link2;
 NOTICE:  DATALINK UNLINK:/var/www/datalink/CHANGELOG.md