Upgrade Jupyter

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.47.0] - 2021-07-12
+### Fixed
+- Upgrading to newer Jupyter and Sparkmagic (fixes some issues in notebooks)
+- Better error handling
+### Added
+- Environment variable completions
+
 ## [0.46.0] - 2021-07-11
 ### Changed
 - Multiple internal improvements

examples/currency_conversion/rates.ipynb

@@ -0,0 +1,67 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%load_ext kamu"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%import_dataset ca.bankofcanada.exchange-rates.daily --alias rates"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rates.printSchema()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%sql\n",
+    "select * from rates"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "PySpark",
+   "language": "python",
+   "name": "pysparkkernel"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "python",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "pyspark",
+   "pygments_lexer": "python3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

examples/housing_prices/heatmap.ipynb

@@ -146,7 +146,9 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {},
+   "metadata": {
+    "scrolled": false
+   },
    "outputs": [],
    "source": [
     "%%local\n",
@@ -179,17 +181,18 @@
  "metadata": {
   "kernelspec": {
    "display_name": "PySpark",
-   "language": "",
+   "language": "python",
    "name": "pysparkkernel"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "python",
-    "version": 2
+    "version": 3
    },
+   "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "pyspark",
-   "pygments_lexer": "python2"
+   "pygments_lexer": "python3"
   }
  },
  "nbformat": 4,

examples/housing_prices/spatial_join.ipynb

@@ -173,17 +173,18 @@
  "metadata": {
   "kernelspec": {
    "display_name": "PySpark",
-   "language": "",
+   "language": "python",
    "name": "pysparkkernel"
   },
   "language_info": {
    "codemirror_mode": {
     "name": "python",
-    "version": 2
+    "version": 3
    },
+   "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "pyspark",
-   "pygments_lexer": "python2"
+   "pygments_lexer": "python3"
   }
  },
  "nbformat": 4,

kamu-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kamu-cli"
-version = "0.46.0"
+version = "0.47.0"
 description = "Decentralized data management tool"
 authors = ["Sergii Mikhtoniuk <mikhtoniuk@gmail.com>"]
 license = "MPL-2.0"

kamu-cli/src/cli_parser.rs

@@ -272,10 +272,10 @@ pub fn cli(binary_name: &'static str, version: &'static str) -> App<'static, 'st
                 .args(&[
                     Arg::with_name("pull-images")
                         .long("pull-images")
-                        .help("Only pull docker images and exit"),
+                        .help("Only pull container images and exit"),
                     Arg::with_name("pull-test-images")
                         .long("pull-test-images")
-                        .help("Only pull test-related docker images and exit")
+                        .help("Only pull test-related container images and exit")
                         .hidden(true),
                 ])
                 .after_help(indoc::indoc!(

kamu-cli/src/commands/complete_command.rs

@@ -40,6 +40,14 @@ impl CompleteCommand {
         println!("{}", Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true));
     }
 
+    fn complete_env_var(&self, prefix: &str) {
+        for (k, _) in std::env::vars() {
+            if k.starts_with(prefix) {
+                println!("{}", k);
+            }
+        }
+    }
+
     fn complete_dataset(&self, prefix: &str) {
         if let Some(repo) = self.metadata_repo.as_ref() {
             for dataset_id in repo.get_all_datasets() {
@@ -144,6 +152,7 @@ impl Command for CompleteCommand {
                             match *name {
                                 "REMOTE" => self.complete_remote(to_complete),
                                 "TIME" => self.complete_timestamp(),
+                                "VAR" => self.complete_env_var(&to_complete),
                                 _ => (),
                             }
                         }

kamu-cli/src/commands/notebook_command.rs

@@ -119,7 +119,7 @@ impl PullImageListener for PullImageProgress {
     fn begin(&mut self, image: &str) {
         let s = indicatif::ProgressBar::new_spinner();
         s.set_style(indicatif::ProgressStyle::default_spinner().template("{spinner:.cyan} {msg}"));
-        s.set_message(format!("Pulling docker image {}", image));
+        s.set_message(format!("Pulling container image {}", image));
         s.enable_steady_tick(100);
         self.progress_bar = Some(s);
     }

kamu-cli/src/commands/sql_shell_command.rs

@@ -89,7 +89,7 @@ impl PullImageListener for PullImageProgress {
     fn begin(&mut self, image: &str) {
         let s = indicatif::ProgressBar::new_spinner();
         s.set_style(indicatif::ProgressStyle::default_spinner().template("{spinner:.cyan} {msg}"));
-        s.set_message(format!("Pulling docker image {}", image));
+        s.set_message(format!("Pulling contaner image {}", image));
         s.enable_steady_tick(100);
         self.progress_bar = Some(s);
     }

kamu-core-test/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kamu-test"
-version = "0.46.0"
+version = "0.47.0"
 authors = ["Sergii Mikhtoniuk <mikhtoniuk@gmail.com>"]
 edition = "2018"
 

kamu-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kamu"
-version = "0.46.0"
+version = "0.47.0"
 authors = ["Sergii Mikhtoniuk <mikhtoniuk@gmail.com>"]
 edition = "2018"
 

kamu-core/src/infra/explore/notebook_server_impl.rs

@@ -97,6 +97,8 @@ impl NotebookServerImpl {
             image: docker_images::JUPYTER.to_owned(),
             container_name: Some("kamu-jupyter".to_owned()),
             network: Some(network_name.to_owned()),
+            user: Some("root".to_owned()),
+            work_dir: Some(PathBuf::from("/opt/workdir")),
             expose_ports: vec![80],
             volume_map: vec![(cwd.clone(), PathBuf::from("/opt/workdir"))],
             environment_vars: environment_vars,
@@ -143,6 +145,7 @@ impl NotebookServerImpl {
         signal_hook::flag::register(libc::SIGINT, exit.clone())?;
         signal_hook::flag::register(libc::SIGTERM, exit.clone())?;
 
+        // TODO: Detect crashed processes
         // Relying on shell to send signal to child processes
         while !exit.load(Ordering::Relaxed) {
             std::thread::sleep(std::time::Duration::from_millis(100));
@@ -162,6 +165,7 @@ impl NotebookServerImpl {
                         .run_shell_cmd(
                             DockerRunArgs {
                                 image: docker_images::JUPYTER.to_owned(),
+                                user: Some("root".to_owned()),
                                 container_name: Some("kamu-jupyter".to_owned()),
                                 volume_map: vec![(cwd, PathBuf::from("/opt/workdir"))],
                                 ..DockerRunArgs::default()

kamu-core/src/infra/utils/docker_client.rs

@@ -138,7 +138,7 @@ impl DockerClient {
             .stdout(Stdio::null())
             .stderr(Stdio::null())
             .status()
-            .expect("Docker process failed")
+            .expect("Process failed")
             .success()
     }
 
@@ -155,6 +155,8 @@ impl DockerClient {
                 .stdout(Stdio::null())
                 .stderr(Stdio::null())
                 .status()
+                .expect("Failed to start pull process")
+                .exit_ok()
                 .expect("Failed to pull image");
 
             listener.success();

kamu-core/src/infra/utils/docker_images.rs

@@ -1,7 +1,7 @@
 pub const SPARK: &str = "docker.io/kamudata/engine-spark:0.10.3";
 pub const LIVY: &str = SPARK;
 pub const FLINK: &str = "docker.io/kamudata/engine-flink:0.8.0-flink_1.13.1-scala_2.12-java8";
-pub const JUPYTER: &str = "docker.io/kamudata/jupyter-uber:0.0.1";
+pub const JUPYTER: &str = "docker.io/kamudata/jupyter:0.1.0";
 
 // Test Images
 pub const HTTPD: &str = "docker.io/httpd:2.4";

kamu-core/src/lib.rs

@@ -1,6 +1,7 @@
 #![feature(backtrace)]
 #![feature(with_options)]
 #![feature(map_first_last)]
+#![feature(exit_status_error)]
 
 pub mod domain;
 pub mod infra;

opendatafabric/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "opendatafabric"
-version = "0.46.0"
+version = "0.47.0"
 authors = ["Sergii Mikhtoniuk <mikhtoniuk@gmail.com>"]
 edition = "2018"