diff --git a/.gitignore b/.gitignore index 047173c..c88d705 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .DS_Store node_modules/ .env +data/index.db diff --git a/db/migration-01.sql b/db/migration-01.sql new file mode 100644 index 0000000..f9d9ab9 --- /dev/null +++ b/db/migration-01.sql @@ -0,0 +1,15 @@ +CREATE TABLE record ( + sensor_id INTEGER, + time_stamp INTEGER, + pm25_alt_a REAL, + pm25_alt_b REAL, + pm25_atm_a REAL, + pm25_atm_b REAL, + pm25_cf_1_a REAL, + pm25_cf_1_b REAL, + temperature REAL, + humidity REAL, + pressure REAL +); + +CREATE INDEX record_idx ON record (sensor_id, time_stamp, pm25_atm_a, pm25_atm_b); diff --git a/package-lock.json b/package-lock.json index 70916e6..bedaf91 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "air-justice-lab", - "version": "0.0.2", + "version": "0.0.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "air-justice-lab", - "version": "0.0.2", + "version": "0.0.3", "license": "MIT", "dependencies": { + "better-sqlite3": "^8.7.0", "csv": "^6.3.3", "dotenv": "^16.3.1", "glob": "^10.3.10" @@ -66,6 +67,53 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/better-sqlite3": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-8.7.0.tgz", + "integrity": "sha512-99jZU4le+f3G6aIl6PmmV0cxUIWqKieHxsiF7G34CVFiE+/UabpYqkU0NJIkY/96mQKikHeBjtR27vFfs5JpEw==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -74,6 +122,34 @@ "balanced-match": "^1.0.0" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -132,6 +208,36 @@ "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.2.tgz", "integrity": "sha512-DXIdnnCUQYjDKTu6TgCSzRDiAuLxDjhl4ErFP9FGMF3wzBGOVMg9bZTLaUcYtuvhXgNbeXPKeaRfpgyqE4xySw==" }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "engines": { + "node": ">=8" + } + }, "node_modules/dotenv": { "version": "16.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", @@ -153,6 +259,27 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, "node_modules/foreground-child": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", @@ -168,6 +295,16 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "node_modules/glob": { "version": "10.3.10", "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", @@ -189,6 +326,35 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, "node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", @@ -227,6 +393,17 @@ "node": "14 || >=16.14" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "9.0.3", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", @@ -241,6 +418,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/minipass": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz", @@ -249,6 +434,35 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, + "node_modules/node-abi": { + "version": "3.47.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.47.0.tgz", + "integrity": "sha512-2s6B2CWZM//kPgwnuI0KrYwNjfdByE25zvAaEpq9IH4zcNsarH8Ihu/UuX6XMPEogDAxkuUFeZn60pXNHAqn3A==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -272,6 +486,111 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/prebuild-install": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.1.tgz", + "integrity": "sha512-jAXscXWMcCK8GgCoHOfIr0ODh5ai8mj63L2nWrjuAgXE6tDyYGnx4/8o/rCgU+B4JSyZBKbeZqzhtwtC3ovxjw==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver/node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -302,11 +621,62 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/stream-transform": { "version": "3.2.8", "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.2.8.tgz", "integrity": "sha512-NUx0mBuI63KbNEEh9Yj0OzKB7iMOSTpkuODM2G7By+TTVihEIJ0cYp5X+pq/TdJRlsznt6CYR8HqxexyC6/bTw==" }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/string-width": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", @@ -395,6 +765,56 @@ "node": ">=8" } }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -492,6 +912,16 @@ "engines": { "node": ">=8" } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" } } } diff --git a/package.json b/package.json index 6105dc8..6f3e8d0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "air-justice-lab", - "version": "0.0.2", + "version": "0.0.3", "description": "Air Justice Lab is a community air monitoring and educational initiative based at NATURE Lab and serving the greater Capital Region.", "main": "scripts/index.js", "type": "module", @@ -16,6 +16,7 @@ "author": "Dan Phiffer ", "license": "MIT", "dependencies": { + "better-sqlite3": "^8.7.0", "csv": "^6.3.3", "dotenv": "^16.3.1", "glob": "^10.3.10" diff --git a/scripts/data.js b/scripts/data.js index 020bb0f..92f77ec 100644 --- a/scripts/data.js +++ b/scripts/data.js @@ -1,20 +1,23 @@ import { getSensorHistory } from "./api.js"; +import Database from 'better-sqlite3'; import { stringify } from 'csv-stringify/sync'; import { parse } from 'csv-parse/sync'; import fs from 'fs'; import { globSync } from 'glob'; +import path from 'path'; export default class Data { constructor(fields) { this.fields = fields; - this.setupIndex(); + this.setupSensors(); this.setupTimestamps(); + this.setupIndex(); } - setupIndex() { + setupSensors() { const csvData = fs.readFileSync(`./data/sensors.csv`, 'utf8'); - this.index = parse(csvData, { + this.sensors = parse(csvData, { columns: true, skip_empty_lines: true }); @@ -23,7 +26,7 @@ export default class Data { setupTimestamps() { this.timestamps = {}; const now = Math.floor(Date.now() / 1000); - for (let sensor of this.index) { + for (let sensor of this.sensors) { const files = globSync(`./data/sensor-${sensor.id}/*.csv`); if (files.length == 0) { this.timestamps[sensor.id] = { @@ -46,17 +49,23 @@ export default class Data { } } - async load() { + setupIndex() { + this.index = new Database('./data/index.db'); + this.index.pragma('journal_mode = WAL'); + this.migrateDatabase(); + } + + async loadRecords() { const now = Math.floor(Date.now() / 1000); - for (let sensor of this.index) { + for (let sensor of this.sensors) { try { console.log(`Loading "${sensor.name}" (${sensor.id})`); const timestamp = this.timestamps[sensor.id]; - this.save(sensor, await this.loadBackward(sensor, timestamp.start)); + this.saveRecords(sensor, await this.loadBackward(sensor, timestamp.start)); await new Promise(resolve => setTimeout(resolve, 1000)); if (now - timestamp.end > 24 * 60 * 60) { // only load forward if at least 24 hours have passed - this.save(sensor, await this.loadForward(sensor, timestamp.end)); + this.saveRecords(sensor, await this.loadForward(sensor, timestamp.end)); await new Promise(resolve => setTimeout(resolve, 1000)); } } catch(error) { @@ -64,7 +73,7 @@ export default class Data { await new Promise(resolve => setTimeout(resolve, 10000)); } } - this.saveIndex(); + this.saveSensors(); } async loadBackward(sensor, startTimestamp) { @@ -85,7 +94,7 @@ export default class Data { return rsp; } - save(sensor, rsp) { + saveRecords(sensor, rsp) { if (!rsp.data) { console.log(` no data found: ${JSON.stringify(rsp)}`); return; @@ -106,7 +115,7 @@ export default class Data { const filename = `${dirname}/sensor-${sensor.id}-${start}-${end}.csv`; fs.writeFileSync(filename, stringify([header, ...rows]), 'utf8'); console.log(` saved ${count} records ${this.timeRange(start, end)}`); - this.updateIndex(sensor, start, end); + this.updateSensor(sensor, start, end); } getRows(rsp, columns) { @@ -139,7 +148,7 @@ export default class Data { return date.toJSON().replace('.000Z', ''); } - updateIndex(sensor, startTimestamp, endTimestamp) { + updateSensor(sensor, startTimestamp, endTimestamp) { const start = this.formatDate(startTimestamp); const end = this.formatDate(endTimestamp); if (!sensor.start || start < sensor.start) { @@ -150,10 +159,180 @@ export default class Data { } } - saveIndex() { - const csvData = stringify(this.index, { + saveSensors() { + const csvData = stringify(this.sensors, { header: true }); fs.writeFileSync('./data/sensors.csv', csvData, 'utf8'); } + + indexData() { + const statement = this.index.prepare(` + INSERT INTO record + ( + sensor_id, + time_stamp, + pm25_alt_a, + pm25_alt_b, + pm25_atm_a, + pm25_atm_b, + pm25_cf_1_a, + pm25_cf_1_b, + temperature, + humidity, + pressure + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `); + const insertRecords = this.index.transaction(records => { + records.shift(); // header + records.map(record => statement.run(record)); + }); + for (let sensor of this.sensors) { + let recordsFiles = globSync(path.join(`./data/sensor-${sensor.id}`, '*.csv')); + recordsFiles.sort(); + for (let file of recordsFiles) { + const match = path.basename(file).match(/^sensor-(\d+)-(\d+)-(\d+).csv$/); + if (!match) { + continue; + } + const sensorId = parseInt(match[1]); + const startTimestamp = parseInt(match[2]); + const endTimestamp = parseInt(match[3]); + if (this.alreadyIndexed(sensorId, startTimestamp, endTimestamp)) { + // console.log(`skipping ${file} (${this.formatDate(startTimestamp)} - ${this.formatDate(endTimestamp)})`); + continue; + } + console.log(`indexing ${file}`); + const csvData = fs.readFileSync(file, 'utf8'); + const data = parse(csvData, { + skip_empty_lines: true + }); + insertRecords(data); + } + } + } + + alreadyIndexed(sensorId, startTimestamp, endTimestamp) { + if (!this.indexRange) { + this.indexRange = {}; + } + if (!this.indexRange[sensorId]) { + const startIndexed = this.index.prepare(` + SELECT time_stamp + FROM record + WHERE sensor_id = ? + ORDER BY time_stamp + LIMIT 1 + `).get([sensorId]); + const endIndexed = this.index.prepare(` + SELECT time_stamp + FROM record + WHERE sensor_id = ? + ORDER BY time_stamp DESC + LIMIT 1 + `).get([sensorId]); + if (!startIndexed || !endIndexed) { + return false; + } + this.indexRange[sensorId] = { + start: parseInt(startIndexed.time_stamp), + end: parseInt(endIndexed.time_stamp) + }; + } + return (startTimestamp >= this.indexRange[sensorId].start && + endTimestamp <= this.indexRange[sensorId].end); + } + + migrateDatabase() { + let dbVersion = this.index.pragma('user_version', { simple: true }); + let migrationsDir = './db'; + let migrations = globSync(path.join(migrationsDir, '*.sql')); + migrations.sort(); + for (let file of migrations) { + let versionMatch = path.basename(file).match(/\d+/); + if (versionMatch) { + let migrationVersion = parseInt(versionMatch[0]); + if (dbVersion < migrationVersion) { + console.log(`migrating index db: ${versionMatch[0]}`); + this.index.transaction(() => { + let sql = fs.readFileSync(file, 'utf8'); + this.index.exec(sql); + })(); + } + this.index.pragma(`user_version = ${migrationVersion}`); + } + } + return this.index; + } + + saveTimeSeries() { + let sensors = []; + const threshold = Math.floor((Date.now() - 7 * 24 * 60 * 60 * 1000) / 1000); + const getRecords = this.index.prepare(` + SELECT sensor_id, time_stamp, pm25_atm_a, pm25_atm_b + FROM record + WHERE time_stamp > ? + ORDER BY time_stamp + `); + const series = {}; + const records = getRecords.all([threshold]); + for (let record of records) { + let date_time = new Date(parseInt(record.time_stamp) * 1000); + date_time.setMilliseconds(0); + date_time.setSeconds(0); + const minuteBin = Math.floor(date_time.getMinutes() / 10) * 10; + date_time.setMinutes(minuteBin); + date_time = date_time.toJSON().replace('.000Z', 'Z'); + if (!series[date_time]) { + series[date_time] = {}; + } + if (!series[date_time][record.sensor_id]) { + series[date_time][record.sensor_id] = []; + } + series[date_time][record.sensor_id].push(record.pm25_atm_a); + series[date_time][record.sensor_id].push(record.pm25_atm_b); + if (sensors.indexOf(record.sensor_id) < 0) { + sensors.push(record.sensor_id); + } + } + sensors.sort(); + + const date_times = Object.keys(series).sort(); + let count = 0; + const output = [ + ['date_time', ...sensors] + ]; + for (let date_time of date_times) { + let row = [date_time]; + for (let sensor_id of sensors) { + row.push(this.getAverage(series[date_time][sensor_id])); + } + output.push(row); + count++; + } + + let cols = null; + for (let line of output) { + if (!cols) { + cols = line.length; + console.log(`${cols} cols.`); + } else if (line.length != cols) { + console.log(line); + } + } + fs.writeFileSync('./data/time-series.csv', stringify(output), 'utf8'); + console.log(`Wrote ${count} rows to time-series.csv`); + } + + getAverage(numbers) { + if (typeof numbers == 'undefined') { + return '-'; + } + let sum = 0.0; + for (let number of numbers) { + sum += parseFloat(number); + } + return (sum / numbers.length).toFixed(2); + } } diff --git a/scripts/index.js b/scripts/index.js index 638e3ef..53a0f72 100644 --- a/scripts/index.js +++ b/scripts/index.js @@ -1,7 +1,4 @@ -import Data from "./data.js"; -import dotenv from "dotenv"; - -dotenv.config(); +import Data from './data.js'; const data = new Data([ 'pm2.5_alt_a', @@ -14,4 +11,6 @@ const data = new Data([ 'humidity', 'pressure' ]); -await data.load(); +await data.loadRecords(); +data.indexData(); +data.saveTimeSeries();