Skip to content

Commit

Permalink
Merge pull request #94 from mao-wfs/#92
Browse files Browse the repository at this point in the history 
Add thermometer module
  • Loading branch information
@kiwakami
kiwakami authored Nov 22, 2023
2 parents fa731b1 + a590993 commit ac15df7
Show file tree
Hide file tree
Showing 5 changed files with 273 additions and 0 deletions.
100 changes: 100 additions & 0 deletions data/thermometer_20220430T150000.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
2022/05/01,00:00:00,0, 3.8,198, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.8, 8.6,865
2022/05/01,00:01:00,0, 4.2,192, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,00:02:00,0, 3.6,199, +5.1, 8.7, 0,00.0,+00.0,00.0, +5.2, 8.8,865
2022/05/01,00:03:00,0, 5.0,209, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.6, 8.4,865
2022/05/01,00:04:00,0, 4.7,203, +5.2, 8.7, 0,00.0,+00.0,00.0, +4.9, 8.6,865
2022/05/01,00:05:00,0, 5.5,210, +5.2, 8.8, 0,00.0,+00.0,00.0, +4.7, 8.5,865
2022/05/01,00:06:00,0, 5.9,196, +5.2, 8.8, 0,00.0,+00.0,00.0, +4.8, 8.5,865
2022/05/01,00:07:00,0, 4.0,190, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.6, 8.4,865
2022/05/01,00:08:00,0, 4.6,194, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.5, 8.3,864
2022/05/01,00:09:00,0, 4.5,189, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.7, 8.5,864
2022/05/01,00:10:00,0, 4.5,204, +5.5, 8.9, 0,00.0,+00.0,00.0, +4.9, 8.6,865
2022/05/01,00:11:00,0, 4.1,203, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.3, 8.2,865
2022/05/01,00:12:00,0, 6.1,197, +5.5, 8.9, 0,00.0,+00.0,00.0, +4.6, 8.4,864
2022/05/01,00:13:00,0, 6.6,201, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.3, 8.2,864
2022/05/01,00:14:00,0, 6.1,209, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.5, 8.3,864
2022/05/01,00:15:00,0, 6.3,212, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.7, 8.5,864
2022/05/01,00:16:00,0, 5.7,195, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.4, 8.3,864
2022/05/01,00:17:00,0, 5.6,204, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.6, 8.4,865
2022/05/01,00:18:00,0, 5.3,201, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.7, 8.5,865
2022/05/01,00:19:00,0, 6.5,199, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.3, 8.3,865
2022/05/01,00:20:00,0, 5.4,196, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.4, 8.3,864
2022/05/01,00:21:00,0, 5.9,204, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.5, 8.3,864
2022/05/01,00:22:00,0, 7.4,202, +5.2, 8.7, 0,00.0,+00.0,00.0, +4.5, 8.4,864
2022/05/01,00:23:00,0, 5.6,193, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,00:24:00,0, 5.1,201, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.5, 8.4,865
2022/05/01,00:25:00,0, 5.3,190, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,00:26:00,0, 6.1,199, +4.9, 8.5, 0,00.0,+00.0,00.0, +4.3, 8.2,865
2022/05/01,00:27:00,0, 4.5,203, +5.2, 8.7, 0,00.0,+00.0,00.0, +4.5, 8.3,865
2022/05/01,00:28:00,0, 5.1,196, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.5, 8.4,865
2022/05/01,00:29:00,0, 4.3,187, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.7, 8.5,865
2022/05/01,00:30:00,0, 5.4,192, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,00:31:00,0, 4.5,210, +5.1, 8.6, 0,00.0,+00.0,00.0, +4.6, 8.5,865
2022/05/01,00:32:00,0, 4.8,206, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,00:33:00,0, 5.0,197, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.1, 8.1,865
2022/05/01,00:34:00,0, 4.0,204, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,00:35:00,0, 5.6,196, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.8, 8.5,865
2022/05/01,00:36:00,0, 2.9,197, +5.3, 8.8, 0,00.0,+00.0,00.0, +4.9, 8.6,865
2022/05/01,00:37:00,0, 5.5,194, +5.2, 8.7, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,00:38:00,0, 5.2,193, +5.0, 8.7, 0,00.0,+00.0,00.0, +4.0, 8.1,865
2022/05/01,00:39:00,0, 4.0,215, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.5, 8.3,865
2022/05/01,00:40:00,0, 2.6,208, +4.9, 8.5, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,00:41:00,0, 4.8,189, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.7, 8.5,864
2022/05/01,00:42:00,0, 4.2,193, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,00:43:00,0, 3.4,192, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.4, 8.3,864
2022/05/01,00:44:00,0, 2.7,199, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.3, 8.3,864
2022/05/01,00:45:00,0, 4.0,203, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.3, 8.3,865
2022/05/01,00:46:00,0, 3.6,186, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,00:47:00,0, 4.6,193, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,00:48:00,0, 3.6,200, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,00:49:00,0, 2.9,203, +4.7, 8.5, 0,00.0,+00.0,00.0, +4.1, 8.1,865
2022/05/01,00:50:00,0, 5.1,202, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.3, 8.2,865
2022/05/01,00:51:00,0, 4.8,189, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.3, 8.2,865
2022/05/01,00:52:00,0, 3.1,203, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,00:53:00,0, 3.2,193, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.6, 8.4,865
2022/05/01,00:54:00,0, 2.7,201, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.5, 8.3,865
2022/05/01,00:55:00,0, 2.8,203, +4.5, 8.3, 0,00.0,+00.0,00.0, +4.1, 8.1,864
2022/05/01,00:56:00,0, 2.0,226, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,00:57:00,0, 4.2,233, +5.1, 8.7, 0,00.0,+00.0,00.0, +4.1, 8.2,865
2022/05/01,00:58:00,0, 3.8,213, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.4, 8.4,865
2022/05/01,00:59:00,0, 5.3,196, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.5, 8.3,865
2022/05/01,01:00:00,0, 3.9,233, +4.6, 8.3, 0,00.0,+00.0,00.0, +4.3, 8.3,865
2022/05/01,01:01:00,0, 6.2,213, +4.9, 8.6, 0,00.0,+00.0,00.0, +3.8, 8.0,865
2022/05/01,01:02:00,0, 3.7,206, +4.8, 8.4, 0,00.0,+00.0,00.0, +4.1, 8.1,865
2022/05/01,01:03:00,0, 5.0,217, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.5, 8.4,865
2022/05/01,01:04:00,0, 3.6,202, +4.7, 8.5, 0,00.0,+00.0,00.0, +3.9, 8.0,865
2022/05/01,01:05:00,0, 5.7,202, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,01:06:00,0, 4.7,212, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.0, 8.1,865
2022/05/01,01:07:00,0, 4.9,205, +5.0, 8.6, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,01:08:00,0, 3.8,229, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,01:09:00,0, 4.1,214, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.4, 8.3,865
2022/05/01,01:10:00,0, 4.3,206, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.3, 8.2,865
2022/05/01,01:11:00,0, 4.2,212, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,865
2022/05/01,01:12:00,0, 4.7,204, +4.7, 8.4, 0,00.0,+00.0,00.0, +3.8, 8.0,865
2022/05/01,01:13:00,0, 5.5,213, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,01:14:00,0, 5.9,205, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,01:15:00,0, 4.4,204, +4.5, 8.3, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,01:16:00,0, 3.7,186, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,01:17:00,0, 5.5,189, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.3, 8.2,864
2022/05/01,01:18:00,0, 5.3,203, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,01:19:00,0, 5.2,190, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.3, 8.2,864
2022/05/01,01:20:00,0, 6.1,198, +4.9, 8.6, 0,00.0,+00.0,00.0, +4.1, 8.1,864
2022/05/01,01:21:00,0, 4.8,194, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.1, 8.2,864
2022/05/01,01:22:00,0, 4.5,187, +4.8, 8.5, 0,00.0,+00.0,00.0, +3.9, 8.0,864
2022/05/01,01:23:00,0, 3.7,184, +4.7, 8.4, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,01:24:00,0, 5.1,216, +4.5, 8.3, 0,00.0,+00.0,00.0, +3.8, 8.0,864
2022/05/01,01:25:00,0, 3.1,236, +4.9, 8.5, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,01:26:00,0, 1.8,216, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.4, 8.3,864
2022/05/01,01:27:00,0, 3.9,192, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,01:28:00,0, 3.3,182, +4.7, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,01:29:00,0, 4.5,192, +4.7, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,01:30:00,0, 2.8,209, +4.5, 8.3, 0,00.0,+00.0,00.0, +4.5, 8.4,864
2022/05/01,01:31:00,0, 3.1,176, +5.0, 8.6, 0,00.0,+00.0,00.0, +3.9, 8.0,864
2022/05/01,01:32:00,0, 1.6,221, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.5, 8.4,864
2022/05/01,01:33:00,0, 4.1,223, +4.7, 8.5, 0,00.0,+00.0,00.0, +4.2, 8.2,864
2022/05/01,01:34:00,0, 3.1,219, +4.8, 8.5, 0,00.0,+00.0,00.0, +3.9, 8.0,864
2022/05/01,01:35:00,0, 3.8,193, +4.7, 8.4, 0,00.0,+00.0,00.0, +3.8, 7.9,864
2022/05/01,01:36:00,0, 3.6,203, +4.4, 8.3, 0,00.0,+00.0,00.0, +3.7, 7.9,864
2022/05/01,01:37:00,0, 2.9,199, +4.8, 8.5, 0,00.0,+00.0,00.0, +4.6, 8.5,864
2022/05/01,01:38:00,0, 2.9,205, +4.6, 8.4, 0,00.0,+00.0,00.0, +4.0, 8.1,864
2022/05/01,01:39:00,0, 2.7,208, +4.4, 8.3, 0,00.0,+00.0,00.0, +4.4, 8.3,864
1 change: 1 addition & 0 deletions mao_merge_45m/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@
from . import correlator
from . import merge
from . import sam45
from . import thermometer
from . import weather
3 changes: 3 additions & 0 deletions mao_merge_45m/merge.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def merge(
path_weather_zarr: Optional[Path] = None,
path_antenna_zarr: Optional[Path] = None,
path_sam45_zarr: Optional[Path] = None,
path_thermometer_zarr: Optional[Path] = None,
interpolation: str = "linear",
time_offset: int = 0,
overwrite: bool = False,
Expand All @@ -38,6 +39,7 @@ def merge(
path_weather_zarr: Path of the weather Zarr file.
path_antenna_zarr: Path of the antenna Zarr file.
path_sam45_zarr: Path of the SAM45 Zarr file.
path_thermometer_zarr: Path of the thermometer Zarr file.
interpolation: Method of interpolation of log data.
time_offset: Offset time in units of ms to add to correlator data
overwrite: Whether to overwrite the merged Zarr file if exists.
Expand Down Expand Up @@ -76,6 +78,7 @@ def merge(
path_weather_zarr,
path_antenna_zarr,
path_sam45_zarr,
path_thermometer_zarr,
):
if path is None:
continue
Expand Down
142 changes: 142 additions & 0 deletions mao_merge_45m/thermometer.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
# standard library
from dataclasses import dataclass
from pathlib import Path
from typing import Literal, Optional, Sequence, Union


# third-party packages
import numpy as np
import pandas as pd
from dask.diagnostics import ProgressBar
from xarray_dataclasses import AsDataset, Attr, Data, Dataof


# constants
LOG_COLS = "time", "temperature_1p5m", "sunshine_flag", "temperature_30m"
JST_HOURS = np.timedelta64(9, "h")
DATE_FORMAT = "%Y/%m/%d,%H:%M:%S"


# type hints
T = Literal["time"]


# dataclasses
@dataclass
class Temperature1p5m:
data: Data[T, float] = 0.0
long_name: Attr[str] = "Temperature at 1.5m"
units: Attr[str] = "deg C"


@dataclass
class Temperature30m:
data: Data[T, float] = 0.0
long_name: Attr[str] = "Temperature at 30m"
units: Attr[str] = "deg C"


@dataclass
class SunshineFlag:
data: Data[T, bool] = False
long_name: Attr[str] = "Sunshine flag"


@dataclass
class Temperature(AsDataset):
"""Representation of antenna logs in xarray."""

temperature_1p5m: Dataof[Temperature1p5m] = 0.0
"""Temperature at 1.5 m above the ground surface at the observation building."""

temperature_30m: Dataof[Temperature30m] = 0.0
"""Temperature at 30 m above the ground surface at the observation building."""

sunshine_flag: Dataof[SunshineFlag] = 0.0
"""Sunshine flag."""


def convert(
path_log: Union[Sequence[Path], Path],
path_zarr: Optional[Path] = None,
*,
length_per_chunk: int = 1000000,
overwrite: bool = False,
progress: bool = False,
) -> Path:
"""Convert a raw log file(s) to a formatted Zarr file.
This function will make a one-dimensional antenna log outputs
with time metadata derived from the raw log file.
Args:
path_log: Path(s) of the raw log file(s).
path_zarr: Path of the formatted Zarr file (optional).
length_per_chunk: Length per chunk in the Zarr file.
overwrite: Whether to overwrite the formatted Zarr file if exists.
progress: Whether to show a progress bar.
Returns:
Path of the formatted Zarr file.
Raises:
FileExistsError: Raised if the formatted Zarr file exists
and overwriting is not allowed (default).
Notes:
The timezone of the Zarr file is not JST but UTC.
"""
# check the existence of the Zarr file
if isinstance(path_log, Path):
path_log = [path_log]

if path_zarr is None:
path_zarr = path_log[0].with_suffix(".zarr")

if path_zarr.exists() and not overwrite:
raise FileExistsError(f"{path_zarr} already exists.")

# read log file(s) and convert them to DataFrame(s)
df = pd.DataFrame(
columns=LOG_COLS[1:],
index=pd.DatetimeIndex([], name=LOG_COLS[0]),
)

for path in path_log:
df_ = pd.read_csv(
path,
header=None,
# index_col=0,
names=[
"Date_YMD",
"Time_HMS",
"temperature_15",
"sunshine_flag",
"temperature_30",
],
usecols=[0, 1, 5, 7, 11],
)

# 1行目と2行目を合成して新しい1行を作成
dt = df_["Date_YMD"].str.cat(df_["Time_HMS"], sep=",")
# 日付と時刻の列を結合してtimestamp列を作成 --> indexに設定
index = pd.to_datetime(dt, format=DATE_FORMAT)
df_.set_index(index, inplace=True)
# 不要な列を削除
df_ = df_.drop(columns=["Date_YMD", "Time_HMS"])
df = pd.concat([df, df_])

# write DataFrame(s) to the Zarr file
print(df)
ds = Temperature.new(df.temperature_15, df.temperature_30, df.sunshine_flag)
ds = ds.assign_coords(time=ds.time - JST_HOURS)
ds = ds.chunk(length_per_chunk)

if progress:
with ProgressBar():
ds.to_zarr(path_zarr, mode="w")
else:
ds.to_zarr(path_zarr, mode="w")

return path_zarr
27 changes: 27 additions & 0 deletions tests/test_thermometer.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# standard library
from pathlib import Path
from tempfile import NamedTemporaryFile, TemporaryDirectory


# third-party packages
import pandas as pd
import xarray as xr
from mao_merge_45m.thermometer import convert


# constants
TEST_TXT = Path("data") / "thermometer_20220430T150000.txt"


# test function
def test_convert() -> None:
"""Test whether the data of a txt file is correctly parsed."""
with TemporaryDirectory() as zarr:
path_zarr = Path(zarr)
path_zarr = convert(TEST_TXT, path_zarr, overwrite=True)
data_ds = xr.open_zarr(path_zarr)

assert data_ds.time[0] == pd.Timestamp("2022-04-30T15:00:00.000000000")
assert data_ds.temperature_1p5m[0] == 5.1
assert data_ds.temperature_30m[0] == 4.8
assert data_ds.sunshine_flag[0] == False

0 comments on commit ac15df7

Please sign in to comment.