Skip to content

Commit

Permalink
Allow for better parametrisation of GA_Detect.py command-line script
Browse files Browse the repository at this point in the history
This includes the ability to configure the metars file via the
command-line.  The loading of the metars file has been moved to the
main process to enable this.
  • Loading branch information
dnouri committed Jun 21, 2020
1 parent 589d1bb commit da748ce
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 35 deletions.
48 changes: 24 additions & 24 deletions GA_Detect.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,29 @@
"""A script to process OpenSky ADS-B data in an attempt to detect go-around events at an airport."""
from traffic.core import Traffic
from datetime import timedelta
import importlib
import multiprocessing as mp
from OS_Airports import VABB
import metar_parse as MEP
import OS_Funcs as OSF
import glob


def main(start_n, fidder, do_write):
import click


@click.command()
@click.option('--top-dir', default='./')
@click.option('--start-n', default=0)
@click.option('--do-write', default=True)
@click.option('--metars-file', default='VABB_METAR')
@click.option('--airport', default='VABB')
@click.option('--n-files-proc', default=55)
@click.option('--pool-proc', default=16)
@click.option('--verbose', default=False)
def main(top_dir, start_n, do_write, metars_file, airport,
n_files_proc, pool_proc, verbose):
"""The main code for detecting go-arounds.
Arguments:
start_n -- The index of the first file to read
fidder -- the id of an open file to write log information into
do_write -- boolean flag specifying whether to output data to textfile
"""
Expand All @@ -22,7 +33,6 @@ def main(start_n, fidder, do_write):
# Of which go-arounds
tot_n_ga = 0

top_dir = '/gf2/eodg/SRP002_PROUD_ADSBREP/GO_AROUNDS/VABB/'
# indir stores the opensky data
indir = top_dir + 'INDATA/'

Expand Down Expand Up @@ -68,10 +78,9 @@ def main(start_n, fidder, do_write):
colormap = {'GND': 'black', 'CL': 'green', 'CR': 'blue',
'DE': 'orange', 'LVL': 'purple', 'NA': 'red'}

# Number of files to open in one go
n_files_proc = 55

pool_proc = 100
metars = MEP.get_metars(metars_file, verbose=verbose)
rwy_list = getattr(importlib.import_module(f'OS_Airports.{airport}'),
'rwy_list')

f_data = []
pool = mp.Pool(processes=pool_proc)
Expand All @@ -80,9 +89,6 @@ def main(start_n, fidder, do_write):
print("Processing batch starting with "
+ str(main_count + 1).zfill(5) + " of "
+ str(fli_len).zfill(5))
fidder.write("Processing batch starting with "
+ str(main_count + 1).zfill(5) + " of "
+ str(fli_len).zfill(5) + '\n')

p_list = []
# First we load several files at once
Expand Down Expand Up @@ -113,11 +119,12 @@ def main(start_n, fidder, do_write):
if (flight.stop + timedelta(minutes=5) < end_time):
p_list.append(pool.apply_async(OSF.proc_fl,
args=(flight,
VABB.rwy_list,
metars,
rwy_list,
odirs,
colormap,
True,
False,)))
verbose,)))
else:
f_data.append(flight)

Expand Down Expand Up @@ -196,12 +203,5 @@ def main(start_n, fidder, do_write):
nogfid.close()


# Use this to start processing from a given file number.
# Can be helpful if processing fails at some point.
init_num = 0

fid = open('/home/proud/Desktop/log.log', 'w')

main(init_num, fid, False)

fid.close()
if __name__ == '__main__':
main()
8 changes: 2 additions & 6 deletions OS_Funcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from scipy.interpolate import UnivariateSpline as UniSpl
from traffic.core import Traffic
from datetime import timedelta
import metar_parse as MEP
import pandas as pd

import flightphase as flph
Expand All @@ -11,10 +10,6 @@
import numpy as np


# Read METARs from disk
metars = MEP.get_metars('/home/proud/Desktop/GoAround_Paper/VABB_METAR')


def estimate_rwy(df, rwy_list, verbose):
"""Guess which runway a flight is attempting to land on.
Expand Down Expand Up @@ -255,11 +250,12 @@ def check_ga(fd, verbose, first_pos=-1):
return ga_flag, bpt


def proc_fl(flight, check_rwys, odirs, colormap, do_save, verbose):
def proc_fl(flight, metars, check_rwys, odirs, colormap, do_save, verbose):
"""Filter, assign phases and determine go-around status for a given flight.
Inputs:
- A 'traffic' flight object
- A dict of METARS, each as a metobs class
- A list storing potential landing runways to check
- A 4-element list specifying various output directories:
- normal plot output
Expand Down
35 changes: 30 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ Requires:

Usage:
First you must download aircraft data, which can be done using the `OpenSky_Get_Data` script. You can then point `GA_Detect` at the download location to scan for go-arounds.
This tool is in very early development, so has manual tweaks that would ideally be changeable via a config file or directly via the command line call. The most important of these tweaks are listed below:

### `OpenSky_Get_Data.py`:

Expand All @@ -34,9 +33,35 @@ python OpenSky_Get_Data.py \
--outdir=INDATA --n-jobs=1
```

### In `GA_Detect.py`
The directory structure is set at the beginning of `main()`. You will probably want to adjust this to your own requirements.
### `GA_Detect.py`

`n_files_proc` specifies how many files to process simultaneously. This should be changed to the optimal value for your hardware.
The script that runs the actual go-around events.

`pool_proc` specifies the number of multiprocessing threads to use. I have found that this can be set slightly higher than the number of cores available, as cores are not fully utilised anyway.
Data is read and written based on a top-level directory. The default
is the current working directory, which can be overridden by passing
the `--top-dir` command-line option.

The file containing the appropriate METARS data can be passed using
the `--metars-file` option.

The airport can be specified using the `--airport` option. See the
`OS_Airports` subpackage which contains the runway definitions for the
currently supported airports.

The `--n-files-proc` option specifies how many files to process
simultaneously. This should be changed to the optimal value for your
hardware.

The `--pool-proc` option specifies the number of multiprocessing
threads to use. I have found that this can be set slightly higher than
the number of cores available, as cores are not fully utilised anyway.

The `GA_Detect.py` command-line script uses a few defaults. The
defaults are chosen to accommodate the defaults in the data fetching
scripts. As such, these two calls are equivalent:

```bash
python GA_Detect.py # does the same thing as the next command:
python GA_Detect.py \
--top-dir=. --metars-file=VABB_METAR --airport=VABB
```

0 comments on commit da748ce

Please sign in to comment.