From 3d1817aa1e1fba8802326318a400605f819fc042 Mon Sep 17 00:00:00 2001 From: lext Date: Wed, 20 Nov 2019 19:52:56 +0200 Subject: [PATCH] Backed is fully functional. Logs are shown, but not yet stored. --- deploy.sh | 2 ++ docker/docker-compose-cpu.yml | 34 ++++++++++++++++++++ docker/docker-compose-gpu.yml | 4 +-- ouludeepknee/inference/app.py | 6 ++-- ouludeepknee/inference/pipeline.py | 51 ++++++++++++++++++------------ 5 files changed, 73 insertions(+), 24 deletions(-) diff --git a/deploy.sh b/deploy.sh index 0f6efef..ffd3e52 100644 --- a/deploy.sh +++ b/deploy.sh @@ -11,4 +11,6 @@ if [ ! -d "snapshots_knee_grading" ]; then sh ./fetch_snapshots.sh fi +docker-compose -f ./docker/docker-compose-$1.yml down +docker-compose -f ./docker/docker-compose-$1.yml build docker-compose -f ./docker/docker-compose-$1.yml up \ No newline at end of file diff --git a/docker/docker-compose-cpu.yml b/docker/docker-compose-cpu.yml index e69de29..7c1fb81 100644 --- a/docker/docker-compose-cpu.yml +++ b/docker/docker-compose-cpu.yml @@ -0,0 +1,34 @@ +version: "2.3" +services: + kneel: + image: "miptmloulu/kneel:cpu" + ports: + - "5000:5000" + volumes: + - type: bind + source: ../snapshots_release_kneel # The snapshots are stored in the root directory + target: /snapshots/ + read_only: true + entrypoint: ["python", "-u", "-m", "kneel.inference.app", + "--lc_snapshot_path", "/snapshots/lext-devbox_2019_07_14_16_04_41", + "--hc_snapshot_path", "/snapshots/lext-devbox_2019_07_14_19_25_40", + "--refine", "True", "--mean_std_path", "/snapshots/mean_std.npy", + "--deploy", "True", "--device", "cpu", "--port", "5000"] + deepknee-backend: + depends_on: + - kneel + build: + context: ../ + dockerfile: ./docker/Dockerfile.cpu + ports: + - "5001:5001" + volumes: + - type: bind + source: ../snapshots_knee_grading/ # The snapshots are stored in the root directory + target: /snapshots/ + read_only: true + environment: + - KNEEL_ADDR=http://kneel:5000 + entrypoint: ["python", "-m", "ouludeepknee.inference.app", + "--snapshots_path", "/snapshots/", + "--device", "cpu", "--deploy", "True"] diff --git a/docker/docker-compose-gpu.yml b/docker/docker-compose-gpu.yml index d20138c..62f4136 100644 --- a/docker/docker-compose-gpu.yml +++ b/docker/docker-compose-gpu.yml @@ -14,7 +14,7 @@ services: "--lc_snapshot_path", "/snapshots/lext-devbox_2019_07_14_16_04_41", "--hc_snapshot_path", "/snapshots/lext-devbox_2019_07_14_19_25_40", "--refine", "True", "--mean_std_path", "/snapshots/mean_std.npy", - "--deploy", "True", "--device", "cuda"] + "--deploy", "True", "--device", "cuda", "--port", "5000"] deepknee-backend: runtime: nvidia depends_on: @@ -30,7 +30,7 @@ services: target: /snapshots/ read_only: true environment: - - KNEEL_ADDR=http://127.0.0.1:5000 + - KNEEL_ADDR=http://kneel:5000 entrypoint: ["python", "-m", "ouludeepknee.inference.app", "--snapshots_path", "/snapshots/", "--device", "cuda", "--deploy", "True"] diff --git a/ouludeepknee/inference/app.py b/ouludeepknee/inference/app.py index e6723b7..6ee805a 100644 --- a/ouludeepknee/inference/app.py +++ b/ouludeepknee/inference/app.py @@ -13,7 +13,7 @@ from flask import Flask, request from flask import jsonify, make_response from gevent.pywsgi import WSGIServer - +import logging from ouludeepknee.inference.pipeline import KneeNetEnsemble app = Flask(__name__) @@ -69,13 +69,15 @@ def analyze_single_knee(): parser.add_argument('--deploy', type=bool, default=False) parser.add_argument('--logs', type=str, default='/tmp/deepknee.log') args = parser.parse_args() + logging.basicConfig(format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) + logger = logging.getLogger(f'deepknee-backend:app') net = KneeNetEnsemble(glob.glob(os.path.join(args.snapshots_path, "*", '*.pth')), mean_std_path=os.path.join(args.snapshots_path, 'mean_std.npy'), device=args.device) if args.deploy: - http_server = WSGIServer((args.deploy_addr, 5001), app) + http_server = WSGIServer((args.deploy_addr, 5001), app, log=logger) http_server.serve_forever() else: app.run(host=args.deploy_addr, port=5001, debug=True) diff --git a/ouludeepknee/inference/pipeline.py b/ouludeepknee/inference/pipeline.py index 7713b68..168889a 100644 --- a/ouludeepknee/inference/pipeline.py +++ b/ouludeepknee/inference/pipeline.py @@ -21,6 +21,7 @@ import requests from pydicom import dcmread from pydicom.filebase import DicomBytesIO +import logging from ouludeepknee.data.utils import read_dicom, process_xray @@ -58,6 +59,7 @@ class KneeNetEnsemble(nn.Module): def __init__(self, snapshots_paths, mean_std_path, device=None): super().__init__() self.states = [] + self.logger = logging.getLogger(f'deepknee-backend:pipeline') if device is None: if torch.cuda.is_available(): device = 'cuda' @@ -66,6 +68,7 @@ def __init__(self, snapshots_paths, mean_std_path, device=None): self.device = device for snap_path in snapshots_paths: self.states.append(torch.load(snap_path, map_location=self.device)) + self.logger.log(logging.INFO, f'Loaded weights from {snap_path}') self.cropper = CenterCrop(300) self.ohe = OneHotEncoder(sparse=False, categories=[range(5)]) @@ -95,6 +98,7 @@ def empty_gradient_arrays(self): self.grads_l1, self.grads_m1 = [], [] self.grads_l2, self.grads_m2 = [], [] self.grads_l3, self.grads_m3 = [], [] + self.logger.log(logging.INFO, f'Gradient arrays have been emptied') def init_networks_from_states(self): models = {} @@ -102,6 +106,7 @@ def init_networks_from_states(self): # Data Parallel was accidentally stored back in 2017. model = nn.DataParallel(KneeNet(64, 0.2, False)).to(self.device) model.load_state_dict(state) + self.logger.log(logging.INFO, f'Model {idx} state has been loaded') # Converting data parallel into a regular model model = model.module # Removing the dropout @@ -117,9 +122,11 @@ def init_networks_from_states(self): branch = fuse_bn_recursively(branch) model.branch = branch models[f'net{idx + 1}'] = deepcopy(model) + self.logger.log(logging.INFO, f'Model {idx} has been initialized') self.__dict__['_modules'].update(models) self.to(self.device) + self.logger.log(logging.INFO, f'The whole pipeline has been moved to {self.device}') def load_picture(self, fname, nbits=16, flip_left=False): """ @@ -128,7 +135,7 @@ def load_picture(self, fname, nbits=16, flip_left=False): Takes either full path to the image or the numpy array :return: """ - + self.logger.log(logging.DEBUG, f'Processing {nbits} bit {"left" if flip_left else "right"} image') if isinstance(fname, str): img = Image.open(fname) elif isinstance(fname, np.ndarray): @@ -157,26 +164,29 @@ def load_picture(self, fname, nbits=16, flip_left=False): lateral = self.patch_transform(lateral).to(self.device) medial = self.patch_transform(medial).to(self.device) - + self.logger.log(logging.DEBUG, f'Image pre-processing has been finished') return img_cropped, lateral.view(1, 1, 128, 128), medial.view(1, 1, 128, 128) - @staticmethod - def decompose_forward_avg(net, l, m): + def decompose_forward_avg(self, net, l, m): # Reducing the memory footprint. # We don't really need gradients to compute the features + self.logger.log(logging.INFO, f'Forward pass started for {hex(id(net))}') with torch.no_grad(): l_o = net.branch(l) m_o = net.branch(m) l_o_avg = F.adaptive_avg_pool2d(l_o, (1, 1)) m_o_avg = F.adaptive_avg_pool2d(m_o, (1, 1)) + self.logger.log(logging.DEBUG, f'Features have been extracted') # These variables will requre features as they will initiate the forward pass to the FC layer # From which we will get the gradients l_o_avg.requires_grad = True m_o_avg.requires_grad = True # A normal forward pass. Concatenating the outputs from the lateral and the medial sides + self.logger.log(logging.DEBUG, f'Pushing the feature maps through FC layer') concat = torch.cat([l_o_avg, m_o_avg], 1) # Passing the results through an FC layer o = net.final(concat.view(l.size(0), net.final.in_features)) + self.logger.log(logging.INFO, f'Model {hex(id(net))} finished predictions') return l_o, m_o, l_o_avg, m_o_avg, o def weigh_maps(self, weights, maps): @@ -190,12 +200,14 @@ def weigh_maps(self, weights, maps): return res def extract_gradcam_weighted_maps(self, o_l, o_m, wl, wm): + self.logger.log(logging.DEBUG, f'GradCAM-based weighing started') # After extracting the features, we weigh them based on the provided weights o_l = self.weigh_maps(wl, o_l) o_m = self.weigh_maps(wm, o_m) return F.relu(o_l), F.relu(o_m) def compute_gradcam(self, features, img_size, ps, smoothing=7): + self.logger.log(logging.INFO, f'GradCAM computation has been started') w_lateral, w_medial = self.grads_l1[0].data, self.grads_m1[0].data ol1, om1 = self.extract_gradcam_weighted_maps(features['net1'][0], features['net1'][1], w_lateral, w_medial) @@ -207,7 +219,7 @@ def compute_gradcam(self, features, img_size, ps, smoothing=7): l_out = (ol1 + ol2 + ol3) / 3. m_out = (om1 + om2 + om3) / 3. - + self.logger.log(logging.INFO, f'Creating the heatmap') heatmap = inverse_pair_mapping(l_out.detach().to('cpu').numpy(), np.fliplr(m_out.detach().to('cpu').numpy()), img_size, ps, smoothing) @@ -216,14 +228,8 @@ def compute_gradcam(self, features, img_size, ps, smoothing=7): return heatmap def forward(self, l, m): - self.grads_l1 = [] - self.grads_m1 = [] - - self.grads_l2 = [] - self.grads_m2 = [] - - self.grads_l3 = [] - self.grads_m3 = [] + self.logger.log(logging.INFO, f'Forward pass started') + self.empty_gradient_arrays() # Producing the branch outputs and registering the corresponding hooks for attention maps # Net 1 @@ -285,6 +291,7 @@ def predict_draw(self, fileobj_in, nbits=16, fname_suffix=None, path_dir_out=Non :return: tuple Image, Heatmap, probabilities """ + self.logger.log(logging.INFO, f'Prediction started') if fname_suffix is not None: pass elif isinstance(fileobj_in, str): @@ -293,6 +300,7 @@ def predict_draw(self, fileobj_in, nbits=16, fname_suffix=None, path_dir_out=Non fname_suffix = '' img, heatmap, probs = self.predict(x=fileobj_in, nbits=nbits, flip_left=flip_left) + self.logger.log(logging.INFO, f'Drawing the heatmap') img = np.asarray(img) img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) if flip_left: @@ -307,6 +315,7 @@ def predict_draw(self, fileobj_in, nbits=16, fname_suffix=None, path_dir_out=Non cv2.imwrite(tmp_fname, img_overlayed) # Making a bar plot for displaying probabilities + self.logger.log(logging.INFO, f'Drawing the vector with probabilities') plt.figure(figsize=(6, 1)) for kl in range(5): plt.text(kl - 0.2, 0.35, "%.2f" % np.round(probs[kl], 2), fontsize=15) @@ -326,24 +335,24 @@ def predict_draw(self, fileobj_in, nbits=16, fname_suffix=None, path_dir_out=Non if path_dir_out is not None: tmp_fname = os.path.join(path_dir_out, f'prob_{fname_suffix}.png') cv2.imwrite(tmp_fname, probs_bar) - + self.logger.log(logging.INFO, f'Sending the results back to the user') return img, img_overlayed, probs_bar, probs.squeeze().argmax() - @staticmethod - def localize_bilateral(dicom_raw, sizemm, pad): + def localize_bilateral(self, dicom_raw, sizemm, pad): files = {'dicom': dicom_raw} - + self.logger.log(logging.INFO, f'Sending the image to KNEEL: {os.environ["KNEEL_ADDR"]}') response = requests.post(f'{os.environ["KNEEL_ADDR"]}/predict/bilateral', files=files) landmarks = response.json() if landmarks['R'] is None: + self.logger.log(logging.INFO, f'Landmarks have not been localized. Returning None') return None - + self.logger.log(logging.INFO, f'Image decoding and pre-processing started') raw = DicomBytesIO(dicom_raw) dicom_data = dcmread(raw) img, spacing, dicom_data = read_dicom(dicom_data) img = process_xray(img, 5, 99, 255).astype(np.uint8) sizepx = int(np.round(sizemm / spacing)) - + self.logger.log(logging.DEBUG, f'Padding the image') row, col = img.shape tmp = np.zeros((row + 2 * pad, col + 2 * pad)) tmp[pad:pad + row, pad:pad + col] = img @@ -361,10 +370,12 @@ def localize_bilateral(dicom_raw, sizemm, pad): img_right = img[(rcy - sizepx // 2):(rcy + sizepx // 2), (rcx - sizepx // 2):(rcx + sizepx // 2)].astype(np.uint8) + self.logger.log(logging.INFO, f'Returning localized left and right knees') return img_left, img_right def predict_draw_bilateral(self, dicom_raw, sizemm, pad): - res_landmarks = KneeNetEnsemble.localize_bilateral(dicom_raw, sizemm, pad) + self.logger.log(logging.INFO, f'Received DICOM') + res_landmarks = self.localize_bilateral(dicom_raw, sizemm, pad) if res_landmarks is None: return None