-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathexperiments_runall.py
151 lines (117 loc) · 6.82 KB
/
experiments_runall.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import random
from clip_sampler import get_frames_from_video_path
from tqdm import trange, tqdm
import torch
import warnings
import gc
import numpy as np
warnings.simplefilter(action='ignore', category=FutureWarning)
import pandas as pd
from torch.utils.data import TensorDataset, SequentialSampler, DataLoader
from utils import load_pickle_file, get_missclassified
def run_experiment_all(model, sampling_strategy, video_paths, batch_size = 10, outer_batch_size = 100, num_examples=100, num_frames=16, frame_rate = 1, seed=10):
print("run experiment")
random.seed(seed)
all_indices = random.sample(range(0, len(video_paths)), num_examples)
new_dataset = TensorDataset(torch.IntTensor(all_indices))
dataloader = DataLoader(new_dataset, sampler=SequentialSampler(new_dataset), batch_size=outer_batch_size)
num_correct = 0
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
if sampling_strategy == "obj-detection-all":
SAMPLING_STRATS = ["obj-detection-top16","obj-detection-low16","obj-detection-top8","obj-detection-top4","obj-detection-top1","obj-detection-mixed"]
elif sampling_strategy == "position-all":
# SAMPLING_STRATS = ["position-fourths","position_beginning","position_middle","position_end","position-mixed"]
SAMPLING_STRATS = ["position_beginning","position_middle","position_end"]
elif sampling_strategy == "position-mixed-all":
SAMPLING_STRATS = ["position_mixed_5","position_mixed_3","position_mixed_1"]
else:
SAMPLING_STRATS = [sampling_strategy]
master_predicted_labels = [[] for i in range(len(SAMPLING_STRATS))]
master_actual_labels = [[] for i in range(len(SAMPLING_STRATS))]
j = 0
for batch in dataloader:
print("Iteration " + str(j) + ":")
torch.cuda.empty_cache()
indices = batch[0].numpy()
model_inputs, video_frame_indices = model.preprocess_videos_all(indices, video_paths, sampling_strategy, len(SAMPLING_STRATS), num_frames, frame_rate)
for i in range(len(model_inputs)):
sampling_model_inputs = model_inputs[i]
dataset = TensorDataset(torch.stack([x for x in sampling_model_inputs]).to(device))
prediction_ids = model.batch_predict(dataset, batch_size).cpu()
predicted_labels = [model.model.config.id2label[pred_id.item()] for pred_id in prediction_ids]
actual_labels = [video_paths[x].parent.name for x in indices]
master_actual_labels[i] = np.concatenate((master_actual_labels[i],actual_labels),axis=0)
master_predicted_labels[i] = np.concatenate((master_predicted_labels[i],predicted_labels),axis=0)
j = j + 1
for num in range(len(SAMPLING_STRATS)):
sampling_strategy = SAMPLING_STRATS[num]
video_paths_arr = []
video_ids_arr = []
video_index_arr = []
sampling_strategies = [sampling_strategy] * len(all_indices)
for i in all_indices:
video_path = video_paths[i]
video_paths_arr.append(video_path)
video_ids_arr.append(str(video_path.stem)[:11])
video_index_arr.append(i)
df = pd.DataFrame(list(zip(master_actual_labels[num], master_predicted_labels[num], video_paths_arr, video_ids_arr, sampling_strategies, video_index_arr)),
columns =["Actual Label", "Predicted Label", "Video Path",
"Video ID", "Sampling Strategy", "Video Index"])
df["Correct"] = df["Actual Label"] == df["Predicted Label"]
accuracy = sum(df["Correct"]) / len(df)
print(f"Accuracy for {sampling_strategy}: " + str(accuracy))
output_file_path = "outputs/" + sampling_strategy + "-" + str(seed) + "-" + str(num_examples) + "-" + str(num_frames) + ".csv"
df.to_csv(output_file_path, index=False)
def run_theoretical_best(model, sampling_strategy, video_paths, indices, batch_size = 10, outer_batch_size = 100, num_examples=100, num_frames=16, frame_rate = 1, seed=10):
print("run experiment")
random.seed(seed)
# all_indices = random.sample(range(0, len(video_paths)), num_examples)
all_indices = range(len(video_paths))
num_correct = 0
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
master_predicted_labels = []
master_actual_labels = []
master_correct_frames = []
torch.cuda.empty_cache()
new_dataset = TensorDataset(torch.IntTensor(indices))
dataloader = DataLoader(new_dataset, sampler=SequentialSampler(new_dataset), batch_size=outer_batch_size)
for batch in tqdm(dataloader, desc="Evaluating"):
torch.cuda.empty_cache()
sub_indices = batch[0].numpy()
model_inputs, video_frame_idxs = model.preprocess_all_frames(sub_indices, video_paths)
actual_labels = [video_paths[x].parent.name for x in sub_indices]
master_actual_labels = np.concatenate((master_actual_labels,actual_labels),axis=0)
for i in range(len(model_inputs)):
video_processed_frames = model_inputs[i]
correct_frames = []
dataset = TensorDataset(torch.stack(video_processed_frames).to(device))
prediction_ids = model.batch_predict_no_tqdm(dataset, batch_size).cpu()
predicted_labels = [model.model.config.id2label[pred_id.item()] for pred_id in prediction_ids]
for frame_label_index in range(len(predicted_labels)):
frame_label = predicted_labels[frame_label_index]
if frame_label == actual_labels[i]:
correct_frames.append(video_frame_idxs[i][frame_label_index])
master_predicted_labels = np.append(master_predicted_labels,frame_label)
break
if len(correct_frames) == 0:
# add the most recent predicted label
master_predicted_labels = np.append(master_predicted_labels,frame_label)
master_correct_frames.append(correct_frames)
print(master_correct_frames)
video_paths_arr = []
video_ids_arr = []
video_index_arr = []
sampling_strategies = [sampling_strategy] * len(all_indices)
for i in indices:
video_path = video_paths[i]
video_paths_arr.append(video_path)
video_ids_arr.append(str(video_path.stem)[:11])
video_index_arr.append(i)
df = pd.DataFrame(list(zip(master_actual_labels, master_predicted_labels, video_paths_arr, video_ids_arr, sampling_strategies, video_index_arr, master_correct_frames)),
columns =["Actual Label", "Predicted Label", "Video Path",
"Video ID", "Sampling Strategy", "Video Index", "First Correct Frame"])
df["Correct"] = df["Actual Label"] == df["Predicted Label"]
accuracy = sum(df["Correct"]) / len(df)
print(f"Accuracy: " + str(accuracy))
output_file_path = "outputs/" + sampling_strategy + "-" + str(seed) + "-" + str(num_examples) + "-" + str(num_frames) + ".csv"
df.to_csv(output_file_path, index=False)