Skip to content

Commit

Permalink
Upload file update (#647)
Browse files Browse the repository at this point in the history 
* Optimal solution (#644)

* uploading file and metadata moved to the same view/request;

* uploading file and its metadata moved to single view; preprocessing moved to a separate process; proper test switched off; urls updated accordingly;

* uploading file and its metadata moved to one view so proper changes on the frontend had to be applied;

* Optimal solution (#645)

* uploading file and metadata moved to the same view/request;

* uploading file and its metadata moved to single view; preprocessing moved to a separate process; proper test switched off; urls updated accordingly;

* uploading file and its metadata moved to one view so proper changes on the frontend had to be applied;

* storage path added

* sending email and handling failed preprocessing added

* function name updated so it doesn't cause conflicts

* file preprocessing wrapped as a separate function, so it's possible to mock it while testing

* tests updated to work for the new approach of data upload

* refreshing time set to 1 minute

* Optimal solution (#646)

* uploading file and metadata moved to the same view/request;

* uploading file and its metadata moved to single view; preprocessing moved to a separate process; proper test switched off; urls updated accordingly;

* uploading file and its metadata moved to one view so proper changes on the frontend had to be applied;

* storage path added

* sending email and handling failed preprocessing added

* function name updated so it doesn't cause conflicts

* file preprocessing wrapped as a separate function, so it's possible to mock it while testing

* tests updated to work for the new approach of data upload

* refreshing time set to 1 minute

* tests updated to test the view for getting datafile by id

* unnecessary printing removed

* not needed imports removed

* pytesmo updated

* condition updated to enable having directories in a zip file
  • Loading branch information
@sheenaze
sheenaze authored Apr 3, 2023
1 parent 3159543 commit 5f2e517
Show file tree
Hide file tree
Showing 15 changed files with 441 additions and 435 deletions.
4 changes: 2 additions & 2 deletions UI/src/app/modules/core/services/dataset/dataset-variable.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ export class DatasetVariableService {
}
}

getVariableById(variableId: number): Observable<DatasetVariableDto> {
getVariableById(variableId: number, refresh= false): Observable<DatasetVariableDto> {
//simplified implementation for demo purposes
if (this.singleRequestCache.isCached(variableId)) {
if (this.singleRequestCache.isCached(variableId) && !refresh) {
return this.singleRequestCache.get(variableId);
} else {
let getURL = DATASET_VARIABLE_URL + '/' + variableId;
Expand Down
20 changes: 12 additions & 8 deletions UI/src/app/modules/user-datasets/components/user-data-row/user-data-row.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ <h5 class="p-my-0"><span class="font-weight-normal">Size: </span> {{ getTheFileS
<h5 class="p-my-0"> Upload date: {{userDataset.upload_date| date: dateFormat :timeZone}} UTC
</h5>
</ng-template>
<div class="p-grid p-mx-0 p-px-0 p-my-0" [style]="{height: '100%'}">
<div class="p-grid p-mx-0 p-px-0 p-my-0" *ngIf="userDataset.metadata_submitted" [style]="{height: '100%'}">
<!-- dataset -->
<div class="p-col-5">
<div><strong>Dataset: </strong></div>
Expand Down Expand Up @@ -94,7 +94,8 @@ <h5 class="p-my-0"> Upload date: {{userDataset.upload_date| date: dateFormat :ti
'You can choose another name by clicking here. Please note, if the chosen variable name is incorrect, data validation will not be possible.' :
'We could not retrieve the applicable soil moisture variable name. Please choose the proper one from the list. A validation will fail if no proper variable is set.'}}"
>
Variable: <b>{{(variableName.shortName$|async)}} ({{variableName.prettyName$|async}}) [{{variableName.unit$|async}}] </b>
Variable: <b>{{(variableName.shortName$|async)}} ({{variableName.prettyName$|async}}
) [{{variableName.unit$|async}}] </b>
<span class="pi pi-question-circle small-edit-icon"></span>
</span>
<div *ngIf="(editVariable.opened)">
Expand Down Expand Up @@ -126,13 +127,13 @@ <h5 class="p-my-0"> Upload date: {{userDataset.upload_date| date: dateFormat :ti
<div class="p-col-5 p-fluid" [style]="{height: '100%'}">
<div><strong>Validation list: </strong></div>
<p-scrollPanel *ngIf="userDataset.validation_list.length > 0" [style]="{width: '80%', height: '80%'}">
<div *ngFor="let validation of userDataset.validation_list; index as ind">
{{ind + 1}}) <a routerLink="/validation-result/{{validation.val_id}}"
pTooltip="View validation">{{validation.val_name}}</a>
</div>
<div *ngFor="let validation of userDataset.validation_list; index as ind">
{{ind + 1}}) <a routerLink="/validation-result/{{validation.val_id}}"
pTooltip="View validation">{{validation.val_name}}</a>
</div>
</p-scrollPanel>
<div id="no-validation-box" *ngIf="userDataset.validation_list.length == 0">
<em> No validation has been run with this data </em>
<div class="no-validation-box" *ngIf="userDataset.validation_list.length == 0">
<em> No validation has been run with this data </em>
</div>
</div>
<!-- actions -->
Expand All @@ -153,6 +154,9 @@ <h5 class="p-my-0"> Upload date: {{userDataset.upload_date| date: dateFormat :ti
</div>
</div>
</div>
<div class="p-grid p-mx-0 p-px-0 p-my-0 no-validation-box" *ngIf="!userDataset.metadata_submitted" [style]="{height: '100%'}">
<h4> <span class="pi pi-spin pi-spinner" style="font-size: 2rem"></span><em> Your file is still being preprocessed. </em> </h4>
</div>

</p-panel>

11 changes: 9 additions & 2 deletions UI/src/app/modules/user-datasets/components/user-data-row/user-data-row.component.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,19 @@
font-size: 1rem;
}

#no-validation-box{
color: lightgray;
.no-validation-box{
color: darkgray;
height: 70%;
display: flex;
align-items:center;
justify-content: center;
}
.warning {
color: red;
}
@import "src/styles";
.expiry-icons{
cursor: auto;
font-size: 1.5rem;
padding: 5px;
}
61 changes: 45 additions & 16 deletions UI/src/app/modules/user-datasets/components/user-data-row/user-data-row.component.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Component, Input, OnInit} from '@angular/core';
import {Component, Input, OnDestroy, OnInit} from '@angular/core';
import {UserDataFileDto} from '../../services/user-data-file.dto';
import {BehaviorSubject, Observable} from 'rxjs';
import {UserDatasetsService} from '../../services/user-datasets.service';
Expand All @@ -15,35 +15,33 @@ import {AuthService} from '../../../core/services/auth/auth.service';
templateUrl: './user-data-row.component.html',
styleUrls: ['./user-data-row.component.scss']
})
export class UserDataRowComponent implements OnInit {
export class UserDataRowComponent implements OnInit, OnDestroy {

@Input() userDataset: UserDataFileDto;
datasetName$: BehaviorSubject<string> = new BehaviorSubject<string>('');
versionName$: BehaviorSubject<string> = new BehaviorSubject<string>('');
variableName: { shortName$: BehaviorSubject<string>, prettyName$: BehaviorSubject<string>, unit$: BehaviorSubject<string> } =
variableName: {
shortName$: BehaviorSubject<string>,
prettyName$: BehaviorSubject<string>,
unit$: BehaviorSubject<string>
} =
{
shortName$: new BehaviorSubject<string>(''),
prettyName$: new BehaviorSubject<string>(''),
unit$: new BehaviorSubject<string>('')
};
variableUnit: string;

datasetFieldName = 'dataset_name';
versionFieldName = 'version_name';
variableFieldName = 'variable_name';
latFieldName = 'lat_name';
lonFieldName = 'lon_name';
timeFiledName = 'time_name';

editDataset = {opened: false};
editVersion = {opened: false};
editVariable = {opened: false};
editLatName = {opened: false};
editLonName = {opened: false};
editTimeName = {opened: false};

dateFormat = 'medium';
timeZone = 'UTC';
filePreprocessingStatus: any;

// variables$: Observable<DatasetVariableDto>[] = [];

Expand All @@ -62,12 +60,8 @@ export class UserDataRowComponent implements OnInit {
this.datasetVersionService.getVersionById(this.userDataset.version).subscribe(versionData => {
this.versionName$.next(versionData.pretty_name);
});
this.datasetVariableService.getVariableById(this.userDataset.variable).subscribe(variableData => {
this.variableName.shortName$.next(variableData.short_name);
this.variableName.prettyName$.next(variableData.pretty_name);
this.variableName.unit$.next(variableData.unit);
// this.variableUnit = variableData.unit;
});
this.updateVariable();
this.refreshFilePreprocessingStatus();
}

removeDataset(dataFileId: string): void {
Expand All @@ -80,6 +74,14 @@ export class UserDataRowComponent implements OnInit {
});
}

updateVariable(): void {
this.datasetVariableService.getVariableById(this.userDataset.variable, true).subscribe(variableData => {
this.variableName.shortName$.next(variableData.short_name);
this.variableName.prettyName$.next(variableData.pretty_name);
this.variableName.unit$.next(variableData.unit);
});
}

getDataset(datasetId): Observable<DatasetDto> {
return this.datasetService.getDatasetById(datasetId);
}
Expand Down Expand Up @@ -132,4 +134,31 @@ export class UserDataRowComponent implements OnInit {
return this.userDatasetService.getTheSizeInProperUnits(this.userDataset.file_size);
}


refreshFilePreprocessingStatus(): void {
if (!this.userDataset.metadata_submitted) {
this.filePreprocessingStatus = setInterval(() => {
this.userDatasetService.getUserDataFileById(this.userDataset.id).subscribe(data => {
if (data.metadata_submitted) {
this.updateVariable();
if (this.variableName.prettyName$.value !== 'none') {
this.userDatasetService.refresh.next(true);
}
}
},
() => {
this.userDatasetService.refresh.next(true);
this.toastService.showErrorWithHeader('File preprocessing failed',
'File could not be preprocessed. Please make sure that you are uploading a proper file and if the ' +
'file fulfills our requirements', 10000);
});
}, 60 * 1000); // one minute
}
}


ngOnDestroy(): void {
clearInterval(this.filePreprocessingStatus);
}

}
31 changes: 12 additions & 19 deletions UI/src/app/modules/user-datasets/components/user-file-upload/user-file-upload.component.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,10 @@ export class UserFileUploadComponent implements OnInit {
private verifyZipContent(): void {
const zip = new JSZip();
zip.loadAsync(this.file).then(contents => {
console.log(contents.files);
const files = Object.keys(contents.files).filter(key =>
!['nc', 'nc4', 'csv', 'yml'].includes(key.split('.').reverse()[0]));
if (files.length !== 0){
!['nc', 'nc4', 'csv', 'yml'].includes(key.split('.').reverse()[0]) && !contents.files[key].dir);
if (files.length !== 0) {
this.toastService.showErrorWithHeader('File can not be uploaded',
'The zip file you are trying to upload contains files with no acceptable extensions (i.e. netCDF or csv + yml');
this.file = null;
Expand All @@ -64,7 +65,7 @@ export class UserFileUploadComponent implements OnInit {
this.file = event.target.files[0];
this.isFileTooBig = false;

if (this.authService.currentUser.space_left && this.file.size > this.authService.currentUser.space_left){
if (this.authService.currentUser.space_left && this.file.size > this.authService.currentUser.space_left) {
this.isFileTooBig = true;
this.file = null;
return null;
Expand Down Expand Up @@ -92,34 +93,26 @@ export class UserFileUploadComponent implements OnInit {
if (this.file) {
this.name = 'uploadedFile';
this.spinnerVisible = true;
const upload$ = this.userDatasetService.userFileUpload(this.name, this.file, this.fileName)
const upload$ = this.userDatasetService.userFileUpload(this.name, this.file, this.fileName, this.metadataForm.value)
.pipe(finalize(() => this.reset));

this.uploadSub = upload$.subscribe(event => {
if (event.type === HttpEventType.UploadProgress) {
this.uploadProgress.next(Math.round(100 * (event.loaded / event.total)));
} else if (event.type === HttpEventType.Response) {
this.userDatasetService.sendMetadata(this.metadataForm.value, event.body.id).subscribe(() => {
this.userDatasetService.refresh.next(true);
this.authService.init();
this.resetFile();
},
(message) => {
this.spinnerVisible = false;
this.toastService.showErrorWithHeader('Metadata not saved.',
`${message.error.error}.\n Provided metadata could not be saved. Please try again or contact our team.`);
},
() => {
this.spinnerVisible = false;
this.metadataForm.reset('');
});
} else {
this.userDatasetService.refresh.next(true);
this.authService.init();
this.resetFile();
}
},
(message) => {
this.spinnerVisible = false;
this.toastService.showErrorWithHeader('File not saved',
`${message.error.error}.\n File could not be uploaded. Please try again or contact our team.`);
},
() => {
this.spinnerVisible = false;
this.metadataForm.reset('');
}
);
}
Expand Down
3 changes: 2 additions & 1 deletion UI/src/app/modules/user-datasets/services/user-data-file.dto.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ export class UserDataFileDto {
public upload_date: Date,
public is_used_in_validation: boolean,
public file_size: number,
public validation_list: {val_id: string, val_name: string}[]
public validation_list: {val_id: string, val_name: string}[],
public metadata_submitted: boolean
) {
}
}
33 changes: 10 additions & 23 deletions UI/src/app/modules/user-datasets/services/user-datasets.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,9 @@ const urlPrefix = environment.API_URL + 'api';
const uploadUserDataUrl: string = urlPrefix + '/upload-user-data';
const userDataListUrl: string = urlPrefix + '/get-list-of-user-data-files';
const userDataDeleteUrl: string = urlPrefix + '/delete-user-datafile';
const userDataMetadataUrl: string = urlPrefix + '/user-file-metadata';
const userDataTestUrl: string = urlPrefix + '/test-user-dataset';
const updateMetadataUrl: string = urlPrefix + '/update-metadata';
const userDataFileUrl: string = urlPrefix + '/get-user-file-by-id';

// const validateUserDataUrl: string = urlPrefix + '/validate-user-data';

const csrfToken = '{{csrf_token}}';
const headers = new HttpHeaders({'X-CSRFToken': csrfToken});
Expand All @@ -28,32 +26,29 @@ export class UserDatasetsService {

constructor(private httpClient: HttpClient) { }

userFileUpload(name, file, fileName): Observable<any> {
userFileUpload(name, file, fileName, metadata): Observable<any> {
const formData = new FormData();
formData.append(name, file, fileName);
const uploadUrl = uploadUserDataUrl + '/' + fileName + '/';
return this.httpClient.post(uploadUrl, formData.get(name), {headers, reportProgress: true, observe: 'events', responseType: 'json'});
const fileHeaders = new HttpHeaders({'X-CSRFToken': csrfToken, fileMetadata: JSON.stringify(metadata)});
return this.httpClient.post(uploadUrl, formData.get(name),
{headers: fileHeaders, reportProgress: true, observe: 'events', responseType: 'json'});
}

getUserDataList(): Observable<UserDataFileDto[]>{
return this.httpClient.get<UserDataFileDto[]>(userDataListUrl);
}

getUserDataFileById(fileId: string): Observable<UserDataFileDto>{
const userDataFileUrlWithId = userDataFileUrl + '/' + fileId + '/';
return this.httpClient.get<UserDataFileDto>(userDataFileUrlWithId);
}

deleteUserData(dataFileId: string): Observable<any>{
const deleteUrl = userDataDeleteUrl + '/' + dataFileId + '/';
return this.httpClient.delete(deleteUrl, {headers});
}

sendMetadata(metadataForm: any, fileId: string): Observable<any> {
const metadataUrl = userDataMetadataUrl + '/' + fileId + '/';
return this.httpClient.post(metadataUrl, metadataForm, {observe: 'response', responseType: 'json'});
}

testDataset(dataFileId: string): Observable<any>{
const testUrl = userDataTestUrl + '/' + dataFileId + '/';
return this.httpClient.get(testUrl);
}

updateMetadata(fieldName: string, fieldValue: string, dataFileId: string): Observable<any>{
const updateUrl = updateMetadataUrl + '/' + dataFileId + '/';
return this.httpClient.put(updateUrl, {field_name: fieldName, field_value: fieldValue});
Expand All @@ -76,12 +71,4 @@ export class UserDatasetsService {

return `${Math.round(properSize * 10) / 10} ${units}`;
}

// userFileValidate(name, file, filename): Observable<any> {
// const formData = new FormData();
// formData.append(name, file, filename);
// const validateUserDataUrlWithFileName = validateUserDataUrl + '/' + file.name + '/';
// return this.httpClient.put(validateUserDataUrlWithFileName, {file: formData.getAll(name)});
// }

}
Loading

0 comments on commit 5f2e517

Please sign in to comment.