Skip to content

Commit

Permalink
Refactor cURL use into helper functions with local handles (#92)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattjala
mattjala authored Mar 19, 2024
1 parent ba9d98f commit 420ff43
Show file tree
Hide file tree
Showing 10 changed files with 824 additions and 2,419 deletions.
462 changes: 399 additions & 63 deletions src/rest_vol.c
View file

Large diffs are not rendered by default.

16 changes: 16 additions & 0 deletions src/rest_vol.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,12 @@ typedef struct dataset_read_info {

typedef enum transfer_type_t { UNINIT = 0, READ = 1, WRITE = 2 } transfer_type_t;

typedef enum content_type_t {
CONTENT_TYPE_UNINIT = 0,
CONTENT_TYPE_JSON = 1,
CONTENT_TYPE_OCTET_STREAM = 2
} content_type_t;

typedef struct dataset_transfer_info {
struct curl_slist *curl_headers;
char *host_headers;
Expand Down Expand Up @@ -759,6 +765,16 @@ herr_t RV_curl_multi_perform(CURL *curl_multi_ptr, dataset_transfer_info *transf
herr_t RV_dataset_read_cb(hid_t mem_type_id, hid_t mem_space_id, hid_t file_type_id, hid_t file_space_id,
void *buf, struct response_buffer resp_buffer);

/* Helper functions for cURL requests to the server */
long RV_curl_delete(CURL *curl_handle, server_info_t *server_info, const char *request_endpoint,
const char *filename);
long RV_curl_put(CURL *curl_handle, server_info_t *server_info, const char *request_endpoint,
const char *filename, upload_info *uinfo, content_type_t content_type);
long RV_curl_get(CURL *curl_handle, server_info_t *server_info, const char *request_endpoint,
const char *filename, content_type_t content_type);
long RV_curl_post(CURL *curl_handle, server_info_t *server_info, const char *request_endpoint,
const char *filename, const char *post_data, size_t post_size, content_type_t content_type);

/* Dtermine if datatype conversion is necessary */
htri_t RV_need_tconv(hid_t src_type_id, hid_t dst_type_id);

Expand Down
1,274 changes: 197 additions & 1,077 deletions src/rest_vol_attr.c
View file

Large diffs are not rendered by default.

12 changes: 6 additions & 6 deletions src/rest_vol_config.h.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,13 @@
/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
significant byte first (like Motorola and SPARC, unlike Intel). */
#if defined AC_APPLE_UNIVERSAL_BUILD
#if defined __BIG_ENDIAN__
#define WORDS_BIGENDIAN 1
#endif
# if defined __BIG_ENDIAN__
# define WORDS_BIGENDIAN 1
# endif
#else
#ifndef WORDS_BIGENDIAN
#undef WORDS_BIGENDIAN
#endif
# ifndef WORDS_BIGENDIAN
# undef WORDS_BIGENDIAN
# endif
#endif

/* Define to empty if `const' does not conform to ANSI C. */
Expand Down
140 changes: 20 additions & 120 deletions src/rest_vol_dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,15 +127,13 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
{
RV_object_t *parent = (RV_object_t *)obj;
RV_object_t *new_dataset = NULL;
curl_off_t create_request_body_len = 0;
size_t host_header_len = 0;
size_t create_request_body_len = 0;
size_t path_size = 0;
size_t path_len = 0;
char *host_header = NULL;
char *create_request_body = NULL;
char request_url[URL_MAX_LENGTH];
const char *base_URL = NULL;
int url_len = 0;
char request_endpoint[URL_MAX_LENGTH];
int url_len = 0;
long http_response;
void *ret_value = NULL;

#ifdef RV_CONNECTOR_DEBUG
Expand All @@ -152,8 +150,6 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na

if (H5I_FILE != parent->obj_type && H5I_GROUP != parent->obj_type)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object not a file or group");
if ((base_URL = parent->domain->u.file.server_info.base_URL) == NULL)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object does not have valid server URL");

/* Check for write access */
if (!(parent->domain->u.file.intent & H5F_ACC_RDWR))
Expand Down Expand Up @@ -202,42 +198,13 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
new_dataset->u.dataset.dcpl_id = H5P_DATASET_CREATE_DEFAULT;

/* Form the request body to give the new Dataset its properties */
{
size_t tmp_len = 0;

if (RV_setup_dataset_create_request_body(obj, name, type_id, space_id, lcpl_id, dcpl_id,
&create_request_body, &tmp_len) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL,
"can't convert dataset creation parameters to JSON");

/* Check to make sure that the size of the create request HTTP body can safely be cast to a curl_off_t
*/
if (sizeof(curl_off_t) < sizeof(size_t))
ASSIGN_TO_SMALLER_SIZE(create_request_body_len, curl_off_t, tmp_len, size_t)
else if (sizeof(curl_off_t) > sizeof(size_t))
create_request_body_len = (curl_off_t)tmp_len;
else
ASSIGN_TO_SAME_SIZE_UNSIGNED_TO_SIGNED(create_request_body_len, curl_off_t, tmp_len, size_t)
}

/* Setup the host header */
host_header_len = strlen(parent->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers = curl_slist_append(curl_headers, strncat(host_header, parent->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

/* Instruct cURL that we are sending JSON */
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");
if (RV_setup_dataset_create_request_body(obj, name, type_id, space_id, lcpl_id, dcpl_id,
&create_request_body, &create_request_body_len) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL,
"can't convert dataset creation parameters to JSON");

/* Redirect cURL from the base URL to "/datasets" to create the dataset */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/datasets", base_URL)) < 0)
if ((url_len = snprintf(request_endpoint, URL_MAX_LENGTH, "/datasets")) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_SYSERRSTR, NULL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
Expand All @@ -248,33 +215,13 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na
printf("-> Dataset creation request URL: %s\n\n", request_url);
#endif

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_USERNAME, new_dataset->domain->u.file.server_info.username))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL username: %s", curl_err_buf);
if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_PASSWORD, new_dataset->domain->u.file.server_info.password))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL password: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL HTTP headers: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POST, 1))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set up cURL to make HTTP POST request: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDS, create_request_body))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL POST data: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, create_request_body_len))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL POST data size: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set cURL request URL: %s", curl_err_buf);
http_response = RV_curl_post(curl, &new_dataset->domain->u.file.server_info, request_endpoint,
new_dataset->domain->u.file.filepath_name, (const char *)create_request_body,
create_request_body_len, CONTENT_TYPE_JSON);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Creating dataset\n\n");

printf(" /***********************************\\\n");
printf("-> | Making POST request to the server |\n");
printf(" \\***********************************/\n\n");
#endif

CURL_PERFORM(curl, H5E_DATASET, H5E_CANTCREATE, NULL);
if (!HTTP_SUCCESS(http_response))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, NULL, "can't create dataset, response HTTP %ld",
http_response);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Created dataset\n\n");
Expand Down Expand Up @@ -309,19 +256,11 @@ RV_dataset_create(void *obj, const H5VL_loc_params_t *loc_params, const char *na

if (create_request_body)
RV_free(create_request_body);
if (host_header)
RV_free(host_header);

/* Clean up allocated dataset object if there was an issue */
if (new_dataset && !ret_value)
if (RV_dataset_close(new_dataset, FAIL, NULL) < 0)
FUNC_DONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close dataset");

if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
} /* end if */

PRINT_ERROR_STACK;

return ret_value;
Expand Down Expand Up @@ -1329,7 +1268,7 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
H5VL_file_specific_args_t vol_flush_args;
size_t host_header_len = 0;
char *host_header = NULL;
char request_url[URL_MAX_LENGTH];
char request_endpoint[URL_MAX_LENGTH];
const char *base_URL = NULL;

#ifdef RV_CONNECTOR_DEBUG
Expand All @@ -1342,8 +1281,6 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r

if (H5I_DATASET != dset->obj_type)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a dataset");
if ((base_URL = dset->domain->u.file.server_info.base_URL) == NULL)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dataset does not have valid server URL");

switch (args->op_type) {
/* H5Dget_access_plist */
Expand Down Expand Up @@ -1397,41 +1334,11 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "can't flush datase's domain");

/* Make GET request to dataset with 'verbose' parameter for HSDS. */
snprintf(request_url, URL_MAX_LENGTH, "%s%s%s%s", base_URL, "/datasets/", dset->URI,
"?verbose=1");
snprintf(request_endpoint, URL_MAX_LENGTH, "%s%s%s", "/datasets/", dset->URI, "?verbose=1");

/* Setup the host header */
host_header_len = strlen(dset->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL,
"can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers =
curl_slist_append(curl_headers, strncat(host_header, dset->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_USERNAME, dset->domain->u.file.server_info.username))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL username: %s", curl_err_buf);
if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_PASSWORD, dset->domain->u.file.server_info.password))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL password: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL HTTP headers: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPGET, 1))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL,
"can't set up cURL to make HTTP GET request: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set cURL request URL: %s",
curl_err_buf);

CURL_PERFORM(curl, H5E_DATASET, H5E_CANTGET, FAIL);
if (RV_curl_get(curl, &dset->domain->u.file.server_info, request_endpoint,
dset->domain->u.file.filepath_name, CONTENT_TYPE_JSON) < 0)
FUNC_GOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset");

if (RV_parse_allocated_size_callback(response_buffer.buffer, NULL,
args->args.get_storage_size.storage_size) < 0)
Expand All @@ -1456,13 +1363,6 @@ RV_dataset_get(void *obj, H5VL_dataset_get_args_t *args, hid_t dxpl_id, void **r
} /* end switch */

done:
if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
}

RV_free(host_header);

PRINT_ERROR_STACK;

return ret_value;
Expand Down
68 changes: 10 additions & 58 deletions src/rest_vol_datatype.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,21 +70,19 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
RV_object_t *new_datatype = NULL;
size_t commit_request_nalloc = 0;
size_t link_body_nalloc = 0;
size_t host_header_len = 0;
size_t datatype_body_len = 0;
size_t path_size = 0;
size_t path_len = 0;
const char *base_URL = NULL;
char *host_header = NULL;
char *commit_request_body = NULL;
char *datatype_body = NULL;
char *link_body = NULL;
char *path_dirname = NULL;
char request_url[URL_MAX_LENGTH];
char request_endpoint[URL_MAX_LENGTH];
int commit_request_len = 0;
int link_body_len = 0;
int url_len = 0;
void *ret_value = NULL;
long http_response;

#ifdef RV_CONNECTOR_DEBUG
printf("-> Received datatype commit call with following parameters:\n");
Expand All @@ -103,9 +101,6 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
if (H5I_FILE != parent->obj_type && H5I_GROUP != parent->obj_type)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object not a file or group");

if ((base_URL = parent->domain->u.file.server_info.base_URL) == NULL)
FUNC_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "parent object does not have valid server URL");

/* Check for write access */
if (!(parent->domain->u.file.intent & H5F_ACC_RDWR))
FUNC_GOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "no write intent on file");
Expand Down Expand Up @@ -231,61 +226,25 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
printf("-> Datatype commit request body:\n%s\n\n", commit_request_body);
#endif

/* Setup the host header */
host_header_len = strlen(parent->domain->u.file.filepath_name) + strlen(host_string) + 1;
if (NULL == (host_header = (char *)RV_malloc(host_header_len)))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, NULL, "can't allocate space for request Host header");

strcpy(host_header, host_string);

curl_headers = curl_slist_append(curl_headers, strncat(host_header, parent->domain->u.file.filepath_name,
host_header_len - strlen(host_string) - 1));

/* Disable use of Expect: 100 Continue HTTP response */
curl_headers = curl_slist_append(curl_headers, "Expect:");

/* Instruct cURL that we are sending JSON */
curl_headers = curl_slist_append(curl_headers, "Content-Type: application/json");

/* Redirect cURL from the base URL to "/datatypes" to commit the datatype */
if ((url_len = snprintf(request_url, URL_MAX_LENGTH, "%s/datatypes", base_URL)) < 0)
if ((url_len = snprintf(request_endpoint, URL_MAX_LENGTH, "/datatypes")) < 0)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, NULL, "snprintf error");

if (url_len >= URL_MAX_LENGTH)
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_SYSERRSTR, NULL,
"datatype create URL size exceeded maximum URL size");

#ifdef RV_CONNECTOR_DEBUG
printf("-> Datatype commit URL: %s\n\n", request_url);
printf("-> Datatype commit URL: %s\n\n", request_endpoint);
#endif

if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_USERNAME, new_datatype->domain->u.file.server_info.username))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL username: %s", curl_err_buf);
if (CURLE_OK !=
curl_easy_setopt(curl, CURLOPT_PASSWORD, new_datatype->domain->u.file.server_info.password))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL password: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL HTTP headers: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POST, 1))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set up cURL to make HTTP POST request: %s",
curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDS, commit_request_body))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL POST data: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)commit_request_len))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL POST data size: %s", curl_err_buf);
if (CURLE_OK != curl_easy_setopt(curl, CURLOPT_URL, request_url))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_CANTSET, NULL, "can't set cURL request URL: %s", curl_err_buf);
http_response = RV_curl_post(curl, &parent->domain->u.file.server_info, request_endpoint,
parent->domain->u.file.filepath_name, (const char *)commit_request_body,
(size_t)commit_request_len, CONTENT_TYPE_JSON);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Committing datatype\n\n");

printf(" /***********************************\\\n");
printf("-> | Making POST request to the server |\n");
printf(" \\***********************************/\n\n");
#endif

CURL_PERFORM(curl, H5E_DATATYPE, H5E_BADVALUE, NULL);
if (!HTTP_SUCCESS(http_response))
FUNC_GOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, NULL, "couldn't commit datatype: received HTTP %ld",
http_response);

#ifdef RV_CONNECTOR_DEBUG
printf("-> Committed datatype\n\n");
Expand Down Expand Up @@ -317,8 +276,6 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
RV_free(path_dirname);
if (commit_request_body)
RV_free(commit_request_body);
if (host_header)
RV_free(host_header);
if (datatype_body)
RV_free(datatype_body);
if (link_body)
Expand All @@ -329,11 +286,6 @@ RV_datatype_commit(void *obj, const H5VL_loc_params_t *loc_params, const char *n
if (RV_datatype_close(new_datatype, FAIL, NULL) < 0)
FUNC_DONE_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, NULL, "can't close datatype");

if (curl_headers) {
curl_slist_free_all(curl_headers);
curl_headers = NULL;
} /* end if */

PRINT_ERROR_STACK;

return ret_value;
Expand Down
Loading

0 comments on commit 420ff43

Please sign in to comment.