Skip to content

Commit

Permalink
Assorted io extension cleanups (pandas-dev#29704)
Browse files Browse the repository at this point in the history
  • Loading branch information
@WillAyd @jacobaustin123
WillAyd authored and jacobaustin123 committed Nov 20, 2019
1 parent 00cfdf5 commit 517221e
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 45 deletions.
1 change: 0 additions & 1 deletion pandas/_libs/src/parser/io.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ The full license is in the LICENSE file, distributed with this software.

#include "io.h"

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

Expand Down
51 changes: 13 additions & 38 deletions pandas/_libs/src/parser/tokenizer.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,38 +25,13 @@ GitHub. See Python Software Foundation License and BSD licenses for these.

#include "../headers/portable.h"

static void *safe_realloc(void *buffer, size_t size) {
void *result;
// OSX is weird.
// http://stackoverflow.com/questions/9560609/
// different-realloc-behaviour-in-linux-and-osx

result = realloc(buffer, size);
TRACE(("safe_realloc: buffer = %p, size = %zu, result = %p\n", buffer, size,
result))

return result;
}

void coliter_setup(coliter_t *self, parser_t *parser, int i, int start) {
// column i, starting at 0
self->words = parser->words;
self->col = i;
self->line_start = parser->line_start + start;
}

coliter_t *coliter_new(parser_t *self, int i) {
// column i, starting at 0
coliter_t *iter = (coliter_t *)malloc(sizeof(coliter_t));

if (NULL == iter) {
return NULL;
}

coliter_setup(iter, self, i, 0);
return iter;
}

static void free_if_not_null(void **ptr) {
TRACE(("free_if_not_null %p\n", *ptr))
if (*ptr != NULL) {
Expand All @@ -80,7 +55,7 @@ static void *grow_buffer(void *buffer, uint64_t length, uint64_t *capacity,
while ((length + space >= cap) && (newbuffer != NULL)) {
cap = cap ? cap << 1 : 2;
buffer = newbuffer;
newbuffer = safe_realloc(newbuffer, elsize * cap);
newbuffer = realloc(newbuffer, elsize * cap);
}

if (newbuffer == NULL) {
Expand Down Expand Up @@ -321,8 +296,8 @@ static int make_stream_space(parser_t *self, size_t nbytes) {
("make_stream_space: cap != self->words_cap, nbytes = %d, "
"self->words_cap=%d\n",
nbytes, self->words_cap))
newptr = safe_realloc((void *)self->word_starts,
sizeof(int64_t) * self->words_cap);
newptr = realloc((void *)self->word_starts,
sizeof(int64_t) * self->words_cap);
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
} else {
Expand All @@ -349,8 +324,8 @@ static int make_stream_space(parser_t *self, size_t nbytes) {
if (cap != self->lines_cap) {
TRACE(("make_stream_space: cap != self->lines_cap, nbytes = %d\n",
nbytes))
newptr = safe_realloc((void *)self->line_fields,
sizeof(int64_t) * self->lines_cap);
newptr = realloc((void *)self->line_fields,
sizeof(int64_t) * self->lines_cap);
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
} else {
Expand Down Expand Up @@ -427,7 +402,7 @@ static void append_warning(parser_t *self, const char *msg) {
snprintf(self->warn_msg, length + 1, "%s", msg);
} else {
ex_length = strlen(self->warn_msg);
newptr = safe_realloc(self->warn_msg, ex_length + length + 1);
newptr = realloc(self->warn_msg, ex_length + length + 1);
if (newptr != NULL) {
self->warn_msg = (char *)newptr;
snprintf(self->warn_msg + ex_length, length + 1, "%s", msg);
Expand Down Expand Up @@ -1290,13 +1265,13 @@ int parser_trim_buffers(parser_t *self) {
new_cap = _next_pow2(self->words_len) + 1;
if (new_cap < self->words_cap) {
TRACE(("parser_trim_buffers: new_cap < self->words_cap\n"));
newptr = safe_realloc((void *)self->words, new_cap * sizeof(char *));
newptr = realloc((void *)self->words, new_cap * sizeof(char *));
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
} else {
self->words = (char **)newptr;
}
newptr = safe_realloc((void *)self->word_starts,
newptr = realloc((void *)self->word_starts,
new_cap * sizeof(int64_t));
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
Expand All @@ -1315,13 +1290,13 @@ int parser_trim_buffers(parser_t *self) {
if (new_cap < self->stream_cap) {
TRACE(
("parser_trim_buffers: new_cap < self->stream_cap, calling "
"safe_realloc\n"));
newptr = safe_realloc((void *)self->stream, new_cap);
"realloc\n"));
newptr = realloc((void *)self->stream, new_cap);
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
} else {
// Update the pointers in the self->words array (char **) if
// `safe_realloc`
// `realloc`
// moved the `self->stream` buffer. This block mirrors a similar
// block in
// `make_stream_space`.
Expand All @@ -1342,14 +1317,14 @@ int parser_trim_buffers(parser_t *self) {
new_cap = _next_pow2(self->lines) + 1;
if (new_cap < self->lines_cap) {
TRACE(("parser_trim_buffers: new_cap < self->lines_cap\n"));
newptr = safe_realloc((void *)self->line_start,
newptr = realloc((void *)self->line_start,
new_cap * sizeof(int64_t));
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
} else {
self->line_start = (int64_t *)newptr;
}
newptr = safe_realloc((void *)self->line_fields,
newptr = realloc((void *)self->line_fields,
new_cap * sizeof(int64_t));
if (newptr == NULL) {
return PARSER_OUT_OF_MEMORY;
Expand Down
6 changes: 0 additions & 6 deletions pandas/_libs/src/parser/tokenizer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ See LICENSE for the license
#define PY_SSIZE_T_CLEAN
#include <Python.h>

#define ERROR_OK 0
#define ERROR_NO_DIGITS 1
#define ERROR_OVERFLOW 2
#define ERROR_INVALID_CHARS 3
Expand All @@ -32,10 +31,6 @@ See LICENSE for the license
#define CALLING_READ_FAILED 2


#if defined(_MSC_VER)
#define strtoll _strtoi64
#endif // _MSC_VER

/*
C flat file parsing low level code for pandas / NumPy
Expand Down Expand Up @@ -180,7 +175,6 @@ typedef struct coliter_t {
} coliter_t;

void coliter_setup(coliter_t *self, parser_t *parser, int i, int start);
coliter_t *coliter_new(parser_t *self, int i);

#define COLITER_NEXT(iter, word) \
do { \
Expand Down

0 comments on commit 517221e

Please sign in to comment.